Ruby 4.1.0dev (2026-09-08 revision 3c2856d9368f618a38cb7b86d7ac7c8006742d6b)
vm_insnhelper.c (3c2856d9368f618a38cb7b86d7ac7c8006742d6b)
1/**********************************************************************
2
3 vm_insnhelper.c - instruction helper functions. Included into vm.c.
4
5 $Author$
6
7 Copyright (C) 2007 Koichi Sasada
8
9**********************************************************************/
10
11#include "ruby/internal/config.h"
12
13#include <math.h>
14
15#ifdef HAVE_STDATOMIC_H
16 #include <stdatomic.h>
17#endif
18
19#include "constant.h"
20#include "debug_counter.h"
21#include "internal.h"
22#include "internal/class.h"
23#include "internal/compar.h"
24#include "internal/hash.h"
25#include "internal/numeric.h"
26#include "internal/proc.h"
27#include "internal/random.h"
28#include "internal/variable.h"
29#include "internal/set_table.h"
30#include "internal/struct.h"
31#include "ruby/thread.h"
32#include "variable.h"
33
34/* finish iseq array */
35#include "insns.inc"
36#include "insns_info.inc"
37
38extern rb_method_definition_t *rb_method_definition_create(rb_method_type_t type, ID mid);
39extern void rb_method_definition_set(const rb_method_entry_t *me, rb_method_definition_t *def, void *opts);
40extern int rb_method_definition_eq(const rb_method_definition_t *d1, const rb_method_definition_t *d2);
41extern VALUE rb_make_no_method_exception(VALUE exc, VALUE format, VALUE obj,
42 int argc, const VALUE *argv, int priv);
43
44static const struct rb_callcache vm_empty_cc;
45static const struct rb_callcache vm_empty_cc_for_super;
46
47/* control stack frame */
48
49static rb_control_frame_t *vm_get_ruby_level_caller_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp);
50
51VALUE
52ruby_vm_special_exception_copy(VALUE exc)
53{
54 VALUE e = rb_obj_alloc(rb_class_real(RBASIC_CLASS(exc)));
55 rb_obj_copy_ivar(e, exc);
56 return e;
57}
58
59NORETURN(static void ec_stack_overflow(rb_execution_context_t *ec, int));
60static void
61ec_stack_overflow(rb_execution_context_t *ec, int setup)
62{
63 VALUE mesg = rb_ec_vm_ptr(ec)->special_exceptions[ruby_error_sysstack];
64 ec->raised_flag = RAISED_STACKOVERFLOW;
65 if (setup) {
66 VALUE at = rb_ec_backtrace_object(ec);
67 mesg = ruby_vm_special_exception_copy(mesg);
68 rb_ivar_set(mesg, idBt, at);
69 rb_ivar_set(mesg, idBt_locations, at);
70 }
71 ec->errinfo = mesg;
72 EC_JUMP_TAG(ec, TAG_RAISE);
73}
74
75NORETURN(static void vm_stackoverflow(void));
76
77static void
78vm_stackoverflow(void)
79{
80 ec_stack_overflow(GET_EC(), TRUE);
81}
82
83void
84rb_ec_stack_overflow(rb_execution_context_t *ec, ruby_stack_overflow_critical_level crit)
85{
86 if (rb_during_gc()) {
87 rb_bug("system stack overflow during GC. Faulty native extension?");
88 }
89 if (crit >= rb_stack_overflow_fatal) {
90 ec->raised_flag = RAISED_STACKOVERFLOW;
91 ec->errinfo = rb_ec_vm_ptr(ec)->special_exceptions[ruby_error_stackfatal];
92 EC_JUMP_TAG(ec, TAG_RAISE);
93 }
94 ec_stack_overflow(ec, crit < rb_stack_overflow_signal);
95}
96
97static inline void stack_check(rb_execution_context_t *ec);
98
99#if VM_CHECK_MODE > 0
100static int
101callable_class_p(VALUE klass)
102{
103#if VM_CHECK_MODE >= 2
104 if (!klass) return FALSE;
105 switch (RB_BUILTIN_TYPE(klass)) {
106 default:
107 break;
108 case T_ICLASS:
109 if (!RB_TYPE_P(RCLASS_SUPER(klass), T_MODULE)) break;
110 case T_MODULE:
111 return TRUE;
112 }
113 while (klass) {
114 if (klass == rb_cBasicObject) {
115 return TRUE;
116 }
117 klass = RCLASS_SUPER(klass);
118 }
119 return FALSE;
120#else
121 return klass != 0;
122#endif
123}
124
125static int
126callable_method_entry_p(const rb_callable_method_entry_t *cme)
127{
128 if (cme == NULL) {
129 return TRUE;
130 }
131 else {
132 VM_ASSERT(IMEMO_TYPE_P((VALUE)cme, imemo_ment), "imemo_type:%s", rb_imemo_name(imemo_type((VALUE)cme)));
133
134 if (callable_class_p(cme->defined_class)) {
135 return TRUE;
136 }
137 else {
138 return FALSE;
139 }
140 }
141}
142
143static void
144vm_check_frame_detail(VALUE type, int req_block, int req_me, int req_cref, VALUE specval, VALUE cref_or_me, int is_cframe, const rb_iseq_t *iseq)
145{
146 unsigned int magic = (unsigned int)(type & VM_FRAME_MAGIC_MASK);
147 enum imemo_type cref_or_me_type = imemo_env; /* impossible value */
148
149 if (RB_TYPE_P(cref_or_me, T_IMEMO)) {
150 cref_or_me_type = imemo_type(cref_or_me);
151 }
152 if (type & VM_FRAME_FLAG_BMETHOD) {
153 req_me = TRUE;
154 }
155
156 if (req_block && (type & VM_ENV_FLAG_LOCAL) == 0) {
157 rb_bug("vm_push_frame: specval (%p) should be a block_ptr on %x frame", (void *)specval, magic);
158 }
159 if (!req_block && (type & VM_ENV_FLAG_LOCAL) != 0) {
160 rb_bug("vm_push_frame: specval (%p) should not be a block_ptr on %x frame", (void *)specval, magic);
161 }
162
163 if (req_me) {
164 if (cref_or_me_type != imemo_ment) {
165 rb_bug("vm_push_frame: (%s) should be method entry on %x frame", rb_obj_info(cref_or_me), magic);
166 }
167 }
168 else {
169 if (req_cref && cref_or_me_type != imemo_cref) {
170 rb_bug("vm_push_frame: (%s) should be CREF on %x frame", rb_obj_info(cref_or_me), magic);
171 }
172 else { /* cref or Qfalse */
173 if (cref_or_me != Qfalse && cref_or_me_type != imemo_cref) {
174 if (((type & VM_FRAME_FLAG_LAMBDA) || magic == VM_FRAME_MAGIC_IFUNC || magic == VM_FRAME_MAGIC_DUMMY) && (cref_or_me_type == imemo_ment)) {
175 /* ignore */
176 }
177 else {
178 rb_bug("vm_push_frame: (%s) should be false or cref on %x frame", rb_obj_info(cref_or_me), magic);
179 }
180 }
181 }
182 }
183
184 if (cref_or_me_type == imemo_ment) {
185 const rb_callable_method_entry_t *me = (const rb_callable_method_entry_t *)cref_or_me;
186
187 if (!callable_method_entry_p(me)) {
188 rb_bug("vm_push_frame: ment (%s) should be callable on %x frame.", rb_obj_info(cref_or_me), magic);
189 }
190 }
191
192 if ((type & VM_FRAME_MAGIC_MASK) == VM_FRAME_MAGIC_DUMMY) {
193 VM_ASSERT(iseq == NULL ||
194 RBASIC_CLASS((VALUE)iseq) == 0 || // dummy frame for loading
195 RUBY_VM_NORMAL_ISEQ_P(iseq) //argument error
196 );
197 }
198 else {
199 VM_ASSERT(is_cframe == !RUBY_VM_NORMAL_ISEQ_P(iseq));
200 }
201}
202
203static void
204vm_check_frame(VALUE type,
205 VALUE specval,
206 VALUE cref_or_me,
207 const rb_iseq_t *iseq)
208{
209 VALUE given_magic = type & VM_FRAME_MAGIC_MASK;
210 VM_ASSERT(FIXNUM_P(type));
211
212#define CHECK(magic, req_block, req_me, req_cref, is_cframe) \
213 case magic: \
214 vm_check_frame_detail(type, req_block, req_me, req_cref, \
215 specval, cref_or_me, is_cframe, iseq); \
216 break
217 switch (given_magic) {
218 /* BLK ME CREF CFRAME */
219 CHECK(VM_FRAME_MAGIC_METHOD, TRUE, TRUE, FALSE, FALSE);
220 CHECK(VM_FRAME_MAGIC_CLASS, TRUE, FALSE, TRUE, FALSE);
221 CHECK(VM_FRAME_MAGIC_TOP, TRUE, FALSE, TRUE, FALSE);
222 CHECK(VM_FRAME_MAGIC_CFUNC, TRUE, TRUE, FALSE, TRUE);
223 CHECK(VM_FRAME_MAGIC_BLOCK, FALSE, FALSE, FALSE, FALSE);
224 CHECK(VM_FRAME_MAGIC_IFUNC, FALSE, FALSE, FALSE, TRUE);
225 CHECK(VM_FRAME_MAGIC_EVAL, FALSE, FALSE, FALSE, FALSE);
226 CHECK(VM_FRAME_MAGIC_RESCUE, FALSE, FALSE, FALSE, FALSE);
227 CHECK(VM_FRAME_MAGIC_DUMMY, TRUE, FALSE, FALSE, FALSE);
228 default:
229 rb_bug("vm_push_frame: unknown type (%x)", (unsigned int)given_magic);
230 }
231#undef CHECK
232}
233
234static VALUE vm_stack_canary; /* Initialized later */
235static bool vm_stack_canary_was_born = false;
236
237// Return the index of the instruction right before the given PC.
238// This is needed because insn_entry advances PC before the insn body.
239static unsigned int
240previous_insn_index(const rb_iseq_t *iseq, const VALUE *pc)
241{
242 unsigned int pos = 0;
243 while (pos < ISEQ_BODY(iseq)->iseq_size) {
244 int opcode = rb_vm_insn_addr2opcode((void *)ISEQ_BODY(iseq)->iseq_encoded[pos]);
245 unsigned int next_pos = pos + insn_len(opcode);
246 if (ISEQ_BODY(iseq)->iseq_encoded + next_pos == pc) {
247 return pos;
248 }
249 pos = next_pos;
250 }
251 rb_bug("failed to find the previous insn");
252}
253
254void
255rb_vm_check_canary(const rb_execution_context_t *ec, VALUE *sp)
256{
257 const struct rb_control_frame_struct *reg_cfp = ec->cfp;
258 const struct rb_iseq_struct *iseq;
259
260 if (! LIKELY(vm_stack_canary_was_born)) {
261 return; /* :FIXME: isn't it rather fatal to enter this branch? */
262 }
263 else if ((VALUE *)reg_cfp == ec->vm_stack + ec->vm_stack_size) {
264 /* This is at the very beginning of a thread. cfp does not exist. */
265 return;
266 }
267 else if (! (iseq = GET_ISEQ())) {
268 return;
269 }
270 else if (LIKELY(sp[0] != vm_stack_canary)) {
271 return;
272 }
273 else {
274 /* we are going to call methods below; squash the canary to
275 * prevent infinite loop. */
276 sp[0] = Qundef;
277 }
278
279 const VALUE *orig = rb_iseq_original_iseq(iseq);
280 const VALUE iseqw = rb_iseqw_new(iseq);
281 const VALUE inspection = rb_inspect(iseqw);
282 const char *stri = rb_str_to_cstr(inspection);
283 const VALUE disasm = rb_iseq_disasm(iseq);
284 const char *strd = rb_str_to_cstr(disasm);
285 const ptrdiff_t pos = previous_insn_index(iseq, GET_PC());
286 const enum ruby_vminsn_type insn = (enum ruby_vminsn_type)orig[pos];
287 const char *name = insn_name(insn);
288
289 /* rb_bug() is not capable of outputting this large contents. It
290 is designed to run form a SIGSEGV handler, which tends to be
291 very restricted. */
292 ruby_debug_printf(
293 "We are killing the stack canary set by %s, "
294 "at %s@pc=%"PRIdPTR"\n"
295 "watch out the C stack trace.\n"
296 "%s",
297 name, stri, pos, strd);
298 rb_bug("see above.");
299}
300#define vm_check_canary(ec, sp) rb_vm_check_canary(ec, sp)
301
302#else
303#define vm_check_canary(ec, sp)
304#define vm_check_frame(a, b, c, d)
305#endif /* VM_CHECK_MODE > 0 */
306
307#if USE_DEBUG_COUNTER
308static void
309vm_push_frame_debug_counter_inc(
310 const struct rb_execution_context_struct *ec,
311 const struct rb_control_frame_struct *reg_cfp,
312 VALUE type)
313{
314 const struct rb_control_frame_struct *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(reg_cfp);
315
316 RB_DEBUG_COUNTER_INC(frame_push);
317
318 if (RUBY_VM_END_CONTROL_FRAME(ec) != prev_cfp) {
319 const bool curr = VM_FRAME_RUBYFRAME_P(reg_cfp);
320 const bool prev = VM_FRAME_RUBYFRAME_P(prev_cfp);
321 if (prev) {
322 if (curr) {
323 RB_DEBUG_COUNTER_INC(frame_R2R);
324 }
325 else {
326 RB_DEBUG_COUNTER_INC(frame_R2C);
327 }
328 }
329 else {
330 if (curr) {
331 RB_DEBUG_COUNTER_INC(frame_C2R);
332 }
333 else {
334 RB_DEBUG_COUNTER_INC(frame_C2C);
335 }
336 }
337 }
338
339 switch (type & VM_FRAME_MAGIC_MASK) {
340 case VM_FRAME_MAGIC_METHOD: RB_DEBUG_COUNTER_INC(frame_push_method); return;
341 case VM_FRAME_MAGIC_BLOCK: RB_DEBUG_COUNTER_INC(frame_push_block); return;
342 case VM_FRAME_MAGIC_CLASS: RB_DEBUG_COUNTER_INC(frame_push_class); return;
343 case VM_FRAME_MAGIC_TOP: RB_DEBUG_COUNTER_INC(frame_push_top); return;
344 case VM_FRAME_MAGIC_CFUNC: RB_DEBUG_COUNTER_INC(frame_push_cfunc); return;
345 case VM_FRAME_MAGIC_IFUNC: RB_DEBUG_COUNTER_INC(frame_push_ifunc); return;
346 case VM_FRAME_MAGIC_EVAL: RB_DEBUG_COUNTER_INC(frame_push_eval); return;
347 case VM_FRAME_MAGIC_RESCUE: RB_DEBUG_COUNTER_INC(frame_push_rescue); return;
348 case VM_FRAME_MAGIC_DUMMY: RB_DEBUG_COUNTER_INC(frame_push_dummy); return;
349 }
350
351 rb_bug("unreachable");
352}
353#else
354#define vm_push_frame_debug_counter_inc(ec, cfp, t) /* void */
355#endif
356
357// Return a poison value to be set above the stack top to verify leafness.
358VALUE
359rb_vm_stack_canary(void)
360{
361#if VM_CHECK_MODE > 0
362 return vm_stack_canary;
363#else
364 return 0;
365#endif
366}
367
368STATIC_ASSERT(VM_ENV_DATA_INDEX_ME_CREF, VM_ENV_DATA_INDEX_ME_CREF == -2);
369STATIC_ASSERT(VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL == -1);
370STATIC_ASSERT(VM_ENV_DATA_INDEX_FLAGS, VM_ENV_DATA_INDEX_FLAGS == -0);
371
372static void
373vm_push_frame(rb_execution_context_t *ec,
374 const rb_iseq_t *iseq,
375 VALUE type,
376 VALUE self,
377 VALUE specval,
378 VALUE cref_or_me,
379 const VALUE *pc,
380 VALUE *sp,
381 int local_size,
382 int stack_max)
383{
384 rb_control_frame_t *const cfp = RUBY_VM_NEXT_CONTROL_FRAME(ec->cfp);
385
386 vm_check_frame(type, specval, cref_or_me, iseq);
387 VM_ASSERT(local_size >= 0);
388
389 /* check stack overflow */
390 CHECK_VM_STACK_OVERFLOW0(cfp, sp, local_size + stack_max);
391 vm_check_canary(ec, sp);
392
393 /* setup vm value stack */
394
395 /* initialize local variables */
396 for (int i=0; i < local_size; i++) {
397 *sp++ = Qnil;
398 }
399
400 /* setup ep with managing data */
401 *sp++ = cref_or_me; /* ep[-2] / Qnil or T_IMEMO(cref) or T_IMEMO(ment) */
402 *sp++ = specval /* ep[-1] / block handler or prev env ptr */;
403 *sp++ = type; /* ep[-0] / ENV_FLAGS */
404
405 /* setup new frame */
406 *cfp = (const struct rb_control_frame_struct) {
407 .pc = pc,
408 .sp = sp,
409 ._iseq = iseq,
410 .self = self,
411 .ep = sp - 1,
412 .block_code = NULL,
413#if VM_DEBUG_BP_CHECK
414 .bp_check = sp,
415#endif
416 .jit_return = NULL,
417 };
418
419 /* Ensure the initialization of `*cfp` above never gets reordered with the update of `ec->cfp` below.
420 This is a no-op in all cases we've looked at (https://godbolt.org/z/3oxd1446K), but should guarantee it for all
421 future/untested compilers/platforms. */
422
423 #if defined HAVE_DECL_ATOMIC_SIGNAL_FENCE && HAVE_DECL_ATOMIC_SIGNAL_FENCE
424 atomic_signal_fence(memory_order_seq_cst);
425 #endif
426
427 ec->cfp = cfp;
428
429 if (VMDEBUG == 2) {
430 SDR();
431 }
432 vm_push_frame_debug_counter_inc(ec, cfp, type);
433}
434
435void
436rb_vm_pop_frame_no_int(rb_execution_context_t *ec)
437{
438 rb_control_frame_t *cfp = ec->cfp;
439
440 if (VMDEBUG == 2) SDR();
441
442 ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
443}
444
445/* return TRUE if the frame is finished */
446static inline int
447vm_pop_frame(rb_execution_context_t *ec, rb_control_frame_t *cfp, const VALUE *ep)
448{
449 VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS];
450
451 if (VMDEBUG == 2) SDR();
452
453 RUBY_VM_CHECK_INTS(ec);
454 ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
455
456 return flags & VM_FRAME_FLAG_FINISH;
457}
458
459void
460rb_vm_pop_frame(rb_execution_context_t *ec)
461{
462 vm_pop_frame(ec, ec->cfp, ec->cfp->ep);
463}
464
465// it pushes pseudo-frame with fname filename.
466VALUE
467rb_vm_push_frame_fname(rb_execution_context_t *ec, VALUE fname)
468{
469 rb_iseq_t *rb_iseq_alloc_with_dummy_path(VALUE fname);
470 rb_iseq_t *dmy_iseq = rb_iseq_alloc_with_dummy_path(fname);
471
472 vm_push_frame(ec,
473 dmy_iseq, //const rb_iseq_t *iseq,
474 VM_FRAME_MAGIC_DUMMY | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, // VALUE type,
475 ec->cfp->self, // VALUE self,
476 VM_BLOCK_HANDLER_NONE, // VALUE specval,
477 Qfalse, // VALUE cref_or_me,
478 NULL, // const VALUE *pc,
479 ec->cfp->sp, // VALUE *sp,
480 0, // int local_size,
481 0); // int stack_max
482
483 return (VALUE)dmy_iseq;
484}
485
486/* method dispatch */
487static inline VALUE
488rb_arity_error_new(int argc, int min, int max)
489{
490 VALUE err_mess = rb_sprintf("wrong number of arguments (given %d, expected %d", argc, min);
491 if (min == max) {
492 /* max is not needed */
493 }
494 else if (max == UNLIMITED_ARGUMENTS) {
495 rb_str_cat_cstr(err_mess, "+");
496 }
497 else {
498 rb_str_catf(err_mess, "..%d", max);
499 }
500 rb_str_cat_cstr(err_mess, ")");
501 return rb_exc_new3(rb_eArgError, err_mess);
502}
503
504void
505rb_error_arity(int argc, int min, int max)
506{
507 rb_exc_raise(rb_arity_error_new(argc, min, max));
508}
509
510/* lvar */
511
512NOINLINE(static void vm_env_write_slowpath(const VALUE *ep, int index, VALUE v));
513
514static void
515vm_env_write_slowpath(const VALUE *ep, int index, VALUE v)
516{
517 const VALUE envval = VM_ENV_ENVVAL(ep);
518
519 if (RB_FL_TEST_RAW(envval, RUBY_FL_SHAREABLE)) {
520 /* Writing to a SHAREABLE env (an isolated proc) can create a shareable ->
521 * unshareable edge; only a full barrier sets the shref bit keeping v alive
522 * through its owner's local GC. WB_REQUIRED stays: later writes need it too. */
523 if (!SPECIAL_CONST_P(v)) {
524 rb_gc_writebarrier(envval, v);
525 }
526 VM_FORCE_WRITE(&ep[index], v);
527 }
528 else {
529 /* remember env value forcely */
530 rb_gc_writebarrier_remember(envval);
531 VM_FORCE_WRITE(&ep[index], v);
532 VM_ENV_FLAGS_UNSET(ep, VM_ENV_FLAG_WB_REQUIRED);
533 }
534 RB_DEBUG_COUNTER_INC(lvar_set_slowpath);
535}
536
537// YJIT assumes this function never runs GC
538static inline void
539vm_env_write(const VALUE *ep, int index, VALUE v)
540{
541 VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS];
542 if (LIKELY((flags & VM_ENV_FLAG_WB_REQUIRED) == 0)) {
543 VM_STACK_ENV_WRITE(ep, index, v);
544 }
545 else {
546 vm_env_write_slowpath(ep, index, v);
547 }
548}
549
550void
551rb_vm_env_write(const VALUE *ep, int index, VALUE v)
552{
553 vm_env_write(ep, index, v);
554}
555
556VALUE
557rb_vm_bh_to_procval(const rb_execution_context_t *ec, VALUE block_handler)
558{
559 if (block_handler == VM_BLOCK_HANDLER_NONE) {
560 return Qnil;
561 }
562 else {
563 switch (vm_block_handler_type(block_handler)) {
564 case block_handler_type_iseq:
565 case block_handler_type_ifunc:
566 return rb_vm_make_proc(ec, VM_BH_TO_CAPT_BLOCK(block_handler), rb_cProc);
567 case block_handler_type_symbol:
568 return rb_sym_to_proc(VM_BH_TO_SYMBOL(block_handler));
569 case block_handler_type_proc:
570 return VM_BH_TO_PROC(block_handler);
571 default:
572 VM_UNREACHABLE(rb_vm_bh_to_procval);
573 }
574 }
575}
576
577/* svar */
578
579#if VM_CHECK_MODE > 0
580static int
581vm_svar_valid_p(VALUE svar)
582{
583 if (RB_TYPE_P((VALUE)svar, T_IMEMO)) {
584 switch (imemo_type(svar)) {
585 case imemo_svar:
586 case imemo_cref:
587 case imemo_ment:
588 return TRUE;
589 default:
590 break;
591 }
592 }
593 rb_bug("vm_svar_valid_p: unknown type: %s", rb_obj_info(svar));
594 return FALSE;
595}
596#endif
597
598/* Should this frame's special variables live in the env's svar slot? A SHAREABLE env
599 * (isolated proc) can run in several Ractors at once, making svar shared mutable state
600 * leaking $~/$_ across Ractors: keep them per-EC instead. */
601static inline bool
602lep_svar_in_env_p(const rb_execution_context_t *ec, const VALUE *lep)
603{
604 if (!lep) return false;
605 if (ec == NULL) return true;
606 if (ec->root_lep == lep) return false;
607 /* lep may be a stale on-stack ep of a frame whose env already escaped: lep[0]
608 * still holds the imemo_env, so flags is not a FIXNUM and VM_ENV_ESCAPED_P
609 * asserts. That is not a live shareable proc's env, so fall back to in-env. */
610 if (FIXNUM_P(lep[VM_ENV_DATA_INDEX_FLAGS]) &&
611 VM_ENV_ESCAPED_P(lep) &&
612 RB_FL_TEST_RAW(VM_ENV_ENVVAL(lep), RUBY_FL_SHAREABLE)) {
613 return false;
614 }
615 return true;
616}
617
618static inline struct vm_svar *
619lep_svar(const rb_execution_context_t *ec, const VALUE *lep)
620{
621 VALUE svar;
622
623 if (lep_svar_in_env_p(ec, lep)) {
624 svar = lep[VM_ENV_DATA_INDEX_ME_CREF];
625 }
626 else {
627 svar = ec->root_svar;
628 }
629
630 VM_ASSERT(svar == Qfalse || vm_svar_valid_p(svar));
631
632 return (struct vm_svar *)svar;
633}
634
635static inline void
636lep_svar_write(const rb_execution_context_t *ec, const VALUE *lep, const struct vm_svar *svar)
637{
638 VM_ASSERT(vm_svar_valid_p((VALUE)svar));
639
640 if (lep_svar_in_env_p(ec, lep)) {
641 vm_env_write(lep, VM_ENV_DATA_INDEX_ME_CREF, (VALUE)svar);
642 }
643 else {
644 RB_OBJ_WRITE(rb_ec_thread_ptr(ec)->self, &ec->root_svar, svar);
645 }
646}
647
648static VALUE
649lep_svar_get(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key)
650{
651 const struct vm_svar *svar = lep_svar(ec, lep);
652
653 if ((VALUE)svar == Qfalse || imemo_type((VALUE)svar) != imemo_svar) return Qnil;
654
655 switch (key) {
656 case VM_SVAR_LASTLINE:
657 return svar->lastline;
658 case VM_SVAR_BACKREF:
659 return svar->backref;
660 default: {
661 const VALUE ary = svar->others;
662
663 if (NIL_P(ary)) {
664 return Qnil;
665 }
666 else {
667 return rb_ary_entry(ary, key - VM_SVAR_EXTRA_START);
668 }
669 }
670 }
671}
672
673static struct vm_svar *
674svar_new(VALUE obj)
675{
676 struct vm_svar *svar = IMEMO_NEW(struct vm_svar, imemo_svar, obj);
677 *((VALUE *)&svar->lastline) = Qnil;
678 *((VALUE *)&svar->backref) = Qnil;
679 *((VALUE *)&svar->others) = Qnil;
680
681 return svar;
682}
683
684static void
685lep_svar_set(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key, VALUE val)
686{
687 struct vm_svar *svar = lep_svar(ec, lep);
688
689 if ((VALUE)svar == Qfalse || imemo_type((VALUE)svar) != imemo_svar) {
690 lep_svar_write(ec, lep, svar = svar_new((VALUE)svar));
691 }
692
693 switch (key) {
694 case VM_SVAR_LASTLINE:
695 RB_OBJ_WRITE(svar, &svar->lastline, val);
696 return;
697 case VM_SVAR_BACKREF:
698 RB_OBJ_WRITE(svar, &svar->backref, val);
699 return;
700 default: {
701 VALUE ary = svar->others;
702
703 if (NIL_P(ary)) {
704 RB_OBJ_WRITE(svar, &svar->others, ary = rb_ary_new());
705 }
706 rb_ary_store(ary, key - VM_SVAR_EXTRA_START, val);
707 }
708 }
709}
710
711static inline VALUE
712vm_getspecial(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key, rb_num_t type)
713{
714 VALUE val;
715
716 if (type == 0) {
717 val = lep_svar_get(ec, lep, key);
718 }
719 else {
720 VALUE backref = lep_svar_get(ec, lep, VM_SVAR_BACKREF);
721
722 if (type & 0x01) {
723 switch (type >> 1) {
724 case '&':
725 val = rb_reg_last_match(backref);
726 break;
727 case '`':
728 val = rb_reg_match_pre(backref);
729 break;
730 case '\'':
731 val = rb_reg_match_post(backref);
732 break;
733 case '+':
734 val = rb_reg_match_last(backref);
735 break;
736 default:
737 rb_bug("unexpected back-ref");
738 }
739 }
740 else {
741 val = rb_reg_nth_match((int)(type >> 1), backref);
742 }
743 }
744 return val;
745}
746
747static inline VALUE
748vm_backref_defined(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t type)
749{
750 VALUE backref = lep_svar_get(ec, lep, VM_SVAR_BACKREF);
751 int nth = 0;
752
753 if (type & 0x01) {
754 switch (type >> 1) {
755 case '&':
756 case '`':
757 case '\'':
758 break;
759 case '+':
760 return rb_reg_last_defined(backref);
761 default:
762 rb_bug("unexpected back-ref");
763 }
764 }
765 else {
766 nth = (int)(type >> 1);
767 }
768 return rb_reg_nth_defined(nth, backref);
769}
770
771PUREFUNC(static rb_callable_method_entry_t *check_method_entry(VALUE obj, int can_be_svar));
772static rb_callable_method_entry_t *
773check_method_entry(VALUE obj, int can_be_svar)
774{
775 if (obj == Qfalse) return NULL;
776
777#if VM_CHECK_MODE > 0
778 if (!RB_TYPE_P(obj, T_IMEMO)) rb_bug("check_method_entry: unknown type: %s", rb_obj_info(obj));
779#endif
780
781 switch (imemo_type(obj)) {
782 case imemo_ment:
783 return (rb_callable_method_entry_t *)obj;
784 case imemo_cref:
785 return NULL;
786 case imemo_svar:
787 if (can_be_svar) {
788 return check_method_entry(((struct vm_svar *)obj)->cref_or_me, FALSE);
789 }
790 default:
791#if VM_CHECK_MODE > 0
792 rb_bug("check_method_entry: svar should not be there:");
793#endif
794 return NULL;
795 }
796}
797
798static rb_callable_method_entry_t *
799env_method_entry_unchecked(VALUE obj, int can_be_svar)
800{
801 if (obj == Qfalse) return NULL;
802
803 switch (imemo_type(obj)) {
804 case imemo_ment:
805 return (rb_callable_method_entry_t *)obj;
806 case imemo_cref:
807 return NULL;
808 case imemo_svar:
809 if (can_be_svar) {
810 return env_method_entry_unchecked(((struct vm_svar *)obj)->cref_or_me, FALSE);
811 }
812 default:
813 return NULL;
814 }
815}
816
817const rb_callable_method_entry_t *
818rb_vm_frame_method_entry(const rb_control_frame_t *cfp)
819{
820 const VALUE *ep = cfp->ep;
821 rb_callable_method_entry_t *me;
822
823 while (!VM_ENV_LOCAL_P(ep)) {
824 if ((me = check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return me;
825 ep = VM_ENV_PREV_EP(ep);
826 }
827
828 return check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
829}
830
831const rb_callable_method_entry_t *
832rb_vm_frame_method_entry_unchecked(const rb_control_frame_t *cfp)
833{
834 const VALUE *ep = cfp->ep;
835 rb_callable_method_entry_t *me;
836
837 while (!VM_ENV_LOCAL_P_UNCHECKED(ep)) {
838 if ((me = env_method_entry_unchecked(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return me;
839 ep = VM_ENV_PREV_EP_UNCHECKED(ep);
840 }
841
842 return env_method_entry_unchecked(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
843}
844
845static const rb_iseq_t *
846method_entry_iseqptr(const rb_callable_method_entry_t *me)
847{
848 switch (me->def->type) {
849 case VM_METHOD_TYPE_ISEQ:
850 return me->def->body.iseq.iseqptr;
851 default:
852 return NULL;
853 }
854}
855
856static rb_cref_t *
857method_entry_cref(const rb_callable_method_entry_t *me)
858{
859 switch (me->def->type) {
860 case VM_METHOD_TYPE_ISEQ:
861 return me->def->body.iseq.cref;
862 default:
863 return NULL;
864 }
865}
866
867#if VM_CHECK_MODE == 0
868PUREFUNC(static rb_cref_t *check_cref(VALUE, int));
869#endif
870static rb_cref_t *
871check_cref(VALUE obj, int can_be_svar)
872{
873 if (obj == Qfalse) return NULL;
874
875#if VM_CHECK_MODE > 0
876 if (!RB_TYPE_P(obj, T_IMEMO)) rb_bug("check_cref: unknown type: %s", rb_obj_info(obj));
877#endif
878
879 switch (imemo_type(obj)) {
880 case imemo_ment:
881 return method_entry_cref((rb_callable_method_entry_t *)obj);
882 case imemo_cref:
883 return (rb_cref_t *)obj;
884 case imemo_svar:
885 if (can_be_svar) {
886 return check_cref(((struct vm_svar *)obj)->cref_or_me, FALSE);
887 }
888 default:
889#if VM_CHECK_MODE > 0
890 rb_bug("check_method_entry: svar should not be there:");
891#endif
892 return NULL;
893 }
894}
895
896static inline rb_cref_t *
897vm_env_cref(const VALUE *ep)
898{
899 rb_cref_t *cref;
900
901 while (!VM_ENV_LOCAL_P(ep)) {
902 if ((cref = check_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return cref;
903 ep = VM_ENV_PREV_EP(ep);
904 }
905
906 return check_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
907}
908
909static int
910is_cref(const VALUE v, int can_be_svar)
911{
912 if (RB_TYPE_P(v, T_IMEMO)) {
913 switch (imemo_type(v)) {
914 case imemo_cref:
915 return TRUE;
916 case imemo_svar:
917 if (can_be_svar) return is_cref(((struct vm_svar *)v)->cref_or_me, FALSE);
918 default:
919 break;
920 }
921 }
922 return FALSE;
923}
924
925static int
926vm_env_cref_by_cref(const VALUE *ep)
927{
928 while (!VM_ENV_LOCAL_P(ep)) {
929 if (is_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) return TRUE;
930 ep = VM_ENV_PREV_EP(ep);
931 }
932 return is_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
933}
934
935static rb_cref_t *
936cref_replace_with_duplicated_cref_each_frame(const VALUE *vptr, int can_be_svar, VALUE parent)
937{
938 const VALUE v = *vptr;
939 rb_cref_t *cref, *new_cref;
940
941 if (RB_TYPE_P(v, T_IMEMO)) {
942 switch (imemo_type(v)) {
943 case imemo_cref:
944 cref = (rb_cref_t *)v;
945 new_cref = rb_vm_cref_dup(cref);
946 if (parent) {
947 RB_OBJ_WRITE(parent, vptr, new_cref);
948 }
949 else {
950 VM_FORCE_WRITE(vptr, (VALUE)new_cref);
951 }
952 return (rb_cref_t *)new_cref;
953 case imemo_svar:
954 if (can_be_svar) {
955 return cref_replace_with_duplicated_cref_each_frame(&((struct vm_svar *)v)->cref_or_me, FALSE, v);
956 }
957 /* fall through */
958 case imemo_ment:
959 rb_bug("cref_replace_with_duplicated_cref_each_frame: unreachable");
960 default:
961 break;
962 }
963 }
964 return NULL;
965}
966
967static rb_cref_t *
968vm_cref_replace_with_duplicated_cref(const VALUE *ep)
969{
970 if (vm_env_cref_by_cref(ep)) {
971 rb_cref_t *cref;
972 VALUE envval;
973
974 while (!VM_ENV_LOCAL_P(ep)) {
975 envval = VM_ENV_ESCAPED_P(ep) ? VM_ENV_ENVVAL(ep) : Qfalse;
976 if ((cref = cref_replace_with_duplicated_cref_each_frame(&ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE, envval)) != NULL) {
977 return cref;
978 }
979 ep = VM_ENV_PREV_EP(ep);
980 }
981 envval = VM_ENV_ESCAPED_P(ep) ? VM_ENV_ENVVAL(ep) : Qfalse;
982 return cref_replace_with_duplicated_cref_each_frame(&ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE, envval);
983 }
984 else {
985 rb_bug("vm_cref_dup: unreachable");
986 }
987}
988
989static rb_cref_t *
990vm_get_cref(const VALUE *ep)
991{
992 rb_cref_t *cref = vm_env_cref(ep);
993
994 if (cref != NULL) {
995 return cref;
996 }
997 else {
998 rb_bug("vm_get_cref: unreachable");
999 }
1000}
1001
1002rb_cref_t *
1003rb_vm_get_cref(const VALUE *ep)
1004{
1005 return vm_get_cref(ep);
1006}
1007
1008static rb_cref_t *
1009vm_ec_cref(const rb_execution_context_t *ec)
1010{
1011 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
1012
1013 if (cfp == NULL) {
1014 return NULL;
1015 }
1016 return vm_get_cref(cfp->ep);
1017}
1018
1019static const rb_cref_t *
1020vm_get_const_key_cref(const VALUE *ep)
1021{
1022 const rb_cref_t *cref = vm_get_cref(ep);
1023 const rb_cref_t *key_cref = cref;
1024
1025 while (cref) {
1026 if (CREF_DYNAMIC(cref)) {
1027 return key_cref;
1028 }
1029 cref = CREF_NEXT(cref);
1030 }
1031
1032 /* no dynamic singleton class or cloned class found */
1033 return NULL;
1034}
1035
1036static rb_cref_t *
1037vm_cref_push(const rb_execution_context_t *ec, VALUE klass, const VALUE *ep, int pushed_by_eval, int singleton)
1038{
1039 rb_cref_t *prev_cref = NULL;
1040
1041 if (ep) {
1042 prev_cref = vm_env_cref(ep);
1043 }
1044 else {
1045 rb_control_frame_t *cfp = vm_get_ruby_level_caller_cfp(ec, ec->cfp);
1046
1047 if (cfp) {
1048 prev_cref = vm_env_cref(cfp->ep);
1049 }
1050 }
1051
1052 return vm_cref_new(klass, METHOD_VISI_PUBLIC, FALSE, prev_cref, pushed_by_eval, singleton);
1053}
1054
1055static inline VALUE
1056vm_get_cbase(const VALUE *ep)
1057{
1058 const rb_cref_t *cref = vm_get_cref(ep);
1059
1060 return CREF_CLASS_FOR_DEFINITION(cref);
1061}
1062
1063static inline VALUE
1064vm_get_const_base(const VALUE *ep)
1065{
1066 const rb_cref_t *cref = vm_get_cref(ep);
1067
1068 while (cref) {
1069 if (!CREF_PUSHED_BY_EVAL(cref)) {
1070 return CREF_CLASS_FOR_DEFINITION(cref);
1071 }
1072 cref = CREF_NEXT(cref);
1073 }
1074
1075 return Qundef;
1076}
1077
1078static inline void
1079vm_check_if_namespace(VALUE klass)
1080{
1081 if (!RB_TYPE_P(klass, T_CLASS) && !RB_TYPE_P(klass, T_MODULE)) {
1082 rb_raise(rb_eTypeError, "%+"PRIsVALUE" is not a class/module", klass);
1083 }
1084}
1085
1086static inline void
1087vm_ensure_not_refinement_module(VALUE self)
1088{
1089 if (RB_TYPE_P(self, T_MODULE) && FL_TEST(self, RMODULE_IS_REFINEMENT)) {
1090 rb_warn("not defined at the refinement, but at the outer class/module");
1091 }
1092}
1093
1094static inline VALUE
1095vm_get_iclass(const rb_control_frame_t *cfp, VALUE klass)
1096{
1097 return klass;
1098}
1099
1100static inline VALUE
1101vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, bool allow_nil, int is_defined)
1102{
1103 void rb_const_warn_if_deprecated(const rb_const_entry_t *ce, VALUE klass, ID id);
1104 VALUE val;
1105
1106 if (NIL_P(orig_klass) && allow_nil) {
1107 /* in current lexical scope */
1108 const rb_cref_t *root_cref = vm_get_cref(ec->cfp->ep);
1109 const rb_cref_t *cref;
1110 VALUE klass = Qnil;
1111
1112 while (root_cref && CREF_PUSHED_BY_EVAL(root_cref)) {
1113 root_cref = CREF_NEXT(root_cref);
1114 }
1115 cref = root_cref;
1116 while (cref && CREF_NEXT(cref)) {
1117 if (CREF_PUSHED_BY_EVAL(cref)) {
1118 klass = Qnil;
1119 }
1120 else {
1121 klass = CREF_CLASS(cref);
1122 }
1123 cref = CREF_NEXT(cref);
1124
1125 if (!NIL_P(klass)) {
1126 VALUE av, am = 0;
1127 rb_const_entry_t *ce;
1128 search_continue:
1129 if ((ce = rb_const_lookup(klass, id))) {
1130 rb_const_warn_if_deprecated(ce, klass, id);
1131 val = ce->value;
1132 if (UNDEF_P(val)) {
1133 if (am == klass) break;
1134 am = klass;
1135 if (is_defined) return 1;
1136 if (rb_autoloading_value(klass, id, &av, NULL)) return av;
1137 rb_autoload_load(klass, id);
1138 goto search_continue;
1139 }
1140 else {
1141 if (is_defined) {
1142 return 1;
1143 }
1144 else {
1145 if (UNLIKELY(!rb_ractor_main_p())) {
1146 if (!rb_ractor_shareable_p(val)) {
1147 rb_raise(rb_eRactorIsolationError,
1148 "can not access non-shareable objects in constant %"PRIsVALUE"::%"PRIsVALUE" by non-main ractor.", rb_class_path(klass), rb_id2str(id));
1149 }
1150 }
1151 return val;
1152 }
1153 }
1154 }
1155 }
1156 }
1157
1158 /* search self */
1159 if (root_cref && !NIL_P(CREF_CLASS(root_cref))) {
1160 klass = vm_get_iclass(ec->cfp, CREF_CLASS(root_cref));
1161 }
1162 else {
1163 klass = CLASS_OF(ec->cfp->self);
1164 }
1165
1166 if (is_defined) {
1167 return rb_const_defined(klass, id);
1168 }
1169 else {
1170 return rb_const_get(klass, id);
1171 }
1172 }
1173 else {
1174 vm_check_if_namespace(orig_klass);
1175 if (is_defined) {
1176 return rb_public_const_defined_from(orig_klass, id);
1177 }
1178 else {
1179 return rb_public_const_get_from(orig_klass, id);
1180 }
1181 }
1182}
1183
1184VALUE
1185rb_vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, VALUE allow_nil)
1186{
1187 return vm_get_ev_const(ec, orig_klass, id, allow_nil == Qtrue, 0);
1188}
1189
1190static inline VALUE
1191vm_get_ev_const_chain(rb_execution_context_t *ec, const ID *segments)
1192{
1193 VALUE val = Qnil;
1194 int idx = 0;
1195 int allow_nil = TRUE;
1196 if (segments[0] == idNULL) {
1197 val = rb_cObject;
1198 idx++;
1199 allow_nil = FALSE;
1200 }
1201 while (segments[idx]) {
1202 ID id = segments[idx++];
1203 val = vm_get_ev_const(ec, val, id, allow_nil, 0);
1204 allow_nil = FALSE;
1205 }
1206 return val;
1207}
1208
1209
1210static inline VALUE
1211vm_get_cvar_base(const rb_cref_t *cref, const rb_control_frame_t *cfp, int top_level_raise)
1212{
1213 VALUE klass;
1214
1215 if (!cref) {
1216 rb_bug("vm_get_cvar_base: no cref");
1217 }
1218
1219 while (CREF_NEXT(cref) &&
1220 (NIL_P(CREF_CLASS(cref)) || RCLASS_SINGLETON_P(CREF_CLASS(cref)) ||
1221 CREF_PUSHED_BY_EVAL(cref) || CREF_SINGLETON(cref))) {
1222 cref = CREF_NEXT(cref);
1223 }
1224 if (top_level_raise && !CREF_NEXT(cref)) {
1225 rb_raise(rb_eRuntimeError, "class variable access from toplevel");
1226 }
1227
1228 klass = vm_get_iclass(cfp, CREF_CLASS(cref));
1229
1230 if (NIL_P(klass)) {
1231 rb_raise(rb_eTypeError, "no class variables available");
1232 }
1233 return klass;
1234}
1235
1236#define ractor_incidental_shareable_p(cond, val) \
1237 (!(cond) || rb_ractor_shareable_p(val))
1238#define ractor_object_incidental_shareable_p(obj, val) \
1239 ractor_incidental_shareable_p(rb_ractor_shareable_p(obj), val)
1240
1241ALWAYS_INLINE(static VALUE vm_getivar(VALUE, ID, const rb_iseq_t *, IVC, const struct rb_callcache *, int, VALUE));
1242static inline VALUE
1243vm_getivar(VALUE obj, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, VALUE default_value)
1244{
1245 VALUE fields_obj;
1246#if OPT_IC_FOR_IVAR
1247 if (SPECIAL_CONST_P(obj)) {
1248 return default_value;
1249 }
1250
1251 switch (BUILTIN_TYPE(obj)) {
1252 case T_OBJECT:
1253 fields_obj = ROBJECT_FIELDS_OBJ(obj);
1254 break;
1255 case T_CLASS:
1256 case T_MODULE:
1257 {
1258 if (UNLIKELY(!rb_ractor_main_p())) {
1259 // For two reasons we can only use the fast path on the main
1260 // ractor.
1261 // First, only the main ractor is allowed to set ivars on classes
1262 // and modules. So we can skip locking.
1263 // Second, other ractors need to check the shareability of the
1264 // values returned from the class ivars.
1265
1266 if (default_value == Qundef) { // defined?
1267 return rb_ivar_defined(obj, id) ? Qtrue : Qundef;
1268 }
1269 else {
1270 goto general_path;
1271 }
1272 }
1273
1274 fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
1275 break;
1276 }
1277 default:
1278 fields_obj = rb_obj_fields(obj, id);
1279 }
1280
1281 if (!fields_obj) {
1282 return default_value;
1283 }
1284
1285 VALUE val = Qundef;
1286
1287 shape_id_t shape_id = RBASIC_SHAPE_ID_FOR_READ(fields_obj);
1288 VALUE *ivar_list = rb_imemo_fields_ptr(fields_obj);
1289
1290 rb_getivar_cache cache = rb_getivar_cache_unpack(vm_cache_attr_index_atomic_read(is_attr, ic, cc));
1291
1292 if (LIKELY(cache.shape_offset == shape_id)) {
1293 if (cache.index == ATTR_INDEX_NOT_SET) {
1294 return default_value;
1295 }
1296
1297 val = ivar_list[cache.index];
1298#if USE_DEBUG_COUNTER
1299 RB_DEBUG_COUNTER_INC(ivar_get_ic_hit);
1300
1301 if (RB_TYPE_P(obj, T_OBJECT)) {
1302 RB_DEBUG_COUNTER_INC(ivar_get_obj_hit);
1303 }
1304#endif
1305 RUBY_ASSERT(!UNDEF_P(val));
1306 }
1307 else { // cache miss case
1308#if USE_DEBUG_COUNTER
1309 if (is_attr) {
1310 if (cache.shape_offset != INVALID_SHAPE_ID) {
1311 RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_set);
1312 }
1313 else {
1314 RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_unset);
1315 }
1316 }
1317 else {
1318 if (cache.shape_offset != INVALID_SHAPE_ID) {
1319 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_set);
1320 }
1321 else {
1322 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_unset);
1323 }
1324 }
1325 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
1326
1327 if (RB_TYPE_P(obj, T_OBJECT)) {
1328 RB_DEBUG_COUNTER_INC(ivar_get_obj_miss);
1329 }
1330#endif
1331
1332 if (UNLIKELY(rb_shape_complex_p(shape_id))) {
1333 st_table *table = (st_table *)ivar_list;
1334
1335 RUBY_ASSERT(table);
1336 RUBY_ASSERT(table == rb_imemo_fields_complex_tbl(fields_obj));
1337
1338 if (!st_lookup(table, id, &val)) {
1339 val = default_value;
1340 }
1341 }
1342 else {
1343 shape_id_t previous_cached_offset = cache.shape_offset;
1344 if (rb_shape_get_iv_index_with_hint(shape_id, id, &cache.index, &cache.shape_offset)) {
1345 if (cache.shape_offset != previous_cached_offset) {
1346 RUBY_ASSERT(!rb_shape_complex_p(cache.shape_offset));
1347 RUBY_ASSERT(cache.shape_offset != INVALID_SHAPE_ID);
1348
1349 uint64_t packed_cache = rb_getivar_cache_pack(cache.shape_offset, cache.index);
1350 vm_cache_attr_index_set(is_attr, ic, cc, packed_cache);
1351 }
1352
1353 if (cache.index == ATTR_INDEX_NOT_SET) {
1354 val = default_value;
1355 }
1356 else {
1357 // We fetched the ivar list above
1358 val = ivar_list[cache.index];
1359 RUBY_ASSERT(!UNDEF_P(val));
1360 }
1361 }
1362 else {
1363 vm_cache_attr_index_set(is_attr, ic, cc, rb_getivar_cache_pack(shape_id, ATTR_INDEX_NOT_SET));
1364 val = default_value;
1365 }
1366 }
1367 }
1368
1369 if (!UNDEF_P(default_value)) {
1370 RUBY_ASSERT(!UNDEF_P(val));
1371 }
1372
1373 return val;
1374
1375general_path:
1376#endif /* OPT_IC_FOR_IVAR */
1377 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
1378
1379 if (is_attr) {
1380 return rb_attr_get(obj, id);
1381 }
1382 else {
1383 return rb_ivar_get(obj, id);
1384 }
1385}
1386
1387ALWAYS_INLINE(static VALUE vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr));
1388NOINLINE(static VALUE vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic));
1389NOINLINE(static VALUE vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc));
1390
1391static VALUE
1392vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr)
1393{
1394#if OPT_IC_FOR_IVAR
1395 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss);
1396
1397 rb_check_frozen(obj);
1398
1399 shape_id_t previous_shape_id = RBASIC_SHAPE_ID(obj);
1400 attr_index_t index = rb_ivar_set_index(obj, id, val);
1401 shape_id_t next_shape_id = RBASIC_SHAPE_ID(obj);
1402
1403 if (!rb_shape_complex_p(next_shape_id)) {
1404 uint64_t packed_cache = rb_setivar_cache_pack(RSHAPE_OFFSET(previous_shape_id), RSHAPE_OFFSET(next_shape_id), index);
1405 vm_cache_attr_index_set(is_attr, ic, cc, packed_cache);
1406 }
1407
1408 RB_DEBUG_COUNTER_INC(ivar_set_obj_miss);
1409 return val;
1410#else
1411 return rb_ivar_set(obj, id, val);
1412#endif
1413}
1414
1415static VALUE
1416vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic)
1417{
1418 return vm_setivar_slowpath(obj, id, val, iseq, ic, NULL, false);
1419}
1420
1421static VALUE
1422vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc)
1423{
1424 return vm_setivar_slowpath(obj, id, val, NULL, NULL, cc, true);
1425}
1426
1427NOINLINE(static VALUE vm_setivar_class(VALUE obj, VALUE val, rb_setivar_cache cache));
1428static VALUE
1429vm_setivar_class(VALUE obj, VALUE val, rb_setivar_cache cache)
1430{
1431 if (UNLIKELY(!rb_ractor_main_p())) {
1432 return Qundef;
1433 }
1434
1435 VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
1436 if (UNLIKELY(!fields_obj)) {
1437 return Qundef;
1438 }
1439
1440 shape_id_t shape_id = RBASIC_SHAPE_ID(fields_obj);
1441 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, RBASIC_SHAPE_ID(fields_obj), cache);
1442 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1443 return Qundef;
1444 }
1445
1446 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[cache.index], val);
1447
1448 if (shape_id != dest_shape_id) {
1449 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1450 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1451 }
1452
1453 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1454
1455 return val;
1456}
1457
1458NOINLINE(static VALUE vm_setivar_default(VALUE obj, ID id, VALUE val, rb_setivar_cache cache));
1459static VALUE
1460vm_setivar_default(VALUE obj, ID id, VALUE val, rb_setivar_cache cache)
1461{
1462 VALUE fields_obj = rb_obj_fields(obj, id);
1463 if (UNLIKELY(!fields_obj)) {
1464 return Qundef;
1465 }
1466
1467 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1468 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, RBASIC_SHAPE_ID(fields_obj), cache);
1469 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1470 return Qundef;
1471 }
1472
1473 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[cache.index], val);
1474
1475 if (shape_id != dest_shape_id) {
1476 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1477 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1478 }
1479
1480 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1481
1482 return val;
1483}
1484
1485static inline VALUE
1486vm_setivar(VALUE obj, VALUE val, rb_setivar_cache cache)
1487{
1488#if OPT_IC_FOR_IVAR
1489 switch (BUILTIN_TYPE(obj)) {
1490 case T_OBJECT:
1491 {
1492 VM_ASSERT(!rb_ractor_shareable_p(obj) || rb_obj_frozen_p(obj));
1493
1494 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1495 shape_id_t dest_shape_id = rb_setivar_cache_revalidate(shape_id, shape_id, cache);
1496 if (UNLIKELY(dest_shape_id == INVALID_SHAPE_ID)) {
1497 break;
1498 }
1499
1500 VALUE fields_obj = ROBJECT_FIELDS_OBJ(obj);
1501 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[cache.index], val);
1502 if (shape_id != dest_shape_id) {
1503 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1504 if (fields_obj != obj) {
1505 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1506 }
1507 }
1508
1509 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1510 RB_DEBUG_COUNTER_INC(ivar_set_obj_hit);
1511 return val;
1512 }
1513 break;
1514 case T_CLASS:
1515 case T_MODULE:
1516 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss_noobject);
1517 default:
1518 break;
1519 }
1520
1521 return Qundef;
1522#endif /* OPT_IC_FOR_IVAR */
1523}
1524
1525static VALUE
1526update_classvariable_cache(const rb_iseq_t *iseq, VALUE klass, ID id, const rb_cref_t * cref, ICVARC ic)
1527{
1528 VALUE defined_class = 0;
1529 VALUE cvar_value = rb_cvar_find(klass, id, &defined_class);
1530
1531 if (RB_TYPE_P(defined_class, T_ICLASS)) {
1532 defined_class = RBASIC(defined_class)->klass;
1533 }
1534
1535 VALUE rb_cvc_tbl = RCLASS_CVC_TBL(defined_class);
1536 if (!rb_cvc_tbl) {
1537 rb_bug("the cvc table should be set");
1538 }
1539
1540 VALUE ent_data;
1541 if (!rb_marked_id_table_lookup(rb_cvc_tbl, id, &ent_data)) {
1542 rb_bug("should have cvar cache entry");
1543 }
1544
1545 struct rb_cvar_class_tbl_entry *ent = (void *)ent_data;
1546
1547 ent->global_cvar_state = GET_GLOBAL_CVAR_STATE();
1548 RB_OBJ_WRITE((VALUE)ent, &ent->cref, cref);
1549 RB_OBJ_WRITE(iseq, &ic->entry, ent);
1550
1551 RUBY_ASSERT(BUILTIN_TYPE((VALUE)cref) == T_IMEMO && IMEMO_TYPE_P(cref, imemo_cref));
1552
1553 return cvar_value;
1554}
1555
1556static inline VALUE
1557vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID id, ICVARC ic)
1558{
1559 const rb_cref_t *cref;
1560 cref = vm_get_cref(GET_EP());
1561
1562 if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && ic->entry->cref == cref && LIKELY(rb_ractor_main_p())) {
1563 RB_DEBUG_COUNTER_INC(cvar_read_inline_hit);
1564
1565 VALUE v = rb_ivar_lookup(ic->entry->class_value, id, Qundef);
1566 RUBY_ASSERT(!UNDEF_P(v));
1567
1568 return v;
1569 }
1570
1571 VALUE klass = vm_get_cvar_base(cref, reg_cfp, 1);
1572
1573 return update_classvariable_cache(iseq, klass, id, cref, ic);
1574}
1575
1576VALUE
1577rb_vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, ICVARC ic)
1578{
1579 return vm_getclassvariable(iseq, cfp, id, ic);
1580}
1581
1582static inline void
1583vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID id, VALUE val, ICVARC ic)
1584{
1585 const rb_cref_t *cref;
1586 cref = vm_get_cref(GET_EP());
1587
1588 if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && ic->entry->cref == cref && LIKELY(rb_ractor_main_p())) {
1589 RB_DEBUG_COUNTER_INC(cvar_write_inline_hit);
1590
1591 rb_class_ivar_set(ic->entry->class_value, id, val);
1592 return;
1593 }
1594
1595 VALUE klass = vm_get_cvar_base(cref, reg_cfp, 1);
1596
1597 rb_cvar_set(klass, id, val);
1598
1599 update_classvariable_cache(iseq, klass, id, cref, ic);
1600}
1601
1602void
1603rb_vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, VALUE val, ICVARC ic)
1604{
1605 vm_setclassvariable(iseq, cfp, id, val, ic);
1606}
1607
1608ALWAYS_INLINE(static VALUE vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic));
1609static inline VALUE
1610vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
1611{
1612 return vm_getivar(obj, id, iseq, ic, NULL, FALSE, Qnil);
1613}
1614
1615static inline void
1616vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
1617{
1618 if (RB_SPECIAL_CONST_P(obj)) {
1619 rb_error_frozen_object(obj);
1620 return;
1621 }
1622
1623 rb_setivar_cache cache = rb_setivar_cache_unpack(vm_ic_atomic_cache_read(ic));
1624 if (UNLIKELY(UNDEF_P(vm_setivar(obj, val, cache)))) {
1625 switch (BUILTIN_TYPE(obj)) {
1626 case T_OBJECT:
1627 break;
1628 case T_CLASS:
1629 case T_MODULE:
1630 if (!UNDEF_P(vm_setivar_class(obj, val, cache))) {
1631 return;
1632 }
1633 break;
1634 default:
1635 if (!UNDEF_P(vm_setivar_default(obj, id, val, cache))) {
1636 return;
1637 }
1638 }
1639 vm_setivar_slowpath_ivar(obj, id, val, iseq, ic);
1640 }
1641}
1642
1643void
1644rb_vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
1645{
1646 vm_setinstancevariable(iseq, obj, id, val, ic);
1647}
1648
1649VALUE
1650rb_vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
1651{
1652 return vm_getinstancevariable(iseq, obj, id, ic);
1653}
1654
1655static VALUE
1656vm_throw_continue(const rb_execution_context_t *ec, VALUE err)
1657{
1658 /* continue throw */
1659
1660 if (FIXNUM_P(err)) {
1661 ec->tag->state = RUBY_TAG_FATAL;
1662 }
1663 else if (SYMBOL_P(err)) {
1664 ec->tag->state = TAG_THROW;
1665 }
1666 else if (THROW_DATA_P(err)) {
1667 ec->tag->state = THROW_DATA_STATE((struct vm_throw_data *)err);
1668 }
1669 else {
1670 ec->tag->state = TAG_RAISE;
1671 }
1672 return err;
1673}
1674
1675static VALUE
1676vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_cfp, enum ruby_tag_type state,
1677 const int flag, const VALUE throwobj)
1678{
1679 const rb_control_frame_t *escape_cfp = NULL;
1680 const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */
1681
1682 if (flag != 0) {
1683 /* do nothing */
1684 }
1685 else if (state == TAG_BREAK) {
1686 int is_orphan = 1;
1687 const VALUE *ep = GET_EP();
1688 const rb_iseq_t *base_iseq = GET_ISEQ();
1689 escape_cfp = reg_cfp;
1690
1691 while (ISEQ_BODY(base_iseq)->type != ISEQ_TYPE_BLOCK) {
1692 if (ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_CLASS) {
1693 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1694 ep = escape_cfp->ep;
1695 base_iseq = CFP_ISEQ(escape_cfp);
1696 }
1697 else {
1698 ep = VM_ENV_PREV_EP(ep);
1699 base_iseq = ISEQ_BODY(base_iseq)->parent_iseq;
1700 escape_cfp = rb_vm_search_cf_from_ep(ec, escape_cfp, ep);
1701 VM_ASSERT(CFP_ISEQ(escape_cfp) == base_iseq);
1702 }
1703 }
1704
1705 if (VM_FRAME_LAMBDA_P(escape_cfp)) {
1706 /* lambda{... break ...} */
1707 is_orphan = 0;
1708 state = TAG_RETURN;
1709 }
1710 else {
1711 ep = VM_ENV_PREV_EP(ep);
1712
1713 while (escape_cfp < eocfp) {
1714 if (escape_cfp->ep == ep) {
1715 const rb_iseq_t *const iseq = CFP_ISEQ(escape_cfp);
1716 const VALUE epc = CFP_PC(escape_cfp) - ISEQ_BODY(iseq)->iseq_encoded;
1717 const struct iseq_catch_table *const ct = ISEQ_BODY(iseq)->catch_table;
1718 unsigned int i;
1719
1720 if (!ct) break;
1721 for (i=0; i < ct->size; i++) {
1722 const struct iseq_catch_table_entry *const entry =
1723 UNALIGNED_MEMBER_PTR(ct, entries[i]);
1724
1725 if (entry->type == CATCH_TYPE_BREAK &&
1726 entry->iseq == base_iseq &&
1727 entry->start < epc && entry->end >= epc) {
1728 if (entry->cont == epc) { /* found! */
1729 is_orphan = 0;
1730 }
1731 break;
1732 }
1733 }
1734 break;
1735 }
1736
1737 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1738 }
1739 }
1740
1741 if (is_orphan) {
1742 rb_vm_localjump_error("break from proc-closure", throwobj, TAG_BREAK);
1743 }
1744 }
1745 else if (state == TAG_RETRY) {
1746 const VALUE *ep = VM_ENV_PREV_EP(GET_EP());
1747
1748 escape_cfp = rb_vm_search_cf_from_ep(ec, reg_cfp, ep);
1749 }
1750 else if (state == TAG_RETURN) {
1751 const VALUE *current_ep = GET_EP();
1752 const VALUE *target_ep = NULL, *target_lep, *ep = current_ep;
1753 int in_class_frame = 0;
1754 int toplevel = 1;
1755 escape_cfp = reg_cfp;
1756
1757 // find target_lep, target_ep
1758 while (!VM_ENV_LOCAL_P(ep)) {
1759 if (VM_ENV_FLAGS(ep, VM_FRAME_FLAG_LAMBDA) && target_ep == NULL) {
1760 target_ep = ep;
1761 }
1762 ep = VM_ENV_PREV_EP(ep);
1763 }
1764 target_lep = ep;
1765
1766 while (escape_cfp < eocfp) {
1767 const VALUE *lep = VM_CF_LEP(escape_cfp);
1768
1769 if (!target_lep) {
1770 target_lep = lep;
1771 }
1772
1773 if (lep == target_lep &&
1774 VM_FRAME_RUBYFRAME_P(escape_cfp) &&
1775 ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_CLASS) {
1776 in_class_frame = 1;
1777 target_lep = 0;
1778 }
1779
1780 if (lep == target_lep) {
1781 if (VM_FRAME_LAMBDA_P(escape_cfp)) {
1782 toplevel = 0;
1783 if (in_class_frame) {
1784 /* lambda {class A; ... return ...; end} */
1785 goto valid_return;
1786 }
1787 else {
1788 const VALUE *tep = current_ep;
1789
1790 while (target_lep != tep) {
1791 if (escape_cfp->ep == tep) {
1792 /* in lambda */
1793 if (tep == target_ep) {
1794 goto valid_return;
1795 }
1796 else {
1797 goto unexpected_return;
1798 }
1799 }
1800 tep = VM_ENV_PREV_EP(tep);
1801 }
1802 }
1803 }
1804 else if (VM_FRAME_RUBYFRAME_P(escape_cfp)) {
1805 switch (ISEQ_BODY(CFP_ISEQ(escape_cfp))->type) {
1806 case ISEQ_TYPE_TOP:
1807 case ISEQ_TYPE_MAIN:
1808 if (toplevel) {
1809 if (in_class_frame) goto unexpected_return;
1810 if (target_ep == NULL) {
1811 goto valid_return;
1812 }
1813 else {
1814 goto unexpected_return;
1815 }
1816 }
1817 break;
1818 case ISEQ_TYPE_EVAL: {
1819 const rb_iseq_t *is = CFP_ISEQ(escape_cfp);
1820 enum rb_iseq_type t = ISEQ_BODY(is)->type;
1821 while (t == ISEQ_TYPE_RESCUE || t == ISEQ_TYPE_ENSURE || t == ISEQ_TYPE_EVAL) {
1822 if (!(is = ISEQ_BODY(is)->parent_iseq)) break;
1823 t = ISEQ_BODY(is)->type;
1824 }
1825 toplevel = t == ISEQ_TYPE_TOP || t == ISEQ_TYPE_MAIN;
1826 break;
1827 }
1828 case ISEQ_TYPE_CLASS:
1829 toplevel = 0;
1830 break;
1831 default:
1832 break;
1833 }
1834 }
1835 }
1836
1837 if (escape_cfp->ep == target_lep && ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_METHOD) {
1838 if (target_ep == NULL) {
1839 goto valid_return;
1840 }
1841 else {
1842 goto unexpected_return;
1843 }
1844 }
1845
1846 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1847 }
1848 unexpected_return:;
1849 rb_vm_localjump_error("unexpected return", throwobj, TAG_RETURN);
1850
1851 valid_return:;
1852 /* do nothing */
1853 }
1854 else {
1855 rb_bug("isns(throw): unsupported throw type");
1856 }
1857
1858 ec->tag->state = state;
1859 return (VALUE)THROW_DATA_NEW(throwobj, escape_cfp, state);
1860}
1861
1862static VALUE
1863vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
1864 rb_num_t throw_state, VALUE throwobj)
1865{
1866 const int state = (int)(throw_state & VM_THROW_STATE_MASK);
1867 const int flag = (int)(throw_state & VM_THROW_NO_ESCAPE_FLAG);
1868
1869 if (state != 0) {
1870 return vm_throw_start(ec, reg_cfp, state, flag, throwobj);
1871 }
1872 else {
1873 return vm_throw_continue(ec, throwobj);
1874 }
1875}
1876
1877// Fallback for YJIT. Prepare throw data and return it.
1878VALUE
1879rb_vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj)
1880{
1881 return vm_throw(ec, reg_cfp, throw_state, throwobj);
1882}
1883
1884NORETURN(VALUE rb_zjit_throw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj));
1885
1886// Fallback for ZJIT. Make a longjmp and unwind to the most recent vm_exec().
1887VALUE
1888rb_zjit_throw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj)
1889{
1890 VALUE val = vm_throw(ec, reg_cfp, throw_state, throwobj);
1891
1892 // vm_throw() has set ec->tag->state. On the longjmp path, vm_exec() reads the throw data from
1893 // ec->errinfo instead of vm_exec_core()'s return value like THROW_EXCEPTION()'s path does, so
1894 // we need to put it there instead.
1895 enum ruby_tag_type state = ec->tag->state;
1896 ec->errinfo = val;
1897 EC_JUMP_TAG(ec, state);
1898
1899 UNREACHABLE_RETURN(Qundef);
1900}
1901
1902static inline void
1903vm_expandarray(struct rb_control_frame_struct *cfp, VALUE ary, rb_num_t num, int flag)
1904{
1905 int is_splat = flag & 0x01;
1906 const VALUE *ptr;
1907 rb_num_t len;
1908 const VALUE obj = ary;
1909
1910 if (!RB_TYPE_P(ary, T_ARRAY) && NIL_P(ary = rb_check_array_type(ary))) {
1911 ary = obj;
1912 ptr = &ary;
1913 len = 1;
1914 }
1915 else {
1916 ptr = RARRAY_CONST_PTR(ary);
1917 len = (rb_num_t)RARRAY_LEN(ary);
1918 }
1919
1920 if (num + is_splat == 0) {
1921 /* no space left on stack */
1922 }
1923 else if (flag & 0x02) {
1924 /* post: ..., nil ,ary[-1], ..., ary[0..-num] # top */
1925 rb_num_t i = 0, j;
1926
1927 if (len < num) {
1928 for (i = 0; i < num - len; i++) {
1929 *cfp->sp++ = Qnil;
1930 }
1931 }
1932
1933 for (j = 0; i < num; i++, j++) {
1934 VALUE v = ptr[len - j - 1];
1935 *cfp->sp++ = v;
1936 }
1937
1938 if (is_splat) {
1939 *cfp->sp++ = rb_ary_new4(len - j, ptr);
1940 }
1941 }
1942 else {
1943 /* normal: ary[num..-1], ary[num-2], ary[num-3], ..., ary[0] # top */
1944 if (is_splat) {
1945 if (num > len) {
1946 *cfp->sp++ = rb_ary_new();
1947 }
1948 else {
1949 *cfp->sp++ = rb_ary_new4(len - num, ptr + num);
1950 }
1951 }
1952
1953 if (num > len) {
1954 rb_num_t i = 0;
1955 for (; i < num - len; i++) {
1956 *cfp->sp++ = Qnil;
1957 }
1958
1959 for (rb_num_t j = 0; i < num; i++, j++) {
1960 *cfp->sp++ = ptr[len - j - 1];
1961 }
1962 }
1963 else {
1964 for (rb_num_t j = 0; j < num; j++) {
1965 *cfp->sp++ = ptr[num - j - 1];
1966 }
1967 }
1968 }
1969
1970 RB_GC_GUARD(ary);
1971}
1972
1973static VALUE vm_call_general(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
1974
1975static VALUE vm_mtbl_dump(VALUE klass, ID target_mid);
1976
1977static struct rb_class_cc_entries *
1978vm_ccs_create(VALUE klass, VALUE cc_tbl, ID mid, const rb_callable_method_entry_t *cme)
1979{
1980 int initial_capa = 2;
1981 struct rb_class_cc_entries *ccs = ruby_xmalloc(vm_ccs_alloc_size(initial_capa));
1982#if VM_CHECK_MODE > 0
1983 ccs->debug_sig = ~(VALUE)ccs;
1984#endif
1985 ccs->capa = initial_capa;
1986 ccs->len = 0;
1987 ccs->cme = cme;
1988 METHOD_ENTRY_CACHED_SET((rb_callable_method_entry_t *)cme);
1989
1990 rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
1991 RB_OBJ_WRITTEN(cc_tbl, Qundef, cme);
1992 return ccs;
1993}
1994
1995static void
1996vm_ccs_push(VALUE cc_tbl, ID mid, struct rb_class_cc_entries *ccs, const struct rb_callinfo *ci, const struct rb_callcache *cc)
1997{
1998 if (! vm_cc_markable(cc)) {
1999 return;
2000 }
2001
2002 if (UNLIKELY(ccs->len == ccs->capa)) {
2003 RUBY_ASSERT(ccs->capa > 0);
2004 ccs->capa *= 2;
2005 ccs = ruby_xrealloc(ccs, vm_ccs_alloc_size(ccs->capa));
2006#if VM_CHECK_MODE > 0
2007 ccs->debug_sig = ~(VALUE)ccs;
2008#endif
2009 // GC?
2010 rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
2011 }
2012 VM_ASSERT(ccs->len < ccs->capa);
2013
2014 const struct rb_callinfo_kwarg *kwarg = vm_ci_kwarg(ci);
2015 const int pos = ccs->len++;
2016 ccs->entries[pos].argc = vm_ci_argc(ci);
2017 ccs->entries[pos].flag = (unsigned short)vm_ci_flag(ci);
2018 ccs->entries[pos].kw_len = (unsigned short)(kwarg ? kwarg->keyword_len : 0);
2019 RB_OBJ_WRITE(cc_tbl, &ccs->entries[pos].cc, cc);
2020
2021 if (RB_DEBUG_COUNTER_SETMAX(ccs_maxlen, ccs->len)) {
2022 // for tuning
2023 // vm_mtbl_dump(klass, 0);
2024 }
2025}
2026
2027#if VM_CHECK_MODE > 0
2028void
2029rb_vm_ccs_dump(struct rb_class_cc_entries *ccs)
2030{
2031 ruby_debug_printf("ccs:%p (%d,%d)\n", (void *)ccs, ccs->len, ccs->capa);
2032 for (int i=0; i<ccs->len; i++) {
2033 ruby_debug_printf("CCS CI ID:flag:%x argc:%u kw_len:%u\n",
2034 ccs->entries[i].flag,
2035 ccs->entries[i].argc,
2036 ccs->entries[i].kw_len);
2037 rp(ccs->entries[i].cc);
2038 }
2039}
2040
2041static int
2042vm_ccs_verify(struct rb_class_cc_entries *ccs, ID mid, VALUE klass)
2043{
2044 VM_ASSERT(vm_ccs_p(ccs));
2045 VM_ASSERT(ccs->len <= ccs->capa);
2046
2047 for (int i=0; i<ccs->len; i++) {
2048 const struct rb_callcache *cc = ccs->entries[i].cc;
2049
2050 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
2051 VM_ASSERT(vm_cc_class_check(cc, klass));
2052 VM_ASSERT(vm_cc_check_cme(cc, ccs->cme));
2053 VM_ASSERT(!vm_cc_super_p(cc));
2054 VM_ASSERT(!vm_cc_refinement_p(cc));
2055 }
2056 return TRUE;
2057}
2058#endif
2059
2060const rb_callable_method_entry_t *rb_check_overloaded_cme(const rb_callable_method_entry_t *cme, const struct rb_callinfo * const ci);
2061
2062static void
2063vm_evict_cc(VALUE klass, VALUE cc_tbl, ID mid)
2064{
2065 ASSERT_vm_locking();
2066
2067 if (rb_multi_ractor_p()) {
2068 if (RCLASS_WRITABLE_CC_TBL(klass) != cc_tbl) {
2069 // Another ractor updated the CC table while we were waiting on the VM lock.
2070 // We have to retry.
2071 return;
2072 }
2073
2074 VALUE ccs_obj = 0;
2075 rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj);
2076 struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_obj;
2077
2078 if (!ccs || !METHOD_ENTRY_INVALIDATED(ccs->cme)) {
2079 // Another ractor replaced that entry while we were waiting on the VM lock.
2080 return;
2081 }
2082
2083 VALUE new_table = rb_vm_cc_table_dup(cc_tbl);
2084 rb_vm_cc_table_delete(new_table, mid);
2085 RB_OBJ_ATOMIC_WRITE(klass, &RCLASS_WRITABLE_CC_TBL(klass), new_table);
2086 }
2087 else {
2088 rb_vm_cc_table_delete(cc_tbl, mid);
2089 }
2090}
2091
2092static const struct rb_callcache *
2093vm_populate_cc(VALUE klass, const struct rb_callinfo * const ci, ID mid)
2094{
2095 ASSERT_vm_locking();
2096
2097 RB_DEBUG_COUNTER_INC(cc_not_found_in_ccs);
2098
2099 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
2100
2101 VM_ASSERT(cme == NULL || IMEMO_TYPE_P(cme, imemo_ment));
2102
2103 if (cme == NULL) {
2104 // undef or not found: can't cache the information
2105 VM_ASSERT(vm_cc_cme(&vm_empty_cc) == NULL);
2106 return &vm_empty_cc;
2107 }
2108
2109 VALUE cc_tbl = RCLASS_WRITABLE_CC_TBL(klass);
2110 const VALUE original_cc_table = cc_tbl;
2111 if (!cc_tbl) {
2112 // Is this possible after rb_callable_method_entry ?
2113 cc_tbl = rb_vm_cc_table_create(1);
2114 }
2115 else if (rb_multi_ractor_p()) {
2116 cc_tbl = rb_vm_cc_table_dup(cc_tbl);
2117 }
2118
2119 VM_ASSERT(cme == rb_callable_method_entry(klass, mid));
2120
2121 METHOD_ENTRY_CACHED_SET((struct rb_callable_method_entry_struct *)cme);
2122
2123 VM_ASSERT(cc_tbl);
2124
2125 struct rb_class_cc_entries *ccs = NULL;
2126 {
2127 VALUE ccs_obj;
2128 if (UNLIKELY(rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj))) {
2129 ccs = (struct rb_class_cc_entries *)ccs_obj;
2130 }
2131 else {
2132 // TODO: required?
2133 ccs = vm_ccs_create(klass, cc_tbl, mid, cme);
2134 }
2135 }
2136
2137 cme = rb_check_overloaded_cme(cme, ci);
2138
2139 const struct rb_callcache *cc = vm_cc_new(klass, cme, vm_call_general, cc_type_normal);
2140 vm_ccs_push(cc_tbl, mid, ccs, ci, cc);
2141
2142 VM_ASSERT(vm_cc_cme(cc) != NULL);
2143 VM_ASSERT(cme->called_id == mid);
2144 VM_ASSERT(vm_cc_cme(cc)->called_id == mid);
2145
2146 if (original_cc_table != cc_tbl) {
2147 RB_OBJ_ATOMIC_WRITE(klass, &RCLASS_WRITABLE_CC_TBL(klass), cc_tbl);
2148 }
2149
2150 return cc;
2151}
2152
2153static const struct rb_callcache *
2154vm_lookup_cc(const VALUE klass, const struct rb_callinfo * const ci, ID mid)
2155{
2156 VALUE cc_tbl;
2157 struct rb_class_cc_entries *ccs;
2158retry:
2159 cc_tbl = RUBY_ATOMIC_VALUE_LOAD(RCLASS_WRITABLE_CC_TBL(klass));
2160 ccs = NULL;
2161
2162 if (cc_tbl) {
2163 // CCS data is keyed on method id, so we don't need the method id
2164 // for doing comparisons in the `for` loop below.
2165
2166 VALUE ccs_obj;
2167 if (rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj)) {
2168 ccs = (struct rb_class_cc_entries *)ccs_obj;
2169 const int ccs_len = ccs->len;
2170
2171 if (UNLIKELY(METHOD_ENTRY_INVALIDATED(ccs->cme))) {
2172 RB_VM_LOCKING() {
2173 vm_evict_cc(klass, cc_tbl, mid);
2174 }
2175 goto retry;
2176 }
2177 else {
2178 VM_ASSERT(vm_ccs_verify(ccs, mid, klass));
2179
2180 // We already know the method id is correct because we had
2181 // to look up the ccs_data by method id. We compare argc and
2182 // flag, plus the keyword argument count: two call sites with
2183 // the same argc and flag can still differ in how many of the
2184 // arguments are keywords (e.g. `m(a: 1, b: 2)` vs `m(1, b: 2)`),
2185 // which selects a different argument setup fastpath.
2186 const struct rb_callinfo_kwarg *kwarg = vm_ci_kwarg(ci);
2187 unsigned int argc = vm_ci_argc(ci);
2188 unsigned int flag = vm_ci_flag(ci);
2189 unsigned int kw_len = kwarg ? kwarg->keyword_len : 0;
2190
2191 for (int i=0; i<ccs_len; i++) {
2192 unsigned int ccs_ci_argc = ccs->entries[i].argc;
2193 unsigned int ccs_ci_flag = ccs->entries[i].flag;
2194 unsigned int ccs_ci_kw_len = ccs->entries[i].kw_len;
2195 const struct rb_callcache *ccs_cc = ccs->entries[i].cc;
2196
2197 VM_ASSERT(IMEMO_TYPE_P(ccs_cc, imemo_callcache));
2198
2199 if (ccs_ci_argc == argc && ccs_ci_flag == flag && ccs_ci_kw_len == kw_len) {
2200 RB_DEBUG_COUNTER_INC(cc_found_in_ccs);
2201
2202 VM_ASSERT(vm_cc_cme(ccs_cc)->called_id == mid);
2203 VM_ASSERT(ccs_cc->klass == klass);
2204 VM_ASSERT(!METHOD_ENTRY_INVALIDATED(vm_cc_cme(ccs_cc)));
2205
2206 return ccs_cc;
2207 }
2208 }
2209 }
2210 }
2211 }
2212
2213 RB_GC_GUARD(cc_tbl);
2214 return NULL;
2215}
2216
2217static const struct rb_callcache *
2218vm_search_cc(const VALUE klass, const struct rb_callinfo * const ci)
2219{
2220 const ID mid = vm_ci_mid(ci);
2221
2222 const struct rb_callcache *cc = vm_lookup_cc(klass, ci, mid);
2223 if (cc) {
2224 return cc;
2225 }
2226
2227 RB_VM_LOCKING() {
2228 if (rb_multi_ractor_p()) {
2229 // The CC may have been populated by another ractor while we were waiting on the lock,
2230 // so we must lookup a second time.
2231 cc = vm_lookup_cc(klass, ci, mid);
2232 }
2233
2234 if (!cc) {
2235 cc = vm_populate_cc(klass, ci, mid);
2236 }
2237 }
2238
2239 return cc;
2240}
2241
2242const struct rb_callcache *
2243rb_vm_search_method_slowpath(const struct rb_callinfo *ci, VALUE klass)
2244{
2245 const struct rb_callcache *cc;
2246
2247 VM_ASSERT(!SPECIAL_CONST_P(klass));
2248
2249 if (RB_BUILTIN_TYPE(klass) == T_NONE) {
2250 // If we find a T_NONE here, it's most likely we called CLASS_OF(obj) on a
2251 // garbage collected object (the freelist is stored in the class pointer),
2252 // but it's possible that just the class was GC'd.
2253 // This message intentionally tries to imply the former, but make an
2254 // accurate statement for either case.
2255 rb_bug("attempted to search method '%s' on a garbage collected object",
2256 rb_id2name(vm_ci_mid(ci)));
2257 }
2258
2259 VM_ASSERT_TYPE2(klass, T_CLASS, T_ICLASS);
2260
2261 cc = vm_search_cc(klass, ci);
2262
2263 VM_ASSERT(cc);
2264 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
2265 VM_ASSERT(cc == vm_cc_empty() || cc->klass == klass);
2266 VM_ASSERT(cc == vm_cc_empty() || callable_method_entry_p(vm_cc_cme(cc)));
2267 VM_ASSERT(cc == vm_cc_empty() || !METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc)));
2268 VM_ASSERT(cc == vm_cc_empty() || vm_cc_cme(cc)->called_id == vm_ci_mid(ci));
2269
2270 return cc;
2271}
2272
2273#if VM_CHECK_MODE > 0
2274static bool
2275vm_cd_owned_by_iseq_p(const struct rb_call_data *cd, const rb_iseq_t *iseq)
2276{
2277 const struct rb_iseq_constant_body *body = ISEQ_BODY(iseq);
2278 return cd >= body->call_data && cd < body->call_data + body->ci_size;
2279}
2280#endif
2281
2282static const struct rb_callcache *
2283vm_search_method_slowpath0(VALUE cd_owner, struct rb_call_data *cd, VALUE klass)
2284{
2285#if USE_DEBUG_COUNTER
2286 const struct rb_callcache *old_cc = cd->cc;
2287 const rb_callable_method_entry_t *const old_cme = old_cc ? old_cc->cme_ : NULL;
2288#endif
2289
2290 const struct rb_callcache *cc = rb_vm_search_method_slowpath(cd->ci, klass);
2291 const rb_callable_method_entry_t *const new_cme = vm_cc_cme(cc);
2292
2293#if OPT_INLINE_METHOD_CACHE
2294 cd->cc = cc;
2295
2296 const struct rb_callcache *empty_cc = &vm_empty_cc;
2297 if (cd_owner && cc != empty_cc) {
2298 // invokesuper dispatches with a cd on the machine stack, which has no
2299 // owning iseq to remember and keeps its ci out of the GC.
2300 VM_ASSERT(!vm_ci_markable(cd->ci) ||
2301 vm_cd_owned_by_iseq_p(cd, (const rb_iseq_t *)cd_owner));
2302 RB_OBJ_WRITTEN(cd_owner, Qundef, cc);
2303 }
2304
2305#if USE_DEBUG_COUNTER
2306 if (!old_cc || old_cc == empty_cc) {
2307 // empty
2308 RB_DEBUG_COUNTER_INC(mc_inline_miss_empty);
2309 }
2310 else if (old_cc == cc) {
2311 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_cc);
2312 }
2313 else if (old_cme == new_cme) {
2314 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_cme);
2315 }
2316 else if (old_cme && new_cme && old_cme->def == new_cme->def) {
2317 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_def);
2318 }
2319 else {
2320 RB_DEBUG_COUNTER_INC(mc_inline_miss_diff);
2321 }
2322#endif
2323#endif // OPT_INLINE_METHOD_CACHE
2324
2325 if (new_cme) VM_ASSERT(new_cme->called_id == vm_ci_mid(cd->ci));
2326
2327 return cc;
2328}
2329
2330ALWAYS_INLINE(static bool vm_cc_hit_p(const struct rb_callcache *cc, const struct rb_call_data *cd, VALUE klass));
2331static bool
2332vm_cc_hit_p(const struct rb_callcache *cc, const struct rb_call_data *cd, VALUE klass)
2333{
2334 VM_ASSERT_TYPE2(klass, T_CLASS, T_ICLASS);
2335
2336#if OPT_INLINE_METHOD_CACHE
2337 if (LIKELY(vm_cc_class_check(cc, klass))) {
2338 if (LIKELY(!METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc)))) {
2339 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
2340 RB_DEBUG_COUNTER_INC(mc_inline_hit);
2341 VM_ASSERT(vm_cc_cme(cc) == NULL || // not found
2342 (vm_ci_flag(cd->ci) & VM_CALL_SUPER) || // search_super w/ define_method
2343 vm_cc_cme(cc)->called_id == vm_ci_mid(cd->ci)); // cme->called_id == ci->mid
2344
2345 return true;
2346 }
2347 RB_DEBUG_COUNTER_INC(mc_inline_miss_invalidated);
2348 }
2349 else {
2350 RB_DEBUG_COUNTER_INC(mc_inline_miss_klass);
2351 }
2352#endif
2353
2354 return false;
2355}
2356
2357ALWAYS_INLINE(static const struct rb_callcache *vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE klass));
2358static const struct rb_callcache *
2359vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE klass)
2360{
2361 const struct rb_callcache *cc = cd->cc;
2362 if (vm_cc_hit_p(cc, cd, klass)) {
2363 return cc;
2364 }
2365
2366 return vm_search_method_slowpath0((VALUE)CFP_ISEQ(reg_cfp), cd, klass);
2367}
2368
2369static const struct rb_callable_method_entry_struct *
2370vm_search_method(struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE recv)
2371{
2372 VALUE klass = CLASS_OF(recv);
2373 VM_ASSERT(klass != Qfalse);
2374 VM_ASSERT(RBASIC_CLASS(klass) == 0 || rb_obj_is_kind_of(klass, rb_cClass));
2375
2376 const struct rb_callcache *cc = vm_search_method_fastpath(reg_cfp, cd, klass);
2377 return vm_cc_cme(cc);
2378}
2379
2380const struct rb_callable_method_entry_struct *
2381rb_zjit_vm_search_method(VALUE cd_owner, struct rb_call_data *cd, VALUE recv)
2382{
2383 // Called from ZJIT with the compile-time iseq, which may differ from
2384 // the iseq on the current CFP.
2385 VALUE klass = CLASS_OF(recv);
2386 const struct rb_callcache *cc = cd->cc;
2387 if (!vm_cc_hit_p(cc, cd, klass)) {
2388 cc = vm_search_method_slowpath0(cd_owner, cd, klass);
2389 }
2390 return vm_cc_cme(cc);
2391}
2392
2393#if __has_attribute(transparent_union)
2394typedef union {
2395 VALUE (*anyargs)(ANYARGS);
2396 VALUE (*f00)(VALUE);
2397 VALUE (*f01)(VALUE, VALUE);
2398 VALUE (*f02)(VALUE, VALUE, VALUE);
2399 VALUE (*f03)(VALUE, VALUE, VALUE, VALUE);
2400 VALUE (*f04)(VALUE, VALUE, VALUE, VALUE, VALUE);
2401 VALUE (*f05)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2402 VALUE (*f06)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2403 VALUE (*f07)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2404 VALUE (*f08)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2405 VALUE (*f09)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2406 VALUE (*f10)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2407 VALUE (*f11)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2408 VALUE (*f12)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2409 VALUE (*f13)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2410 VALUE (*f14)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2411 VALUE (*f15)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2412 VALUE (*fm1)(int, union { VALUE *x; const VALUE *y; } __attribute__((__transparent_union__)), VALUE);
2413} __attribute__((__transparent_union__)) cfunc_type;
2414# define make_cfunc_type(f) (cfunc_type){.anyargs = (VALUE (*)(ANYARGS))(f)}
2415#else
2416typedef VALUE (*cfunc_type)(ANYARGS);
2417# define make_cfunc_type(f) (cfunc_type)(f)
2418#endif
2419
2420static inline int
2421check_cfunc(const rb_callable_method_entry_t *me, cfunc_type func)
2422{
2423 if (! me) {
2424 return false;
2425 }
2426 else {
2427 VM_ASSERT(IMEMO_TYPE_P(me, imemo_ment));
2428 VM_ASSERT(callable_method_entry_p(me));
2429 VM_ASSERT(me->def);
2430 if (me->def->type != VM_METHOD_TYPE_CFUNC) {
2431 return false;
2432 }
2433 else {
2434#if __has_attribute(transparent_union)
2435 return me->def->body.cfunc.func == func.anyargs;
2436#else
2437 return me->def->body.cfunc.func == func;
2438#endif
2439 }
2440 }
2441}
2442
2443static inline int
2444check_method_basic_definition(const rb_callable_method_entry_t *me)
2445{
2446 return me && METHOD_ENTRY_BASIC(me);
2447}
2448
2449static inline int
2450vm_method_cfunc_is(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv, cfunc_type func)
2451{
2452 VM_ASSERT(reg_cfp != NULL);
2453 const struct rb_callable_method_entry_struct *cme = vm_search_method(reg_cfp, cd, recv);
2454 return check_cfunc(cme, func);
2455}
2456
2457bool
2458rb_zjit_cme_is_cfunc(const rb_callable_method_entry_t *me, const cfunc_type func)
2459{
2460 return check_cfunc(me, func);
2461}
2462
2463int
2464rb_vm_method_cfunc_is(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv, cfunc_type func)
2465{
2466 const struct rb_callable_method_entry_struct *cme = rb_zjit_vm_search_method((VALUE)iseq, cd, recv);
2467 return check_cfunc(cme, func);
2468}
2469
2470#define check_cfunc(me, func) check_cfunc(me, make_cfunc_type(func))
2471#define vm_method_cfunc_is(reg_cfp, cd, recv, func) vm_method_cfunc_is(reg_cfp, cd, recv, make_cfunc_type(func))
2472
2473#define OP_UNREDEFINED_P(op, t) BASIC_OP_UNREDEFINED_P(BOP_##op, t##_REDEFINED_OP_FLAG)
2474#define EQ_UNREDEFINED_P(t) OP_UNREDEFINED_P(EQ, t)
2475
2476static inline bool
2477FIXNUM_2_P(VALUE a, VALUE b)
2478{
2479 /* FIXNUM_P(a) && FIXNUM_P(b)
2480 * == ((a & 1) && (b & 1))
2481 * == a & b & 1 */
2482 SIGNED_VALUE x = a;
2483 SIGNED_VALUE y = b;
2484 SIGNED_VALUE z = x & y & 1;
2485 return z == 1;
2486}
2487
2488static inline bool
2489FLONUM_2_P(VALUE a, VALUE b)
2490{
2491#if USE_FLONUM
2492 /* FLONUM_P(a) && FLONUM_P(b)
2493 * == ((a & 3) == 2) && ((b & 3) == 2)
2494 * == ! ((a ^ 2) | (b ^ 2) & 3)
2495 */
2496 SIGNED_VALUE x = a;
2497 SIGNED_VALUE y = b;
2498 SIGNED_VALUE z = ((x ^ 2) | (y ^ 2)) & 3;
2499 return !z;
2500#else
2501 return false;
2502#endif
2503}
2504
2505static VALUE
2506opt_equality_specialized(VALUE recv, VALUE obj)
2507{
2508 if (FIXNUM_2_P(recv, obj) && EQ_UNREDEFINED_P(INTEGER)) {
2509 goto compare_by_identity;
2510 }
2511 else if (FLONUM_2_P(recv, obj) && EQ_UNREDEFINED_P(FLOAT)) {
2512 goto compare_by_identity;
2513 }
2514 else if (STATIC_SYM_P(recv) && STATIC_SYM_P(obj) && EQ_UNREDEFINED_P(SYMBOL)) {
2515 goto compare_by_identity;
2516 }
2517 else if (SPECIAL_CONST_P(recv)) {
2518 //
2519 }
2520 else if (RBASIC_CLASS(recv) == rb_cFloat && RB_FLOAT_TYPE_P(obj) && EQ_UNREDEFINED_P(FLOAT)) {
2521 double a = RFLOAT_VALUE(recv);
2522 double b = RFLOAT_VALUE(obj);
2523
2524 return RBOOL(a == b);
2525 }
2526 else if (RBASIC_CLASS(recv) == rb_cString && EQ_UNREDEFINED_P(STRING)) {
2527 if (recv == obj) {
2528 return Qtrue;
2529 }
2530 else if (RB_TYPE_P(obj, T_STRING)) {
2531 return rb_str_eql_internal(obj, recv);
2532 }
2533 }
2534 return Qundef;
2535
2536 compare_by_identity:
2537 return RBOOL(recv == obj);
2538}
2539
2540static VALUE
2541opt_equality(struct rb_control_frame_struct *reg_cfp, VALUE recv, VALUE obj, CALL_DATA cd)
2542{
2543 VM_ASSERT(reg_cfp != NULL);
2544
2545 VALUE val = opt_equality_specialized(recv, obj);
2546 if (!UNDEF_P(val)) return val;
2547
2548 if (!vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_equal)) {
2549 return Qundef;
2550 }
2551 else {
2552 return RBOOL(recv == obj);
2553 }
2554}
2555
2556static inline const struct rb_callcache *gccct_method_search(rb_execution_context_t *ec, VALUE recv, ID mid, const struct rb_callinfo *ci); // vm_eval.c
2557NOINLINE(static VALUE opt_equality_by_mid_slowpath(VALUE recv, VALUE obj, ID mid));
2558
2559static VALUE
2560opt_equality_by_mid_slowpath(VALUE recv, VALUE obj, ID mid)
2561{
2562 const struct rb_callcache *cc = gccct_method_search(GET_EC(), recv, mid, &VM_CI_ON_STACK(mid, 0, 1, NULL));
2563
2564 if (cc && check_cfunc(vm_cc_cme(cc), rb_obj_equal)) {
2565 return RBOOL(recv == obj);
2566 }
2567 else {
2568 return Qundef;
2569 }
2570}
2571
2572static VALUE
2573opt_equality_by_mid(VALUE recv, VALUE obj, ID mid)
2574{
2575 VALUE val = opt_equality_specialized(recv, obj);
2576 if (!UNDEF_P(val)) {
2577 return val;
2578 }
2579 else {
2580 return opt_equality_by_mid_slowpath(recv, obj, mid);
2581 }
2582}
2583
2584VALUE
2585rb_equal_opt(VALUE obj1, VALUE obj2)
2586{
2587 return opt_equality_by_mid(obj1, obj2, idEq);
2588}
2589
2590VALUE
2591rb_eql_opt(VALUE obj1, VALUE obj2)
2592{
2593 return opt_equality_by_mid(obj1, obj2, idEqlP);
2594}
2595
2596extern VALUE rb_vm_call0(rb_execution_context_t *ec, VALUE, ID, int, const VALUE*, const rb_callable_method_entry_t *, int kw_splat);
2597extern VALUE rb_vm_call_with_refinements(rb_execution_context_t *, VALUE, ID, int, const VALUE *, int);
2598
2599static VALUE
2600check_match(rb_execution_context_t *ec, VALUE pattern, VALUE target, enum vm_check_match_type type)
2601{
2602 switch (type) {
2603 case VM_CHECKMATCH_TYPE_WHEN:
2604 return pattern;
2605 case VM_CHECKMATCH_TYPE_RESCUE:
2606 if (!rb_obj_is_kind_of(pattern, rb_cModule)) {
2607 rb_raise(rb_eTypeError, "class or module required for rescue clause");
2608 }
2609 /* fall through */
2610 case VM_CHECKMATCH_TYPE_CASE: {
2611 return rb_vm_call_with_refinements(ec, pattern, idEqq, 1, &target, RB_NO_KEYWORDS);
2612 }
2613 default:
2614 rb_bug("check_match: unreachable");
2615 }
2616}
2617
2618
2619static inline VALUE
2620double_cmp_lt(double a, double b)
2621{
2622 return RBOOL(a < b);
2623}
2624
2625static inline VALUE
2626double_cmp_le(double a, double b)
2627{
2628 return RBOOL(a <= b);
2629}
2630
2631static inline VALUE
2632double_cmp_gt(double a, double b)
2633{
2634 return RBOOL(a > b);
2635}
2636
2637static inline VALUE
2638double_cmp_ge(double a, double b)
2639{
2640 return RBOOL(a >= b);
2641}
2642
2643// Copied by vm_dump.c
2644static inline VALUE *
2645vm_base_ptr(const rb_control_frame_t *cfp)
2646{
2647 const rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
2648
2649 if (CFP_ISEQ(cfp) && VM_FRAME_RUBYFRAME_P(cfp)) {
2650 VALUE *bp = prev_cfp->sp + ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size + VM_ENV_DATA_SIZE;
2651
2652 if (ISEQ_BODY(CFP_ISEQ(cfp))->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) {
2653 int lts = ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size;
2654 int params = ISEQ_BODY(CFP_ISEQ(cfp))->param.size;
2655
2656 CALL_INFO ci = (CALL_INFO)cfp->ep[-(VM_ENV_DATA_SIZE + (lts - params))]; // skip EP stuff, CI should be last local
2657 bp += vm_ci_argc(ci);
2658 }
2659
2660 if (ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) {
2661 /* adjust `self' */
2662 bp += 1;
2663 }
2664#if VM_DEBUG_BP_CHECK
2665 if (bp != cfp->bp_check) {
2666 ruby_debug_printf("bp_check: %ld, bp: %ld\n",
2667 (long)(cfp->bp_check - GET_EC()->vm_stack),
2668 (long)(bp - GET_EC()->vm_stack));
2669 rb_bug("vm_base_ptr: unreachable");
2670 }
2671#endif
2672 return bp;
2673 }
2674 else {
2675 return NULL;
2676 }
2677}
2678
2679VALUE *
2680rb_vm_base_ptr(const rb_control_frame_t *cfp)
2681{
2682 return vm_base_ptr(cfp);
2683}
2684
2685/* method call processes with call_info */
2686
2687#include "vm_args.c"
2688
2689static inline VALUE vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc, int param_size, int local_size);
2690ALWAYS_INLINE(static VALUE vm_call_iseq_setup_normal(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const rb_callable_method_entry_t *me, int opt_pc, int param_size, int local_size));
2691static inline VALUE vm_call_iseq_setup_tailcall(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc);
2692static VALUE vm_call_super_method(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
2693static VALUE vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2694static VALUE vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2695static inline VALUE vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2696
2697static vm_call_handler vm_call_iseq_setup_func(const struct rb_callinfo *ci, const int param_size, const int local_size);
2698
2699static VALUE
2700vm_call_iseq_setup_tailcall_0start(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2701{
2702 RB_DEBUG_COUNTER_INC(ccf_iseq_setup_tailcall_0start);
2703
2704 return vm_call_iseq_setup_tailcall(ec, cfp, calling, 0);
2705}
2706
2707static VALUE
2708vm_call_iseq_setup_normal_0start(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2709{
2710 RB_DEBUG_COUNTER_INC(ccf_iseq_setup_0start);
2711
2712 const struct rb_callcache *cc = calling->cc;
2713 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
2714 int param = ISEQ_BODY(iseq)->param.size;
2715 int local = ISEQ_BODY(iseq)->local_table_size;
2716 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
2717}
2718
2719bool
2720rb_simple_iseq_p(const rb_iseq_t *iseq)
2721{
2722 return ISEQ_BODY(iseq)->param.flags.has_opt == FALSE &&
2723 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2724 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2725 ISEQ_BODY(iseq)->param.flags.has_kw == FALSE &&
2726 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2727 ISEQ_BODY(iseq)->param.flags.accepts_no_kwarg == FALSE &&
2728 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2729 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2730 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2731}
2732
2733bool
2734rb_iseq_only_optparam_p(const rb_iseq_t *iseq)
2735{
2736 return ISEQ_BODY(iseq)->param.flags.has_opt == TRUE &&
2737 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2738 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2739 ISEQ_BODY(iseq)->param.flags.has_kw == FALSE &&
2740 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2741 ISEQ_BODY(iseq)->param.flags.accepts_no_kwarg == FALSE &&
2742 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2743 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2744 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2745}
2746
2747bool
2748rb_iseq_only_kwparam_p(const rb_iseq_t *iseq)
2749{
2750 return ISEQ_BODY(iseq)->param.flags.has_opt == FALSE &&
2751 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2752 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2753 ISEQ_BODY(iseq)->param.flags.has_kw == TRUE &&
2754 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2755 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2756 ISEQ_BODY(iseq)->param.flags.has_block == FALSE &&
2757 ISEQ_BODY(iseq)->param.flags.accepts_no_block == FALSE;
2758}
2759
2760#define ALLOW_HEAP_ARGV (-2)
2761#define ALLOW_HEAP_ARGV_KEEP_KWSPLAT (-3)
2762
2763static inline bool
2764vm_caller_setup_arg_splat(rb_control_frame_t *cfp, struct rb_calling_info *calling, VALUE ary, int max_args)
2765{
2766 vm_check_canary(GET_EC(), cfp->sp);
2767 bool ret = false;
2768
2769 if (!NIL_P(ary)) {
2770 const VALUE *ptr = RARRAY_CONST_PTR(ary);
2771 long len = RARRAY_LEN(ary);
2772 int argc = calling->argc;
2773
2774 if (UNLIKELY(max_args <= ALLOW_HEAP_ARGV && len + argc > VM_ARGC_STACK_MAX)) {
2775 /* Avoid SystemStackError when splatting large arrays by storing arguments in
2776 * a temporary array, instead of trying to keeping arguments on the VM stack.
2777 */
2778 VALUE *argv = cfp->sp - argc;
2779 VALUE argv_ary = rb_ary_hidden_new(len + argc + 1);
2780 rb_ary_cat(argv_ary, argv, argc);
2781 rb_ary_cat(argv_ary, ptr, len);
2782 cfp->sp -= argc - 1;
2783 cfp->sp[-1] = argv_ary;
2784 calling->argc = 1;
2785 calling->heap_argv = argv_ary;
2786 RB_GC_GUARD(ary);
2787 }
2788 else {
2789 long i;
2790
2791 if (max_args >= 0 && len + argc > max_args) {
2792 /* If only a given max_args is allowed, copy up to max args.
2793 * Used by vm_callee_setup_block_arg for non-lambda blocks,
2794 * where additional arguments are ignored.
2795 *
2796 * Also, copy up to one more argument than the maximum,
2797 * in case it is an empty keyword hash that will be removed.
2798 */
2799 calling->argc += len - (max_args - argc + 1);
2800 len = max_args - argc + 1;
2801 ret = true;
2802 }
2803 else {
2804 /* Unset heap_argv if set originally. Can happen when
2805 * forwarding modified arguments, where heap_argv was used
2806 * originally, but heap_argv not supported by the forwarded
2807 * method in all cases.
2808 */
2809 calling->heap_argv = 0;
2810 }
2811 CHECK_VM_STACK_OVERFLOW(cfp, len);
2812
2813 for (i = 0; i < len; i++) {
2814 *cfp->sp++ = ptr[i];
2815 }
2816 calling->argc += i;
2817 }
2818 }
2819
2820 return ret;
2821}
2822
2823static inline void
2824vm_caller_setup_arg_kw(rb_control_frame_t *cfp, struct rb_calling_info *calling, const struct rb_callinfo *ci)
2825{
2826 const VALUE *const passed_keywords = vm_ci_kwarg(ci)->keywords;
2827 const int kw_len = vm_ci_kwarg(ci)->keyword_len;
2828 const VALUE h = rb_hash_new_capa(kw_len);
2829 VALUE *sp = cfp->sp;
2830 int i;
2831
2832 for (i=0; i<kw_len; i++) {
2833 rb_hash_aset(h, passed_keywords[i], (sp - kw_len)[i]);
2834 }
2835 (sp-kw_len)[0] = h;
2836
2837 cfp->sp -= kw_len - 1;
2838 calling->argc -= kw_len - 1;
2839 calling->kw_splat = 1;
2840}
2841
2842static inline VALUE
2843vm_caller_setup_keyword_hash(const struct rb_callinfo *ci, VALUE keyword_hash)
2844{
2845 if (UNLIKELY(!RB_TYPE_P(keyword_hash, T_HASH))) {
2846 if (keyword_hash != Qnil) {
2847 /* Convert a non-hash keyword splat to a new hash */
2848 keyword_hash = rb_hash_dup(rb_to_hash_type(keyword_hash));
2849 }
2850 }
2851 else if (!IS_ARGS_KW_SPLAT_MUT(ci) && !RHASH_EMPTY_P(keyword_hash)) {
2852 /* Convert a hash keyword splat to a new hash unless
2853 * a mutable keyword splat was passed.
2854 * Skip allocating new hash for empty keyword splat, as empty
2855 * keyword splat will be ignored by both callers.
2856 */
2857 keyword_hash = rb_hash_dup(keyword_hash);
2858 }
2859 return keyword_hash;
2860}
2861
2862static inline void
2863CALLER_SETUP_ARG(struct rb_control_frame_struct *restrict cfp,
2864 struct rb_calling_info *restrict calling,
2865 const struct rb_callinfo *restrict ci, int max_args)
2866{
2867 if (UNLIKELY(IS_ARGS_SPLAT(ci))) {
2868 if (IS_ARGS_KW_SPLAT(ci)) {
2869 // f(*a, **kw)
2870 VM_ASSERT(calling->kw_splat == 1);
2871
2872 cfp->sp -= 2;
2873 calling->argc -= 2;
2874 VALUE ary = cfp->sp[0];
2875 VALUE kwh = vm_caller_setup_keyword_hash(ci, cfp->sp[1]);
2876
2877 // splat a
2878 if (vm_caller_setup_arg_splat(cfp, calling, ary, max_args)) return;
2879
2880 // put kw
2881 if (kwh != Qnil && !RHASH_EMPTY_P(kwh)) {
2882 if (UNLIKELY(calling->heap_argv)) {
2883 rb_ary_push(calling->heap_argv, kwh);
2884 ((struct RHash *)kwh)->basic.flags |= RHASH_PASS_AS_KEYWORDS;
2885 if (max_args != ALLOW_HEAP_ARGV_KEEP_KWSPLAT) {
2886 calling->kw_splat = 0;
2887 }
2888 }
2889 else {
2890 cfp->sp[0] = kwh;
2891 cfp->sp++;
2892 calling->argc++;
2893
2894 VM_ASSERT(calling->kw_splat == 1);
2895 }
2896 }
2897 else {
2898 calling->kw_splat = 0;
2899 }
2900 }
2901 else {
2902 // f(*a)
2903 VM_ASSERT(calling->kw_splat == 0);
2904
2905 cfp->sp -= 1;
2906 calling->argc -= 1;
2907 VALUE ary = cfp->sp[0];
2908
2909 if (vm_caller_setup_arg_splat(cfp, calling, ary, max_args)) {
2910 goto check_keyword;
2911 }
2912
2913 // check the last argument
2914 VALUE last_hash, argv_ary;
2915 if (UNLIKELY(argv_ary = calling->heap_argv)) {
2916 if (!IS_ARGS_KEYWORD(ci) &&
2917 RARRAY_LEN(argv_ary) > 0 &&
2918 RB_TYPE_P((last_hash = rb_ary_last(0, NULL, argv_ary)), T_HASH) &&
2919 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS)) {
2920
2921 rb_ary_pop(argv_ary);
2922 if (!RHASH_EMPTY_P(last_hash)) {
2923 rb_ary_push(argv_ary, rb_hash_dup(last_hash));
2924 calling->kw_splat = 1;
2925 }
2926 }
2927 }
2928 else {
2929check_keyword:
2930 if (!IS_ARGS_KEYWORD(ci) &&
2931 calling->argc > 0 &&
2932 RB_TYPE_P((last_hash = cfp->sp[-1]), T_HASH) &&
2933 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS)) {
2934
2935 if (RHASH_EMPTY_P(last_hash)) {
2936 calling->argc--;
2937 cfp->sp -= 1;
2938 }
2939 else {
2940 cfp->sp[-1] = rb_hash_dup(last_hash);
2941 calling->kw_splat = 1;
2942 }
2943 }
2944 }
2945 }
2946 }
2947 else if (UNLIKELY(IS_ARGS_KW_SPLAT(ci))) {
2948 // f(**kw)
2949 VM_ASSERT(calling->kw_splat == 1);
2950 VALUE kwh = vm_caller_setup_keyword_hash(ci, cfp->sp[-1]);
2951
2952 if (kwh == Qnil || RHASH_EMPTY_P(kwh)) {
2953 cfp->sp--;
2954 calling->argc--;
2955 calling->kw_splat = 0;
2956 }
2957 else {
2958 cfp->sp[-1] = kwh;
2959 }
2960 }
2961 else if (UNLIKELY(IS_ARGS_KEYWORD(ci))) {
2962 // f(k1:1, k2:2)
2963 VM_ASSERT(calling->kw_splat == 0);
2964
2965 /* This converts VM_CALL_KWARG style to VM_CALL_KW_SPLAT style
2966 * by creating a keyword hash.
2967 * So, vm_ci_flag(ci) & VM_CALL_KWARG is now inconsistent.
2968 */
2969 vm_caller_setup_arg_kw(cfp, calling, ci);
2970 }
2971}
2972
2973#define USE_OPT_HIST 0
2974
2975#if USE_OPT_HIST
2976#define OPT_HIST_MAX 64
2977static int opt_hist[OPT_HIST_MAX+1];
2978
2979__attribute__((destructor))
2980static void
2981opt_hist_show_results_at_exit(void)
2982{
2983 for (int i=0; i<OPT_HIST_MAX; i++) {
2984 ruby_debug_printf("opt_hist\t%d\t%d\n", i, opt_hist[i]);
2985 }
2986}
2987#endif
2988
2989static VALUE
2990vm_call_iseq_setup_normal_opt_start(rb_execution_context_t *ec, rb_control_frame_t *cfp,
2991 struct rb_calling_info *calling)
2992{
2993 const struct rb_callcache *cc = calling->cc;
2994 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
2995 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
2996 const int opt = calling->argc - lead_num;
2997 const int opt_num = ISEQ_BODY(iseq)->param.opt_num;
2998 const int opt_pc = (int)ISEQ_BODY(iseq)->param.opt_table[opt];
2999 const int param = ISEQ_BODY(iseq)->param.size;
3000 const int local = ISEQ_BODY(iseq)->local_table_size;
3001 const int delta = opt_num - opt;
3002
3003 RB_DEBUG_COUNTER_INC(ccf_iseq_opt);
3004
3005#if USE_OPT_HIST
3006 if (opt_pc < OPT_HIST_MAX) {
3007 opt_hist[opt]++;
3008 }
3009 else {
3010 opt_hist[OPT_HIST_MAX]++;
3011 }
3012#endif
3013
3014 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), opt_pc, param - delta, local);
3015}
3016
3017static VALUE
3018vm_call_iseq_setup_tailcall_opt_start(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3019 struct rb_calling_info *calling)
3020{
3021 const struct rb_callcache *cc = calling->cc;
3022 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3023 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3024 const int opt = calling->argc - lead_num;
3025 const int opt_pc = (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3026
3027 RB_DEBUG_COUNTER_INC(ccf_iseq_opt);
3028
3029#if USE_OPT_HIST
3030 if (opt_pc < OPT_HIST_MAX) {
3031 opt_hist[opt]++;
3032 }
3033 else {
3034 opt_hist[OPT_HIST_MAX]++;
3035 }
3036#endif
3037
3038 return vm_call_iseq_setup_tailcall(ec, cfp, calling, opt_pc);
3039}
3040
3041static void
3042args_setup_kw_parameters(rb_execution_context_t *const ec, const rb_iseq_t *const iseq, const rb_callable_method_entry_t *cme,
3043 VALUE *const passed_values, const int passed_keyword_len, const VALUE *const passed_keywords,
3044 VALUE *const locals);
3045
3046static VALUE
3047vm_call_iseq_forwardable(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3048 struct rb_calling_info *calling)
3049{
3050 const struct rb_callcache *cc = calling->cc;
3051 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3052 int param_size = ISEQ_BODY(iseq)->param.size;
3053 int local_size = ISEQ_BODY(iseq)->local_table_size;
3054
3055 // Setting up local size and param size
3056 VM_ASSERT(ISEQ_BODY(iseq)->param.flags.forwardable);
3057
3058 local_size = local_size + vm_ci_argc(calling->cd->ci);
3059 param_size = param_size + vm_ci_argc(calling->cd->ci);
3060
3061 cfp->sp[0] = (VALUE)calling->cd->ci;
3062
3063 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param_size, local_size);
3064}
3065
3066static VALUE
3067vm_call_iseq_setup_kwparm_kwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3068 struct rb_calling_info *calling)
3069{
3070 const struct rb_callinfo *ci = calling->cd->ci;
3071 const struct rb_callcache *cc = calling->cc;
3072
3073 VM_ASSERT(vm_ci_flag(ci) & VM_CALL_KWARG);
3074 RB_DEBUG_COUNTER_INC(ccf_iseq_kw1);
3075
3076 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3077 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3078 const struct rb_callinfo_kwarg *kw_arg = vm_ci_kwarg(ci);
3079 const int ci_kw_len = kw_arg->keyword_len;
3080 const VALUE * const ci_keywords = kw_arg->keywords;
3081 VALUE *argv = cfp->sp - calling->argc;
3082 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3083 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3084 VALUE * const ci_kws = ALLOCA_N(VALUE, ci_kw_len);
3085 MEMCPY(ci_kws, argv + lead_num, VALUE, ci_kw_len);
3086 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), ci_kws, ci_kw_len, ci_keywords, klocals);
3087
3088 int param = ISEQ_BODY(iseq)->param.size;
3089 int local = ISEQ_BODY(iseq)->local_table_size;
3090 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
3091}
3092
3093static VALUE
3094vm_call_iseq_setup_kwparm_nokwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3095 struct rb_calling_info *calling)
3096{
3097 const struct rb_callinfo *MAYBE_UNUSED(ci) = calling->cd->ci;
3098 const struct rb_callcache *cc = calling->cc;
3099
3100 VM_ASSERT((vm_ci_flag(ci) & VM_CALL_KWARG) == 0);
3101 RB_DEBUG_COUNTER_INC(ccf_iseq_kw2);
3102
3103 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3104 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3105 VALUE * const argv = cfp->sp - calling->argc;
3106 VALUE * const klocals = argv + kw_param->bits_start - kw_param->num;
3107
3108 int i;
3109 for (i=0; i<kw_param->num; i++) {
3110 klocals[i] = kw_param->default_values[i];
3111 }
3112 klocals[i] = INT2FIX(0); // kw specify flag
3113 // NOTE:
3114 // nobody check this value, but it should be cleared because it can
3115 // points invalid VALUE (T_NONE objects, raw pointer and so on).
3116
3117 int param = ISEQ_BODY(iseq)->param.size;
3118 int local = ISEQ_BODY(iseq)->local_table_size;
3119 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
3120}
3121
3122static VALUE builtin_invoker0(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr);
3123
3124static VALUE
3125vm_call_single_noarg_leaf_builtin(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3126 struct rb_calling_info *calling)
3127{
3128 const struct rb_builtin_function *bf = calling->cc->aux_.bf;
3129 cfp->sp -= (calling->argc + 1);
3130 rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr;
3131 return builtin_invoker0(ec, calling->recv, NULL, func_ptr);
3132}
3133
3134VALUE rb_gen_method_name(VALUE owner, VALUE name); // in vm_backtrace.c
3135
3136static void
3137warn_unused_block(const rb_callable_method_entry_t *cme, const rb_iseq_t *iseq, void *pc)
3138{
3139 rb_vm_t *vm = GET_VM();
3140 set_table *dup_check_table = &vm->unused_block_warning_table;
3141 st_data_t key;
3142 bool strict_unused_block = rb_warning_category_enabled_p(RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK);
3143
3144 union {
3145 VALUE v;
3146 unsigned char b[SIZEOF_VALUE];
3147 } k1 = {
3148 .v = (VALUE)pc,
3149 }, k2 = {
3150 .v = (VALUE)cme->def,
3151 };
3152
3153 // relax check
3154 if (!strict_unused_block) {
3155 key = (st_data_t)cme->def->original_id;
3156
3157 if (set_table_lookup(dup_check_table, key)) {
3158 return;
3159 }
3160 }
3161
3162 // strict check
3163 // make unique key from pc and me->def pointer
3164 key = 0;
3165 for (int i=0; i<SIZEOF_VALUE; i++) {
3166 // fprintf(stderr, "k1:%3d k2:%3d\n", k1.b[i], k2.b[SIZEOF_VALUE-1-i]);
3167 key |= (st_data_t)(k1.b[i] ^ k2.b[SIZEOF_VALUE-1-i]) << (8 * i);
3168 }
3169
3170 if (0) {
3171 fprintf(stderr, "SIZEOF_VALUE:%d\n", SIZEOF_VALUE);
3172 fprintf(stderr, "pc:%p def:%p\n", pc, (void *)cme->def);
3173 fprintf(stderr, "key:%p\n", (void *)key);
3174 }
3175
3176 // duplication check
3177 if (set_insert(dup_check_table, key)) {
3178 // already shown
3179 }
3180 else if (RTEST(ruby_verbose) || strict_unused_block) {
3181 VALUE m_loc = rb_method_entry_location((const rb_method_entry_t *)cme);
3182 VALUE name = rb_gen_method_name(cme->defined_class, rb_iseq_base_label(iseq));
3183
3184 if (!NIL_P(m_loc)) {
3185 rb_warn("the block passed to '%"PRIsVALUE"' defined at %"PRIsVALUE":%"PRIsVALUE" may be ignored",
3186 name, RARRAY_AREF(m_loc, 0), RARRAY_AREF(m_loc, 1));
3187 }
3188 else {
3189 rb_warn("the block may be ignored because '%"PRIsVALUE"' does not use a block", name);
3190 }
3191 }
3192}
3193
3194static inline int
3195vm_callee_setup_arg(rb_execution_context_t *ec, struct rb_calling_info *calling,
3196 const rb_iseq_t *iseq, VALUE *argv, int param_size, int local_size)
3197{
3198 const struct rb_callinfo *ci = calling->cd->ci;
3199 const struct rb_callcache *cc = calling->cc;
3200
3201 VM_ASSERT((vm_ci_argc(ci), 1));
3202 VM_ASSERT(vm_cc_cme(cc) != NULL);
3203
3204 if (UNLIKELY(!ISEQ_BODY(iseq)->param.flags.use_block &&
3205 calling->block_handler != VM_BLOCK_HANDLER_NONE &&
3206 !(vm_ci_flag(calling->cd->ci) & (VM_CALL_OPT_SEND | VM_CALL_SUPER)))) {
3207 warn_unused_block(vm_cc_cme(cc), iseq, (void *)CFP_PC(ec->cfp));
3208 }
3209
3210 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_KW_SPLAT))) {
3211 if (LIKELY(rb_simple_iseq_p(iseq))) {
3212 rb_control_frame_t *cfp = ec->cfp;
3213 int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3214 CALLER_SETUP_ARG(cfp, calling, ci, lead_num);
3215
3216 if (calling->argc != lead_num) {
3217 argument_arity_error(ec, iseq, vm_cc_cme(cc), calling->argc, lead_num, lead_num);
3218 }
3219
3220 //VM_ASSERT(ci == calling->cd->ci);
3221 VM_ASSERT(cc == calling->cc);
3222
3223 if (vm_call_iseq_optimizable_p(ci, cc)) {
3224 if ((ISEQ_BODY(iseq)->builtin_attrs & BUILTIN_ATTR_SINGLE_NOARG_LEAF) && ruby_vm_c_events_enabled == 0) {
3225 VM_ASSERT(ISEQ_BODY(iseq)->builtin_attrs & BUILTIN_ATTR_LEAF);
3226 vm_cc_bf_set(cc, (void *)ISEQ_BODY(iseq)->iseq_encoded[1]);
3227 CC_SET_FASTPATH(cc, vm_call_single_noarg_leaf_builtin, true);
3228 }
3229 else {
3230 CC_SET_FASTPATH(cc, vm_call_iseq_setup_func(ci, param_size, local_size), true);
3231 }
3232 }
3233 return 0;
3234 }
3235 else if (rb_iseq_only_optparam_p(iseq)) {
3236 rb_control_frame_t *cfp = ec->cfp;
3237
3238 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3239 const int opt_num = ISEQ_BODY(iseq)->param.opt_num;
3240
3241 CALLER_SETUP_ARG(cfp, calling, ci, lead_num + opt_num);
3242 const int argc = calling->argc;
3243 const int opt = argc - lead_num;
3244
3245 if (opt < 0 || opt > opt_num) {
3246 argument_arity_error(ec, iseq, vm_cc_cme(cc), argc, lead_num, lead_num + opt_num);
3247 }
3248
3249 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) {
3250 CC_SET_FASTPATH(cc, vm_call_iseq_setup_normal_opt_start,
3251 !IS_ARGS_SPLAT(ci) && !IS_ARGS_KEYWORD(ci) &&
3252 vm_call_cacheable(ci, cc));
3253 }
3254 else {
3255 CC_SET_FASTPATH(cc, vm_call_iseq_setup_tailcall_opt_start,
3256 !IS_ARGS_SPLAT(ci) && !IS_ARGS_KEYWORD(ci) &&
3257 vm_call_cacheable(ci, cc));
3258 }
3259
3260 /* initialize opt vars for self-references */
3261 VM_ASSERT((int)ISEQ_BODY(iseq)->param.size == lead_num + opt_num);
3262 for (int i=argc; i<lead_num + opt_num; i++) {
3263 argv[i] = Qnil;
3264 }
3265 return (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3266 }
3267 else if (rb_iseq_only_kwparam_p(iseq) && !IS_ARGS_SPLAT(ci)) {
3268 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3269 const int argc = calling->argc;
3270 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3271
3272 if (vm_ci_flag(ci) & VM_CALL_KWARG) {
3273 const struct rb_callinfo_kwarg *kw_arg = vm_ci_kwarg(ci);
3274
3275 if (argc - kw_arg->keyword_len == lead_num) {
3276 const int ci_kw_len = kw_arg->keyword_len;
3277 const VALUE * const ci_keywords = kw_arg->keywords;
3278 VALUE * const ci_kws = ALLOCA_N(VALUE, ci_kw_len);
3279 MEMCPY(ci_kws, argv + lead_num, VALUE, ci_kw_len);
3280
3281 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3282 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), ci_kws, ci_kw_len, ci_keywords, klocals);
3283
3284 CC_SET_FASTPATH(cc, vm_call_iseq_setup_kwparm_kwarg,
3285 vm_call_cacheable(ci, cc));
3286
3287 return 0;
3288 }
3289 }
3290 else if (argc == lead_num) {
3291 /* no kwarg */
3292 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3293 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), NULL, 0, NULL, klocals);
3294
3295 if (klocals[kw_param->num] == INT2FIX(0)) {
3296 /* copy from default_values */
3297 CC_SET_FASTPATH(cc, vm_call_iseq_setup_kwparm_nokwarg,
3298 vm_call_cacheable(ci, cc));
3299 }
3300
3301 return 0;
3302 }
3303 }
3304 }
3305
3306 // Called iseq is using ... param
3307 // def foo(...) # <- iseq for foo will have "forwardable"
3308 //
3309 // We want to set the `...` local to the caller's CI
3310 // foo(1, 2) # <- the ci for this should end up as `...`
3311 //
3312 // So hopefully the stack looks like:
3313 //
3314 // => 1
3315 // => 2
3316 // => *
3317 // => **
3318 // => &
3319 // => ... # <- points at `foo`s CI
3320 // => cref_or_me
3321 // => specval
3322 // => type
3323 //
3324 if (ISEQ_BODY(iseq)->param.flags.forwardable) {
3325 bool can_fastpath = true;
3326
3327 if ((vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
3328 struct rb_forwarding_call_data * forward_cd = (struct rb_forwarding_call_data *)calling->cd;
3329 if (vm_ci_argc(ci) != vm_ci_argc(forward_cd->caller_ci)) {
3330 ci = vm_ci_new_runtime(
3331 vm_ci_mid(ci),
3332 vm_ci_flag(ci),
3333 vm_ci_argc(ci),
3334 vm_ci_kwarg(ci));
3335 }
3336 else {
3337 ci = forward_cd->caller_ci;
3338 }
3339 can_fastpath = false;
3340 }
3341 // C functions calling iseqs will stack allocate a CI,
3342 // so we need to convert it to heap allocated
3343 if (!vm_ci_markable(ci)) {
3344 ci = vm_ci_new_runtime(
3345 vm_ci_mid(ci),
3346 vm_ci_flag(ci),
3347 vm_ci_argc(ci),
3348 vm_ci_kwarg(ci));
3349 can_fastpath = false;
3350 }
3351 argv[param_size - 1] = (VALUE)ci;
3352 CC_SET_FASTPATH(cc, vm_call_iseq_forwardable, can_fastpath);
3353 return 0;
3354 }
3355
3356 return setup_parameters_complex(ec, iseq, calling, ci, argv, arg_setup_method);
3357}
3358
3359static void
3360vm_adjust_stack_forwarding(const struct rb_execution_context_struct *ec, struct rb_control_frame_struct *cfp, int argc, VALUE splat)
3361{
3362 // This case is when the caller is using a ... parameter.
3363 // For example `bar(...)`. The call info will have VM_CALL_FORWARDING
3364 // In this case the caller's caller's CI will be on the stack.
3365 //
3366 // For example:
3367 //
3368 // def bar(a, b); a + b; end
3369 // def foo(...); bar(...); end
3370 // foo(1, 2) # <- this CI will be on the stack when we call `bar(...)`
3371 //
3372 // Stack layout will be:
3373 //
3374 // > 1
3375 // > 2
3376 // > CI for foo(1, 2)
3377 // > cref_or_me
3378 // > specval
3379 // > type
3380 // > receiver
3381 // > CI for foo(1, 2), via `getlocal ...`
3382 // > ( SP points here )
3383 const VALUE * lep = VM_CF_LEP(cfp);
3384
3385 const rb_iseq_t *iseq;
3386
3387 // If we're in an escaped environment (lambda for example), get the iseq
3388 // from the captured env.
3389 if (VM_ENV_FLAGS(lep, VM_ENV_FLAG_ESCAPED)) {
3390 rb_env_t * env = (rb_env_t *)lep[VM_ENV_DATA_INDEX_ENV];
3391 iseq = env->iseq;
3392 }
3393 else { // Otherwise use the lep to find the caller
3394 iseq = CFP_ISEQ(rb_vm_search_cf_from_ep(ec, cfp, lep));
3395 }
3396
3397 // Our local storage is below the args we need to copy
3398 int local_size = ISEQ_BODY(iseq)->local_table_size + argc;
3399
3400 const VALUE * from = lep - (local_size + VM_ENV_DATA_SIZE - 1); // 2 for EP values
3401 VALUE * to = cfp->sp - 1; // clobber the CI
3402
3403 if (RTEST(splat)) {
3404 to -= 1; // clobber the splat array
3405 CHECK_VM_STACK_OVERFLOW0(cfp, to, RARRAY_LEN(splat));
3406 MEMCPY(to, RARRAY_CONST_PTR(splat), VALUE, RARRAY_LEN(splat));
3407 to += RARRAY_LEN(splat);
3408 }
3409
3410 CHECK_VM_STACK_OVERFLOW0(cfp, to, argc);
3411 MEMCPY(to, from, VALUE, argc);
3412 cfp->sp = to + argc;
3413
3414 // Stack layout should now be:
3415 //
3416 // > 1
3417 // > 2
3418 // > CI for foo(1, 2)
3419 // > cref_or_me
3420 // > specval
3421 // > type
3422 // > receiver
3423 // > 1
3424 // > 2
3425 // > ( SP points here )
3426}
3427
3428static VALUE
3429vm_call_iseq_setup(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3430{
3431 RB_DEBUG_COUNTER_INC(ccf_iseq_setup);
3432
3433 const struct rb_callcache *cc = calling->cc;
3434 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3435 int param_size = ISEQ_BODY(iseq)->param.size;
3436 int local_size = ISEQ_BODY(iseq)->local_table_size;
3437
3438 RUBY_ASSERT(!ISEQ_BODY(iseq)->param.flags.forwardable);
3439
3440 const int opt_pc = vm_callee_setup_arg(ec, calling, iseq, cfp->sp - calling->argc, param_size, local_size);
3441 return vm_call_iseq_setup_2(ec, cfp, calling, opt_pc, param_size, local_size);
3442}
3443
3444static VALUE
3445vm_call_iseq_fwd_setup(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3446{
3447 RB_DEBUG_COUNTER_INC(ccf_iseq_setup);
3448
3449 const struct rb_callcache *cc = calling->cc;
3450 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3451 int param_size = ISEQ_BODY(iseq)->param.size;
3452 int local_size = ISEQ_BODY(iseq)->local_table_size;
3453
3454 RUBY_ASSERT(ISEQ_BODY(iseq)->param.flags.forwardable);
3455
3456 // Setting up local size and param size
3457 local_size = local_size + vm_ci_argc(calling->cd->ci);
3458 param_size = param_size + vm_ci_argc(calling->cd->ci);
3459
3460 const int opt_pc = vm_callee_setup_arg(ec, calling, iseq, cfp->sp - calling->argc, param_size, local_size);
3461 return vm_call_iseq_setup_2(ec, cfp, calling, opt_pc, param_size, local_size);
3462}
3463
3464static inline VALUE
3465vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
3466 int opt_pc, int param_size, int local_size)
3467{
3468 const struct rb_callinfo *ci = calling->cd->ci;
3469 const struct rb_callcache *cc = calling->cc;
3470
3471 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) {
3472 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), opt_pc, param_size, local_size);
3473 }
3474 else {
3475 return vm_call_iseq_setup_tailcall(ec, cfp, calling, opt_pc);
3476 }
3477}
3478
3479static inline VALUE
3480vm_call_iseq_setup_normal(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const rb_callable_method_entry_t *me,
3481 int opt_pc, int param_size, int local_size)
3482{
3483 const rb_iseq_t *iseq = def_iseq_ptr(me->def);
3484 VALUE *argv = cfp->sp - calling->argc;
3485 VALUE *sp = argv + param_size;
3486 cfp->sp = argv - 1 /* recv */;
3487
3488 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL, calling->recv,
3489 calling->block_handler, (VALUE)me,
3490 ISEQ_BODY(iseq)->iseq_encoded + opt_pc, sp,
3491 local_size - param_size,
3492 ISEQ_BODY(iseq)->stack_max);
3493 return Qundef;
3494}
3495
3496static inline VALUE
3497vm_call_iseq_setup_tailcall(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc)
3498{
3499 const struct rb_callcache *cc = calling->cc;
3500 unsigned int i;
3501 VALUE *argv = cfp->sp - calling->argc;
3502 const rb_callable_method_entry_t *me = vm_cc_cme(cc);
3503 const rb_iseq_t *iseq = def_iseq_ptr(me->def);
3504 VALUE *src_argv = argv;
3505 VALUE *sp_orig, *sp;
3506 VALUE finish_flag = VM_FRAME_FINISHED_P(cfp) ? VM_FRAME_FLAG_FINISH : 0;
3507
3508 if (VM_BH_FROM_CFP_P(calling->block_handler, cfp)) {
3509 struct rb_captured_block *dst_captured = VM_CFP_TO_CAPTURED_BLOCK(RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp));
3510 const struct rb_captured_block *src_captured = VM_BH_TO_CAPT_BLOCK(calling->block_handler);
3511 dst_captured->code.val = src_captured->code.val;
3512 if (VM_BH_ISEQ_BLOCK_P(calling->block_handler)) {
3513 calling->block_handler = VM_BH_FROM_ISEQ_BLOCK(dst_captured);
3514 }
3515 else {
3516 calling->block_handler = VM_BH_FROM_IFUNC_BLOCK(dst_captured);
3517 }
3518 }
3519
3520 vm_pop_frame(ec, cfp, cfp->ep);
3521 cfp = ec->cfp;
3522
3523 sp_orig = sp = cfp->sp;
3524
3525 /* push self */
3526 sp[0] = calling->recv;
3527 sp++;
3528
3529 /* copy arguments */
3530 for (i=0; i < ISEQ_BODY(iseq)->param.size; i++) {
3531 *sp++ = src_argv[i];
3532 }
3533
3534 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL | finish_flag,
3535 calling->recv, calling->block_handler, (VALUE)me,
3536 ISEQ_BODY(iseq)->iseq_encoded + opt_pc, sp,
3537 ISEQ_BODY(iseq)->local_table_size - ISEQ_BODY(iseq)->param.size,
3538 ISEQ_BODY(iseq)->stack_max);
3539
3540 cfp->sp = sp_orig;
3541
3542 return Qundef;
3543}
3544
3545static void
3546ractor_unsafe_check(void)
3547{
3548 if (!rb_ractor_main_p()) {
3549 rb_raise(rb_eRactorUnsafeError, "ractor unsafe method called from not main ractor");
3550 }
3551}
3552
3553static VALUE
3554call_cfunc_m2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3555{
3556 ractor_unsafe_check();
3557 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3558 return (*f)(recv, rb_ary_new4(argc, argv));
3559}
3560
3561static VALUE
3562call_cfunc_m1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3563{
3564 ractor_unsafe_check();
3565 VALUE(*f)(int, const VALUE *, VALUE) = (VALUE(*)(int, const VALUE *, VALUE))func;
3566 return (*f)(argc, argv, recv);
3567}
3568
3569static VALUE
3570call_cfunc_0(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3571{
3572 ractor_unsafe_check();
3573 VALUE(*f)(VALUE) = (VALUE(*)(VALUE))func;
3574 return (*f)(recv);
3575}
3576
3577static VALUE
3578call_cfunc_1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3579{
3580 ractor_unsafe_check();
3581 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3582 return (*f)(recv, argv[0]);
3583}
3584
3585static VALUE
3586call_cfunc_2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3587{
3588 ractor_unsafe_check();
3589 VALUE(*f)(VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE))func;
3590 return (*f)(recv, argv[0], argv[1]);
3591}
3592
3593static VALUE
3594call_cfunc_3(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3595{
3596 ractor_unsafe_check();
3597 VALUE(*f)(VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE))func;
3598 return (*f)(recv, argv[0], argv[1], argv[2]);
3599}
3600
3601static VALUE
3602call_cfunc_4(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3603{
3604 ractor_unsafe_check();
3605 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE))func;
3606 return (*f)(recv, argv[0], argv[1], argv[2], argv[3]);
3607}
3608
3609static VALUE
3610call_cfunc_5(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3611{
3612 ractor_unsafe_check();
3613 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3614 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4]);
3615}
3616
3617static VALUE
3618call_cfunc_6(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3619{
3620 ractor_unsafe_check();
3621 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3622 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
3623}
3624
3625static VALUE
3626call_cfunc_7(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3627{
3628 ractor_unsafe_check();
3629 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3630 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
3631}
3632
3633static VALUE
3634call_cfunc_8(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3635{
3636 ractor_unsafe_check();
3637 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3638 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
3639}
3640
3641static VALUE
3642call_cfunc_9(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3643{
3644 ractor_unsafe_check();
3645 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3646 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
3647}
3648
3649static VALUE
3650call_cfunc_10(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3651{
3652 ractor_unsafe_check();
3653 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3654 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
3655}
3656
3657static VALUE
3658call_cfunc_11(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3659{
3660 ractor_unsafe_check();
3661 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3662 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
3663}
3664
3665static VALUE
3666call_cfunc_12(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3667{
3668 ractor_unsafe_check();
3669 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3670 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
3671}
3672
3673static VALUE
3674call_cfunc_13(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3675{
3676 ractor_unsafe_check();
3677 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3678 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
3679}
3680
3681static VALUE
3682call_cfunc_14(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3683{
3684 ractor_unsafe_check();
3685 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3686 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
3687}
3688
3689static VALUE
3690call_cfunc_15(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3691{
3692 ractor_unsafe_check();
3693 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3694 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
3695}
3696
3697static VALUE
3698ractor_safe_call_cfunc_m2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3699{
3700 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3701 return (*f)(recv, rb_ary_new4(argc, argv));
3702}
3703
3704static VALUE
3705ractor_safe_call_cfunc_m1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3706{
3707 VALUE(*f)(int, const VALUE *, VALUE) = (VALUE(*)(int, const VALUE *, VALUE))func;
3708 return (*f)(argc, argv, recv);
3709}
3710
3711static VALUE
3712ractor_safe_call_cfunc_0(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3713{
3714 VALUE(*f)(VALUE) = (VALUE(*)(VALUE))func;
3715 return (*f)(recv);
3716}
3717
3718static VALUE
3719ractor_safe_call_cfunc_1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3720{
3721 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3722 return (*f)(recv, argv[0]);
3723}
3724
3725static VALUE
3726ractor_safe_call_cfunc_2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3727{
3728 VALUE(*f)(VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE))func;
3729 return (*f)(recv, argv[0], argv[1]);
3730}
3731
3732static VALUE
3733ractor_safe_call_cfunc_3(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3734{
3735 VALUE(*f)(VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE))func;
3736 return (*f)(recv, argv[0], argv[1], argv[2]);
3737}
3738
3739static VALUE
3740ractor_safe_call_cfunc_4(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3741{
3742 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE))func;
3743 return (*f)(recv, argv[0], argv[1], argv[2], argv[3]);
3744}
3745
3746static VALUE
3747ractor_safe_call_cfunc_5(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3748{
3749 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3750 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4]);
3751}
3752
3753static VALUE
3754ractor_safe_call_cfunc_6(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3755{
3756 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3757 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
3758}
3759
3760static VALUE
3761ractor_safe_call_cfunc_7(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3762{
3763 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3764 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
3765}
3766
3767static VALUE
3768ractor_safe_call_cfunc_8(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3769{
3770 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3771 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
3772}
3773
3774static VALUE
3775ractor_safe_call_cfunc_9(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3776{
3777 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3778 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
3779}
3780
3781static VALUE
3782ractor_safe_call_cfunc_10(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3783{
3784 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3785 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
3786}
3787
3788static VALUE
3789ractor_safe_call_cfunc_11(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3790{
3791 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3792 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
3793}
3794
3795static VALUE
3796ractor_safe_call_cfunc_12(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3797{
3798 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3799 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
3800}
3801
3802static VALUE
3803ractor_safe_call_cfunc_13(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3804{
3805 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3806 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
3807}
3808
3809static VALUE
3810ractor_safe_call_cfunc_14(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3811{
3812 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3813 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
3814}
3815
3816static VALUE
3817ractor_safe_call_cfunc_15(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3818{
3819 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3820 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
3821}
3822
3823static inline int
3824vm_cfp_consistent_p(rb_execution_context_t *ec, const rb_control_frame_t *reg_cfp)
3825{
3826 const int ov_flags = RAISED_STACKOVERFLOW;
3827 if (LIKELY(reg_cfp == ec->cfp + 1)) return TRUE;
3828 if (rb_ec_raised_p(ec, ov_flags)) {
3829 rb_ec_raised_reset(ec, ov_flags);
3830 return TRUE;
3831 }
3832 return FALSE;
3833}
3834
3835#define CHECK_CFP_CONSISTENCY(func) \
3836 (LIKELY(vm_cfp_consistent_p(ec, reg_cfp)) ? (void)0 : \
3837 rb_bug(func ": cfp consistency error (%p, %p)", (void *)reg_cfp, (void *)(ec->cfp+1)))
3838
3839static inline
3840const rb_method_cfunc_t *
3841vm_method_cfunc_entry(const rb_callable_method_entry_t *me)
3842{
3843#if VM_DEBUG_VERIFY_METHOD_CACHE
3844 switch (me->def->type) {
3845 case VM_METHOD_TYPE_CFUNC:
3846 case VM_METHOD_TYPE_NOTIMPLEMENTED:
3847 break;
3848# define METHOD_BUG(t) case VM_METHOD_TYPE_##t: rb_bug("wrong method type: " #t)
3849 METHOD_BUG(ISEQ);
3850 METHOD_BUG(ATTRSET);
3851 METHOD_BUG(IVAR);
3852 METHOD_BUG(BMETHOD);
3853 METHOD_BUG(ZSUPER);
3854 METHOD_BUG(UNDEF);
3855 METHOD_BUG(OPTIMIZED);
3856 METHOD_BUG(MISSING);
3857 METHOD_BUG(REFINED);
3858 METHOD_BUG(ALIAS);
3859# undef METHOD_BUG
3860 default:
3861 rb_bug("wrong method type: %d", me->def->type);
3862 }
3863#endif
3864 return UNALIGNED_MEMBER_PTR(me->def, body.cfunc);
3865}
3866
3867static VALUE
3868vm_call_cfunc_with_frame_(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling,
3869 int argc, VALUE *argv, VALUE *stack_bottom)
3870{
3871 RB_DEBUG_COUNTER_INC(ccf_cfunc_with_frame);
3872 const struct rb_callinfo *ci = calling->cd->ci;
3873 const struct rb_callcache *cc = calling->cc;
3874 VALUE val;
3875 const rb_callable_method_entry_t *me = vm_cc_cme(cc);
3876 const rb_method_cfunc_t *cfunc = vm_method_cfunc_entry(me);
3877
3878 VALUE recv = calling->recv;
3879 VALUE block_handler = calling->block_handler;
3880 VALUE frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
3881
3882 if (UNLIKELY(calling->kw_splat)) {
3883 frame_type |= VM_FRAME_FLAG_CFRAME_KW;
3884 }
3885
3886 VM_ASSERT(reg_cfp == ec->cfp);
3887
3888 RUBY_DTRACE_CMETHOD_ENTRY_HOOK(ec, me->owner, me->def->original_id);
3889 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_CALL, recv, me->def->original_id, vm_ci_mid(ci), me->owner, Qundef);
3890
3891 vm_push_frame(ec, NULL, frame_type, recv,
3892 block_handler, (VALUE)me,
3893 0, ec->cfp->sp, 0, 0);
3894
3895 int len = cfunc->argc;
3896 if (len >= 0) rb_check_arity(argc, len, len);
3897
3898 reg_cfp->sp = stack_bottom;
3899 val = (*cfunc->invoker)(recv, argc, argv, cfunc->func);
3900
3901 CHECK_CFP_CONSISTENCY("vm_call_cfunc");
3902
3903 rb_vm_pop_frame(ec);
3904
3905 VM_ASSERT(ec->cfp->sp == stack_bottom);
3906
3907 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, recv, me->def->original_id, vm_ci_mid(ci), me->owner, val);
3908 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
3909
3910 return val;
3911}
3912
3913// Push a C method frame for a given cme. This is called when JIT code skipped
3914// pushing a frame but the C method reached a point where a frame is needed.
3915void
3916rb_vm_push_cfunc_frame(const rb_callable_method_entry_t *cme, int recv_idx)
3917{
3918 VM_ASSERT(cme->def->type == VM_METHOD_TYPE_CFUNC);
3919 rb_execution_context_t *ec = GET_EC();
3920 VALUE *sp = ec->cfp->sp;
3921 VALUE recv = *(sp - recv_idx - 1);
3922 VALUE frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
3923 VALUE block_handler = VM_BLOCK_HANDLER_NONE;
3924#if VM_CHECK_MODE > 0
3925 // Clean up the stack canary since we're about to satisfy the "leaf or lazy push" assumption
3926 *(GET_EC()->cfp->sp) = Qfalse;
3927#endif
3928 vm_push_frame(ec, NULL, frame_type, recv, block_handler, (VALUE)cme, 0, ec->cfp->sp, 0, 0);
3929}
3930
3931// If true, cc->call needs to include `CALLER_SETUP_ARG` (i.e. can't be skipped in fastpath)
3932bool
3933rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci)
3934{
3935 return IS_ARGS_SPLAT(ci) || IS_ARGS_KW_OR_KW_SPLAT(ci);
3936}
3937
3938static VALUE
3939vm_call_cfunc_with_frame(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3940{
3941 int argc = calling->argc;
3942 VALUE *stack_bottom = reg_cfp->sp - argc - 1;
3943 VALUE *argv = &stack_bottom[1];
3944
3945 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, argv, stack_bottom);
3946}
3947
3948static VALUE
3949vm_call_cfunc_other(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3950{
3951 const struct rb_callinfo *ci = calling->cd->ci;
3952 RB_DEBUG_COUNTER_INC(ccf_cfunc_other);
3953
3954 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT);
3955 VALUE argv_ary;
3956 if (UNLIKELY(argv_ary = calling->heap_argv)) {
3957 VM_ASSERT(!IS_ARGS_KEYWORD(ci));
3958 int argc = RARRAY_LENINT(argv_ary);
3959 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
3960 VALUE *stack_bottom = reg_cfp->sp - 2;
3961
3962 VM_ASSERT(calling->argc == 1);
3963 VM_ASSERT(RB_TYPE_P(argv_ary, T_ARRAY));
3964 VM_ASSERT(RBASIC_CLASS(argv_ary) == 0); // hidden ary
3965
3966 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, argv, stack_bottom);
3967 }
3968 else {
3969 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_with_frame, !rb_splat_or_kwargs_p(ci) && !calling->kw_splat && !(vm_ci_flag(ci) & VM_CALL_FORWARDING));
3970
3971 return vm_call_cfunc_with_frame(ec, reg_cfp, calling);
3972 }
3973}
3974
3975static inline VALUE
3976vm_call_cfunc_array_argv(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int stack_offset, int argc_offset)
3977{
3978 VALUE argv_ary = reg_cfp->sp[-1 - stack_offset];
3979 int argc = RARRAY_LENINT(argv_ary) - argc_offset;
3980
3981 if (UNLIKELY(argc > VM_ARGC_STACK_MAX)) {
3982 return vm_call_cfunc_other(ec, reg_cfp, calling);
3983 }
3984
3985 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
3986 calling->kw_splat = 0;
3987 int i;
3988 VALUE *stack_bottom = reg_cfp->sp - 2 - stack_offset;
3989 VALUE *sp = stack_bottom;
3990 CHECK_VM_STACK_OVERFLOW(reg_cfp, argc);
3991 for(i = 0; i < argc; i++) {
3992 *++sp = argv[i];
3993 }
3994 reg_cfp->sp = sp+1;
3995
3996 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, stack_bottom+1, stack_bottom);
3997}
3998
3999static inline VALUE
4000vm_call_cfunc_only_splat(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4001{
4002 RB_DEBUG_COUNTER_INC(ccf_cfunc_only_splat);
4003 VALUE argv_ary = reg_cfp->sp[-1];
4004 int argc = RARRAY_LENINT(argv_ary);
4005 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
4006 VALUE last_hash;
4007 int argc_offset = 0;
4008
4009 if (UNLIKELY(argc > 0 &&
4010 RB_TYPE_P((last_hash = argv[argc-1]), T_HASH) &&
4011 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS))) {
4012 if (!RHASH_EMPTY_P(last_hash)) {
4013 return vm_call_cfunc_other(ec, reg_cfp, calling);
4014 }
4015 argc_offset++;
4016 }
4017 return vm_call_cfunc_array_argv(ec, reg_cfp, calling, 0, argc_offset);
4018}
4019
4020static inline VALUE
4021vm_call_cfunc_only_splat_kw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4022{
4023 RB_DEBUG_COUNTER_INC(ccf_cfunc_only_splat_kw);
4024 VALUE keyword_hash = reg_cfp->sp[-1];
4025
4026 if (keyword_hash == Qnil || (RB_TYPE_P(keyword_hash, T_HASH) && RHASH_EMPTY_P(keyword_hash))) {
4027 return vm_call_cfunc_array_argv(ec, reg_cfp, calling, 1, 0);
4028 }
4029
4030 return vm_call_cfunc_other(ec, reg_cfp, calling);
4031}
4032
4033static VALUE
4034vm_call_cfunc(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4035{
4036 const struct rb_callinfo *ci = calling->cd->ci;
4037 RB_DEBUG_COUNTER_INC(ccf_cfunc);
4038
4039 if (IS_ARGS_SPLAT(ci) && !(vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
4040 if (!IS_ARGS_KW_SPLAT(ci) && vm_ci_argc(ci) == 1) {
4041 // f(*a)
4042 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_only_splat, TRUE);
4043 return vm_call_cfunc_only_splat(ec, reg_cfp, calling);
4044 }
4045 if (IS_ARGS_KW_SPLAT(ci) && vm_ci_argc(ci) == 2) {
4046 // f(*a, **kw)
4047 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_only_splat_kw, TRUE);
4048 return vm_call_cfunc_only_splat_kw(ec, reg_cfp, calling);
4049 }
4050 }
4051
4052 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_other, TRUE);
4053 return vm_call_cfunc_other(ec, reg_cfp, calling);
4054}
4055
4056static VALUE
4057vm_call_ivar(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4058{
4059 const struct rb_callcache *cc = calling->cc;
4060 RB_DEBUG_COUNTER_INC(ccf_ivar);
4061 cfp->sp -= 1;
4062 VALUE ivar = vm_getivar(calling->recv, vm_cc_cme(cc)->def->body.attr.id, NULL, NULL, cc, TRUE, Qnil);
4063 return ivar;
4064}
4065
4066static VALUE
4067vm_call_attrset_direct(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_callcache *cc, VALUE obj)
4068{
4069 RB_DEBUG_COUNTER_INC(ccf_attrset);
4070 VALUE val = *(cfp->sp - 1);
4071 cfp->sp -= 2;
4072 rb_setivar_cache cache = rb_setivar_cache_unpack(vm_cc_atomic_cache_read(cc));
4073 ID id = vm_cc_cme(cc)->def->body.attr.id;
4074 rb_check_frozen(obj);
4075 VALUE res = vm_setivar(obj, val, cache);
4076 if (UNDEF_P(res)) {
4077 switch (BUILTIN_TYPE(obj)) {
4078 case T_OBJECT:
4079 break;
4080 case T_CLASS:
4081 case T_MODULE:
4082 {
4083 res = vm_setivar_class(obj, val, cache);
4084 if (!UNDEF_P(res)) {
4085 return res;
4086 }
4087 }
4088 break;
4089 default:
4090 {
4091 res = vm_setivar_default(obj, id, val, cache);
4092 if (!UNDEF_P(res)) {
4093 return res;
4094 }
4095 }
4096 }
4097 res = vm_setivar_slowpath_attr(obj, id, val, cc);
4098 }
4099 return res;
4100}
4101
4102static VALUE
4103vm_call_attrset(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4104{
4105 return vm_call_attrset_direct(ec, cfp, calling->cc, calling->recv);
4106}
4107
4108static inline VALUE
4109vm_call_bmethod_body(rb_execution_context_t *ec, struct rb_calling_info *calling, const VALUE *argv)
4110{
4111 rb_proc_t *proc;
4112 VALUE val;
4113 const struct rb_callcache *cc = calling->cc;
4114 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4115 VALUE procv = cme->def->body.bmethod.proc;
4116
4117 if (!RB_OBJ_SHAREABLE_P(procv) &&
4118 cme->def->body.bmethod.defined_ractor_id != rb_ec_ractor_id(ec)) {
4119 rb_raise(rb_eRuntimeError, "defined with an un-shareable Proc in a different Ractor");
4120 }
4121
4122 /* control block frame */
4123 GetProcPtr(procv, proc);
4124 val = vm_invoke_bmethod(ec, proc, calling->recv, CALLING_ARGC(calling), argv, calling->kw_splat, calling->block_handler, vm_cc_cme(cc));
4125
4126 return val;
4127}
4128
4129static int vm_callee_setup_block_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const struct rb_callinfo *ci, const rb_iseq_t *iseq, VALUE *argv, const enum arg_setup_type arg_setup_type);
4130
4131static VALUE
4132vm_call_iseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4133{
4134 RB_DEBUG_COUNTER_INC(ccf_iseq_bmethod);
4135
4136 const struct rb_callcache *cc = calling->cc;
4137 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4138 VALUE procv = cme->def->body.bmethod.proc;
4139
4140 if (!RB_OBJ_SHAREABLE_P(procv) &&
4141 cme->def->body.bmethod.defined_ractor_id != rb_ec_ractor_id(ec)) {
4142 rb_raise(rb_eRuntimeError, "defined with an un-shareable Proc in a different Ractor");
4143 }
4144
4145 rb_proc_t *proc;
4146 GetProcPtr(procv, proc);
4147 const struct rb_block *block = &proc->block;
4148
4149 while (vm_block_type(block) == block_type_proc) {
4150 block = vm_proc_block(block->as.proc);
4151 }
4152 VM_ASSERT(vm_block_type(block) == block_type_iseq);
4153
4154 const struct rb_captured_block *captured = &block->as.captured;
4155 const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
4156 VALUE * const argv = cfp->sp - calling->argc;
4157 const int arg_size = ISEQ_BODY(iseq)->param.size;
4158
4159 int opt_pc;
4160 if (vm_ci_flag(calling->cd->ci) & VM_CALL_ARGS_SIMPLE) {
4161 opt_pc = vm_callee_setup_block_arg(ec, calling, calling->cd->ci, iseq, argv, arg_setup_method);
4162 }
4163 else {
4164 opt_pc = setup_parameters_complex(ec, iseq, calling, calling->cd->ci, argv, arg_setup_method);
4165 }
4166
4167 cfp->sp = argv - 1; // -1 for the receiver
4168
4169 vm_push_frame(ec, iseq,
4170 VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_BMETHOD | VM_FRAME_FLAG_LAMBDA,
4171 calling->recv,
4172 VM_GUARDED_PREV_EP(captured->ep),
4173 (VALUE)cme,
4174 ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
4175 argv + arg_size,
4176 ISEQ_BODY(iseq)->local_table_size - arg_size,
4177 ISEQ_BODY(iseq)->stack_max);
4178
4179 return Qundef;
4180}
4181
4182static VALUE
4183vm_call_noniseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4184{
4185 RB_DEBUG_COUNTER_INC(ccf_noniseq_bmethod);
4186
4187 VALUE *argv;
4188 int argc;
4189 CALLER_SETUP_ARG(cfp, calling, calling->cd->ci, ALLOW_HEAP_ARGV);
4190 if (UNLIKELY(calling->heap_argv)) {
4191 argv = RARRAY_PTR(calling->heap_argv);
4192 cfp->sp -= 2;
4193 }
4194 else {
4195 argc = calling->argc;
4196 argv = ALLOCA_N(VALUE, argc);
4197 MEMCPY(argv, cfp->sp - argc, VALUE, argc);
4198 cfp->sp += - argc - 1;
4199 }
4200
4201 return vm_call_bmethod_body(ec, calling, argv);
4202}
4203
4204static VALUE
4205vm_call_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4206{
4207 RB_DEBUG_COUNTER_INC(ccf_bmethod);
4208
4209 const struct rb_callcache *cc = calling->cc;
4210 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4211 VALUE procv = cme->def->body.bmethod.proc;
4212 rb_proc_t *proc;
4213 GetProcPtr(procv, proc);
4214 const struct rb_block *block = &proc->block;
4215
4216 while (vm_block_type(block) == block_type_proc) {
4217 block = vm_proc_block(block->as.proc);
4218 }
4219 if (vm_block_type(block) == block_type_iseq) {
4220 CC_SET_FASTPATH(cc, vm_call_iseq_bmethod, TRUE);
4221 return vm_call_iseq_bmethod(ec, cfp, calling);
4222 }
4223
4224 CC_SET_FASTPATH(cc, vm_call_noniseq_bmethod, TRUE);
4225 return vm_call_noniseq_bmethod(ec, cfp, calling);
4226}
4227
4228VALUE
4229rb_find_defined_class_by_owner(VALUE current_class, VALUE target_owner)
4230{
4231 VALUE klass = current_class;
4232
4233 /* for prepended Module, then start from cover class */
4234 if (RB_TYPE_P(klass, T_ICLASS) && RICLASS_IS_ORIGIN_P(klass) &&
4235 RB_TYPE_P(RBASIC_CLASS(klass), T_CLASS)) {
4236 klass = RBASIC_CLASS(klass);
4237 }
4238
4239 while (RTEST(klass)) {
4240 VALUE owner = RB_TYPE_P(klass, T_ICLASS) ? RBASIC_CLASS(klass) : klass;
4241 if (owner == target_owner) {
4242 return klass;
4243 }
4244 klass = RCLASS_SUPER(klass);
4245 }
4246
4247 return current_class; /* maybe module function */
4248}
4249
4250static const rb_callable_method_entry_t *
4251aliased_callable_method_entry(const rb_callable_method_entry_t *me)
4252{
4253 const rb_method_entry_t *orig_me = me->def->body.alias.original_me;
4254 const rb_callable_method_entry_t *cme;
4255
4256 if (orig_me->defined_class == 0) {
4257 VALUE defined_class = rb_find_defined_class_by_owner(me->defined_class, orig_me->owner);
4258 VM_ASSERT_TYPE(orig_me->owner, T_MODULE);
4259 cme = rb_method_entry_complement_defined_class(orig_me, me->called_id, defined_class);
4260
4261 if (me->def->reference_count == 1) {
4262 RB_OBJ_WRITE(me, &me->def->body.alias.original_me, cme);
4263 }
4264 else {
4265 rb_method_definition_t *def =
4266 rb_method_definition_create(VM_METHOD_TYPE_ALIAS, me->def->original_id);
4267 rb_method_definition_set((rb_method_entry_t *)me, def, (void *)cme);
4268 }
4269 }
4270 else {
4271 cme = (const rb_callable_method_entry_t *)orig_me;
4272 }
4273
4274 VM_ASSERT(callable_method_entry_p(cme));
4275 return cme;
4276}
4277
4278const rb_callable_method_entry_t *
4279rb_aliased_callable_method_entry(const rb_callable_method_entry_t *me)
4280{
4281 return aliased_callable_method_entry(me);
4282}
4283
4284static VALUE
4285vm_call_alias(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4286{
4287 calling->cc = &VM_CC_ON_STACK(Qundef,
4288 vm_call_general,
4289 {{0}},
4290 aliased_callable_method_entry(vm_cc_cme(calling->cc)));
4291
4292 return vm_call_method_each_type(ec, cfp, calling);
4293}
4294
4295static enum method_missing_reason
4296ci_missing_reason(const struct rb_callinfo *ci)
4297{
4298 enum method_missing_reason stat = MISSING_NOENTRY;
4299 if (vm_ci_flag(ci) & VM_CALL_VCALL && !(vm_ci_flag(ci) & VM_CALL_FORWARDING)) stat |= MISSING_VCALL;
4300 if (vm_ci_flag(ci) & VM_CALL_FCALL) stat |= MISSING_FCALL;
4301 if (vm_ci_flag(ci) & VM_CALL_SUPER) stat |= MISSING_SUPER;
4302 return stat;
4303}
4304
4305static VALUE vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
4306
4307static VALUE
4308vm_call_symbol(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4309 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE symbol, int flags)
4310{
4311 ASSUME(calling->argc >= 0);
4312
4313 enum method_missing_reason missing_reason = MISSING_NOENTRY;
4314 int argc = calling->argc;
4315 VALUE recv = calling->recv;
4316 VALUE klass = CLASS_OF(recv);
4317 ID mid = rb_check_id(&symbol);
4318 flags |= VM_CALL_OPT_SEND;
4319
4320 if (UNLIKELY(! mid)) {
4321 mid = idMethodMissing;
4322 missing_reason = ci_missing_reason(ci);
4323 ec->method_missing_reason = missing_reason;
4324
4325 VALUE argv_ary;
4326 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4327 if (rb_method_basic_definition_p(klass, idMethodMissing)) {
4328 rb_ary_unshift(argv_ary, symbol);
4329
4330 /* Inadvertent symbol creation shall be forbidden, see [Feature #5112] */
4331 int priv = vm_ci_flag(ci) & (VM_CALL_FCALL | VM_CALL_VCALL);
4332 VALUE exc = rb_make_no_method_exception(
4333 rb_eNoMethodError, 0, recv, RARRAY_LENINT(argv_ary), RARRAY_CONST_PTR(argv_ary), priv);
4334
4335 rb_exc_raise(exc);
4336 }
4337 rb_ary_unshift(argv_ary, rb_str_intern(symbol));
4338 }
4339 else {
4340 /* E.g. when argc == 2
4341 *
4342 * | | | | TOPN
4343 * | | +------+
4344 * | | +---> | arg1 | 0
4345 * +------+ | +------+
4346 * | arg1 | -+ +-> | arg0 | 1
4347 * +------+ | +------+
4348 * | arg0 | ---+ | sym | 2
4349 * +------+ +------+
4350 * | recv | | recv | 3
4351 * --+------+--------+------+------
4352 */
4353 int i = argc;
4354 CHECK_VM_STACK_OVERFLOW(reg_cfp, 1);
4355 INC_SP(1);
4356 MEMMOVE(&TOPN(i - 1), &TOPN(i), VALUE, i);
4357 argc = ++calling->argc;
4358
4359 if (rb_method_basic_definition_p(klass, idMethodMissing)) {
4360 /* Inadvertent symbol creation shall be forbidden, see [Feature #5112] */
4361 TOPN(i) = symbol;
4362 int priv = vm_ci_flag(ci) & (VM_CALL_FCALL | VM_CALL_VCALL);
4363 const VALUE *argv = STACK_ADDR_FROM_TOP(argc);
4364 VALUE exc = rb_make_no_method_exception(
4365 rb_eNoMethodError, 0, recv, argc, argv, priv);
4366
4367 rb_exc_raise(exc);
4368 }
4369 else {
4370 TOPN(i) = rb_str_intern(symbol);
4371 }
4372 }
4373 }
4374
4375 struct rb_forwarding_call_data new_fcd = {
4376 .cd = {
4377 .ci = &VM_CI_ON_STACK(mid, flags, argc, vm_ci_kwarg(ci)),
4378 .cc = NULL,
4379 },
4380 .caller_ci = NULL,
4381 };
4382
4383 if (!(vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
4384 calling->cd = &new_fcd.cd;
4385 }
4386 else {
4387 const struct rb_callinfo *caller_ci = ((struct rb_forwarding_call_data *)calling->cd)->caller_ci;
4388 VM_ASSERT((vm_ci_argc(caller_ci), 1));
4389 new_fcd.caller_ci = caller_ci;
4390 calling->cd = (struct rb_call_data *)&new_fcd;
4391 }
4392 calling->cc = &VM_CC_ON_STACK(klass,
4393 vm_call_general,
4394 { .method_missing_reason = missing_reason },
4395 rb_callable_method_entry_with_refinements(klass, mid, NULL));
4396
4397 if (flags & VM_CALL_FCALL) {
4398 return vm_call_method(ec, reg_cfp, calling);
4399 }
4400
4401 const struct rb_callcache *cc = calling->cc;
4402 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
4403
4404 if (vm_cc_cme(cc) != NULL) {
4405 switch (METHOD_ENTRY_VISI(vm_cc_cme(cc))) {
4406 case METHOD_VISI_PUBLIC: /* likely */
4407 return vm_call_method_each_type(ec, reg_cfp, calling);
4408 case METHOD_VISI_PRIVATE:
4409 vm_cc_method_missing_reason_set(cc, MISSING_PRIVATE);
4410 break;
4411 case METHOD_VISI_PROTECTED:
4412 vm_cc_method_missing_reason_set(cc, MISSING_PROTECTED);
4413 break;
4414 default:
4415 VM_UNREACHABLE(vm_call_method);
4416 }
4417 return vm_call_method_missing(ec, reg_cfp, calling);
4418 }
4419
4420 return vm_call_method_nome(ec, reg_cfp, calling);
4421}
4422
4423static VALUE
4424vm_call_opt_send0(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int flags)
4425{
4426 const struct rb_callinfo *ci = calling->cd->ci;
4427 int i;
4428 VALUE sym;
4429
4430 i = calling->argc - 1;
4431
4432 if (calling->argc == 0) {
4433 rb_raise(rb_eArgError, "no method name given");
4434 }
4435
4436 sym = TOPN(i);
4437 /* E.g. when i == 2
4438 *
4439 * | | | | TOPN
4440 * +------+ | |
4441 * | arg1 | ---+ | | 0
4442 * +------+ | +------+
4443 * | arg0 | -+ +-> | arg1 | 1
4444 * +------+ | +------+
4445 * | sym | +---> | arg0 | 2
4446 * +------+ +------+
4447 * | recv | | recv | 3
4448 * --+------+--------+------+------
4449 */
4450 /* shift arguments */
4451 if (i > 0) {
4452 MEMMOVE(&TOPN(i), &TOPN(i-1), VALUE, i);
4453 }
4454 calling->argc -= 1;
4455 DEC_SP(1);
4456
4457 return vm_call_symbol(ec, reg_cfp, calling, ci, sym, flags);
4458}
4459
4460static VALUE
4461vm_call_opt_send_complex(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4462{
4463 RB_DEBUG_COUNTER_INC(ccf_opt_send_complex);
4464 const struct rb_callinfo *ci = calling->cd->ci;
4465 int flags = VM_CALL_FCALL;
4466 VALUE sym;
4467
4468 VALUE argv_ary;
4469 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV);
4470 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4471 sym = rb_ary_shift(argv_ary);
4472 flags |= VM_CALL_ARGS_SPLAT;
4473 if (calling->kw_splat) {
4474 VALUE last_hash = rb_ary_last(0, NULL, argv_ary);
4475 ((struct RHash *)last_hash)->basic.flags |= RHASH_PASS_AS_KEYWORDS;
4476 calling->kw_splat = 0;
4477 }
4478 return vm_call_symbol(ec, reg_cfp, calling, ci, sym, flags);
4479 }
4480
4481 if (calling->kw_splat) flags |= VM_CALL_KW_SPLAT;
4482 return vm_call_opt_send0(ec, reg_cfp, calling, flags);
4483}
4484
4485static VALUE
4486vm_call_opt_send_simple(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4487{
4488 RB_DEBUG_COUNTER_INC(ccf_opt_send_simple);
4489 return vm_call_opt_send0(ec, reg_cfp, calling, vm_ci_flag(calling->cd->ci) | VM_CALL_FCALL);
4490}
4491
4492static VALUE
4493vm_call_opt_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4494{
4495 RB_DEBUG_COUNTER_INC(ccf_opt_send);
4496
4497 const struct rb_callinfo *ci = calling->cd->ci;
4498 int flags = vm_ci_flag(ci);
4499
4500 if (UNLIKELY((flags & VM_CALL_FORWARDING) || (!(flags & VM_CALL_ARGS_SIMPLE) &&
4501 ((calling->argc == 1 && (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT))) ||
4502 (calling->argc == 2 && (flags & VM_CALL_ARGS_SPLAT) && (flags & VM_CALL_KW_SPLAT)) ||
4503 ((flags & VM_CALL_KWARG) && (vm_ci_kwarg(ci)->keyword_len == calling->argc)))))) {
4504 CC_SET_FASTPATH(calling->cc, vm_call_opt_send_complex, TRUE);
4505 return vm_call_opt_send_complex(ec, reg_cfp, calling);
4506 }
4507
4508 CC_SET_FASTPATH(calling->cc, vm_call_opt_send_simple, TRUE);
4509 return vm_call_opt_send_simple(ec, reg_cfp, calling);
4510}
4511
4512static VALUE
4513vm_call_method_missing_body(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling,
4514 const struct rb_callinfo *orig_ci, enum method_missing_reason reason)
4515{
4516 RB_DEBUG_COUNTER_INC(ccf_method_missing);
4517
4518 VALUE *argv = STACK_ADDR_FROM_TOP(calling->argc);
4519 unsigned int argc, flag;
4520
4521 flag = VM_CALL_FCALL | VM_CALL_OPT_SEND | vm_ci_flag(orig_ci);
4522 argc = ++calling->argc;
4523
4524 /* shift arguments: m(a, b, c) #=> method_missing(:m, a, b, c) */
4525 CHECK_VM_STACK_OVERFLOW(reg_cfp, 1);
4526 vm_check_canary(ec, reg_cfp->sp);
4527 if (argc > 1) {
4528 MEMMOVE(argv+1, argv, VALUE, argc-1);
4529 }
4530 argv[0] = ID2SYM(vm_ci_mid(orig_ci));
4531 INC_SP(1);
4532
4533 ec->method_missing_reason = reason;
4534
4535 struct rb_forwarding_call_data new_fcd = {
4536 .cd = {
4537 .ci = &VM_CI_ON_STACK(idMethodMissing, flag, argc, vm_ci_kwarg(orig_ci)),
4538 .cc = NULL,
4539 },
4540 .caller_ci = NULL,
4541 };
4542
4543 if (!(flag & VM_CALL_FORWARDING)) {
4544 calling->cd = &new_fcd.cd;
4545 }
4546 else {
4547 const struct rb_callinfo *caller_ci = ((struct rb_forwarding_call_data *)calling->cd)->caller_ci;
4548 VM_ASSERT((vm_ci_argc(caller_ci), 1));
4549 new_fcd.caller_ci = caller_ci;
4550 calling->cd = (struct rb_call_data *)&new_fcd;
4551 }
4552
4553 calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }},
4554 rb_callable_method_entry_without_refinements(CLASS_OF(calling->recv), idMethodMissing, NULL));
4555 return vm_call_method(ec, reg_cfp, calling);
4556}
4557
4558static VALUE
4559vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4560{
4561 return vm_call_method_missing_body(ec, reg_cfp, calling, calling->cd->ci, vm_cc_cmethod_missing_reason(calling->cc));
4562}
4563
4564static const rb_callable_method_entry_t *refined_method_callable_without_refinement(const rb_callable_method_entry_t *me);
4565static VALUE
4566vm_call_zsuper(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, VALUE klass)
4567{
4568 klass = RCLASS_SUPER(klass);
4569
4570 const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, vm_ci_mid(calling->cd->ci)) : NULL;
4571 if (cme == NULL) {
4572 return vm_call_method_nome(ec, cfp, calling);
4573 }
4574 if (cme->def->type == VM_METHOD_TYPE_REFINED &&
4575 cme->def->body.refined.orig_me) {
4576 cme = refined_method_callable_without_refinement(cme);
4577 }
4578
4579 calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, cme);
4580
4581 return vm_call_method_each_type(ec, cfp, calling);
4582}
4583
4584static inline VALUE
4585find_refinement(VALUE refinements, VALUE klass)
4586{
4587 if (NIL_P(refinements)) {
4588 return Qnil;
4589 }
4590 return rb_hash_lookup(refinements, klass);
4591}
4592
4593PUREFUNC(static rb_control_frame_t * current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp));
4594static rb_control_frame_t *
4595current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp)
4596{
4597 rb_control_frame_t *top_cfp = cfp;
4598
4599 if (CFP_ISEQ(cfp) && ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_BLOCK) {
4600 const rb_iseq_t *local_iseq = ISEQ_BODY(CFP_ISEQ(cfp))->local_iseq;
4601
4602 do {
4603 cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
4604 if (RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
4605 /* TODO: orphan block */
4606 return top_cfp;
4607 }
4608 } while (CFP_ISEQ(cfp) != local_iseq);
4609 }
4610 return cfp;
4611}
4612
4613static const rb_callable_method_entry_t *
4614refined_method_callable_without_refinement(const rb_callable_method_entry_t *me)
4615{
4616 const rb_method_entry_t *orig_me = me->def->body.refined.orig_me;
4617 const rb_callable_method_entry_t *cme;
4618
4619 if (orig_me->defined_class == 0) {
4620 cme = NULL;
4621 rb_notimplement();
4622 }
4623 else {
4624 cme = (const rb_callable_method_entry_t *)orig_me;
4625 }
4626
4627 VM_ASSERT(callable_method_entry_p(cme));
4628
4629 if (UNDEFINED_METHOD_ENTRY_P(cme)) {
4630 cme = NULL;
4631 }
4632
4633 return cme;
4634}
4635
4636static const rb_callable_method_entry_t *
4637search_refined_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4638{
4639 ID mid = vm_ci_mid(calling->cd->ci);
4640 const rb_cref_t *cref = vm_get_cref(cfp->ep);
4641 const struct rb_callcache * const cc = calling->cc;
4642 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4643
4644 for (; cref; cref = CREF_NEXT(cref)) {
4645 const VALUE refinement = find_refinement(CREF_REFINEMENTS(cref), vm_cc_cme(cc)->owner);
4646 if (NIL_P(refinement)) continue;
4647
4648 const rb_callable_method_entry_t *const ref_me =
4649 rb_callable_method_entry(refinement, mid);
4650
4651 if (ref_me) {
4652 if (vm_cc_call(cc) == vm_call_super_method) {
4653 const rb_control_frame_t *top_cfp = current_method_entry(ec, cfp);
4654 const rb_callable_method_entry_t *top_me = rb_vm_frame_method_entry(top_cfp);
4655 if (top_me && rb_method_definition_eq(ref_me->def, top_me->def)) {
4656 continue;
4657 }
4658 }
4659
4660 if (cme->def->type != VM_METHOD_TYPE_REFINED ||
4661 cme->def != ref_me->def) {
4662 cme = ref_me;
4663 }
4664 if (ref_me->def->type != VM_METHOD_TYPE_REFINED) {
4665 return cme;
4666 }
4667 }
4668 else {
4669 return NULL;
4670 }
4671 }
4672
4673 if (vm_cc_cme(cc)->def->body.refined.orig_me) {
4674 return refined_method_callable_without_refinement(vm_cc_cme(cc));
4675 }
4676 else {
4677 VALUE klass = RCLASS_SUPER(vm_cc_cme(cc)->defined_class);
4678 const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, mid) : NULL;
4679 return cme;
4680 }
4681}
4682
4683static VALUE
4684vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4685{
4686 const rb_callable_method_entry_t *ref_cme = search_refined_method(ec, cfp, calling);
4687
4688 if (ref_cme) {
4689 if (calling->cd->cc) {
4690 const struct rb_callcache *cc = calling->cc = vm_cc_new(vm_cc_cme(calling->cc)->defined_class, ref_cme, vm_call_general, cc_type_refinement);
4691 RB_OBJ_WRITE(CFP_ISEQ(cfp), &calling->cd->cc, cc);
4692 return vm_call_method(ec, cfp, calling);
4693 }
4694 else {
4695 struct rb_callcache *ref_cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, ref_cme);
4696 calling->cc= ref_cc;
4697 return vm_call_method(ec, cfp, calling);
4698 }
4699 }
4700 else {
4701 return vm_call_method_nome(ec, cfp, calling);
4702 }
4703}
4704
4705static inline VALUE vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, const struct rb_callinfo *ci, bool is_lambda, VALUE block_handler);
4706
4707NOINLINE(static VALUE
4708 vm_invoke_block_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4709 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE block_handler));
4710
4711static VALUE
4712vm_invoke_block_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4713 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE block_handler)
4714{
4715 int argc = calling->argc;
4716
4717 /* remove self */
4718 if (argc > 0) MEMMOVE(&TOPN(argc), &TOPN(argc-1), VALUE, argc);
4719 DEC_SP(1);
4720
4721 return vm_invoke_block(ec, reg_cfp, calling, ci, false, block_handler);
4722}
4723
4724static VALUE
4725vm_call_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4726{
4727 RB_DEBUG_COUNTER_INC(ccf_opt_call);
4728
4729 const struct rb_callinfo *ci = calling->cd->ci;
4730 VALUE procval = calling->recv;
4731 return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, VM_BH_FROM_PROC(procval));
4732}
4733
4734static VALUE
4735vm_call_opt_block_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4736{
4737 RB_DEBUG_COUNTER_INC(ccf_opt_block_call);
4738
4739 VALUE block_handler = VM_ENV_BLOCK_HANDLER(VM_CF_LEP(reg_cfp));
4740 const struct rb_callinfo *ci = calling->cd->ci;
4741
4742 enum ruby_basic_operators bop;
4743 switch (vm_ci_mid(ci)) {
4744 case idAREF: bop = BOP_AREF; break;
4745 case idYield: bop = BOP_YIELD; break;
4746 case idEqq: bop = BOP_EQQ; break;
4747 default: bop = BOP_CALL; break;
4748 }
4749
4750 if (BASIC_OP_UNREDEFINED_P(bop, PROC_REDEFINED_OP_FLAG)) {
4751 return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, block_handler);
4752 }
4753 else {
4754 calling->recv = rb_vm_bh_to_procval(ec, block_handler);
4755 calling->cc = rb_vm_search_method_slowpath(ci, CLASS_OF(calling->recv));
4756 return vm_call_general(ec, reg_cfp, calling);
4757 }
4758}
4759
4760static VALUE
4761vm_call_opt_struct_aref0(rb_execution_context_t *ec, struct rb_calling_info *calling)
4762{
4763 VALUE recv = calling->recv;
4764
4765 VM_ASSERT(RB_TYPE_P(recv, T_STRUCT));
4766 VM_ASSERT(vm_cc_cme(calling->cc)->def->type == VM_METHOD_TYPE_OPTIMIZED);
4767 VM_ASSERT(vm_cc_cme(calling->cc)->def->body.optimized.type == OPTIMIZED_METHOD_TYPE_STRUCT_AREF);
4768
4769 const unsigned int off = vm_cc_cme(calling->cc)->def->body.optimized.index;
4770 return RSTRUCT_GET_RAW(recv, off);
4771}
4772
4773static VALUE
4774vm_call_opt_struct_aref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4775{
4776 RB_DEBUG_COUNTER_INC(ccf_opt_struct_aref);
4777
4778 VALUE ret = vm_call_opt_struct_aref0(ec, calling);
4779 reg_cfp->sp -= 1;
4780 return ret;
4781}
4782
4783static VALUE
4784vm_call_opt_struct_aset0(rb_execution_context_t *ec, struct rb_calling_info *calling, VALUE val)
4785{
4786 VALUE recv = calling->recv;
4787
4788 VM_ASSERT(RB_TYPE_P(recv, T_STRUCT));
4789 VM_ASSERT(vm_cc_cme(calling->cc)->def->type == VM_METHOD_TYPE_OPTIMIZED);
4790 VM_ASSERT(vm_cc_cme(calling->cc)->def->body.optimized.type == OPTIMIZED_METHOD_TYPE_STRUCT_ASET);
4791
4792 rb_check_frozen(recv);
4793
4794 const unsigned int off = vm_cc_cme(calling->cc)->def->body.optimized.index;
4795 RSTRUCT_SET_RAW(recv, off, val);
4796
4797 return val;
4798}
4799
4800static VALUE
4801vm_call_opt_struct_aset(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4802{
4803 RB_DEBUG_COUNTER_INC(ccf_opt_struct_aset);
4804
4805 VALUE ret = vm_call_opt_struct_aset0(ec, calling, *(reg_cfp->sp - 1));
4806 reg_cfp->sp -= 2;
4807 return ret;
4808}
4809
4810NOINLINE(static VALUE vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
4811 const struct rb_callinfo *ci, const struct rb_callcache *cc));
4812
4813#define VM_CALL_METHOD_ATTR(var, func, nohook) \
4814 if (UNLIKELY(ruby_vm_c_events_enabled > 0)) { \
4815 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_CALL, calling->recv, vm_cc_cme(cc)->def->original_id, \
4816 vm_ci_mid(ci), vm_cc_cme(cc)->owner, Qundef); \
4817 var = func; \
4818 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, calling->recv, vm_cc_cme(cc)->def->original_id, \
4819 vm_ci_mid(ci), vm_cc_cme(cc)->owner, (var)); \
4820 } \
4821 else { \
4822 nohook; \
4823 var = func; \
4824 }
4825
4826static VALUE
4827vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
4828 const struct rb_callinfo *ci, const struct rb_callcache *cc)
4829{
4830 switch (vm_cc_cme(cc)->def->body.optimized.type) {
4831 case OPTIMIZED_METHOD_TYPE_SEND:
4832 CC_SET_FASTPATH(cc, vm_call_opt_send, TRUE);
4833 return vm_call_opt_send(ec, cfp, calling);
4834 case OPTIMIZED_METHOD_TYPE_CALL:
4835 CC_SET_FASTPATH(cc, vm_call_opt_call, TRUE);
4836 return vm_call_opt_call(ec, cfp, calling);
4837 case OPTIMIZED_METHOD_TYPE_BLOCK_CALL:
4838 CC_SET_FASTPATH(cc, vm_call_opt_block_call, TRUE);
4839 return vm_call_opt_block_call(ec, cfp, calling);
4840 case OPTIMIZED_METHOD_TYPE_STRUCT_AREF: {
4841 CALLER_SETUP_ARG(cfp, calling, ci, 0);
4842 rb_check_arity(calling->argc, 0, 0);
4843
4844 VALUE v;
4845 VM_CALL_METHOD_ATTR(v,
4846 vm_call_opt_struct_aref(ec, cfp, calling),
4847 set_vm_cc_ivar(cc); \
4848 CC_SET_FASTPATH(cc, vm_call_opt_struct_aref, (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE)))
4849 return v;
4850 }
4851 case OPTIMIZED_METHOD_TYPE_STRUCT_ASET: {
4852 CALLER_SETUP_ARG(cfp, calling, ci, 1);
4853 rb_check_arity(calling->argc, 1, 1);
4854
4855 VALUE v;
4856 VM_CALL_METHOD_ATTR(v,
4857 vm_call_opt_struct_aset(ec, cfp, calling),
4858 set_vm_cc_ivar(cc); \
4859 CC_SET_FASTPATH(cc, vm_call_opt_struct_aset, (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE)))
4860 return v;
4861 }
4862 default:
4863 rb_bug("vm_call_method: unsupported optimized method type (%d)", vm_cc_cme(cc)->def->body.optimized.type);
4864 }
4865}
4866
4867static VALUE
4868vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4869{
4870 const struct rb_callinfo *ci = calling->cd->ci;
4871 const struct rb_callcache *cc = calling->cc;
4872 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4873 VALUE v;
4874
4875 VM_ASSERT(! METHOD_ENTRY_INVALIDATED(cme));
4876
4877 switch (cme->def->type) {
4878 case VM_METHOD_TYPE_ISEQ:
4879 if (ISEQ_BODY(def_iseq_ptr(cme->def))->param.flags.forwardable) {
4880 CC_SET_FASTPATH(cc, vm_call_iseq_fwd_setup, TRUE);
4881 return vm_call_iseq_fwd_setup(ec, cfp, calling);
4882 }
4883 else {
4884 CC_SET_FASTPATH(cc, vm_call_iseq_setup, TRUE);
4885 return vm_call_iseq_setup(ec, cfp, calling);
4886 }
4887
4888 case VM_METHOD_TYPE_NOTIMPLEMENTED:
4889 case VM_METHOD_TYPE_CFUNC:
4890 CC_SET_FASTPATH(cc, vm_call_cfunc, TRUE);
4891 return vm_call_cfunc(ec, cfp, calling);
4892
4893 case VM_METHOD_TYPE_ATTRSET:
4894 CALLER_SETUP_ARG(cfp, calling, ci, 1);
4895
4896 rb_check_arity(calling->argc, 1, 1);
4897
4898 const unsigned int aset_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_KWARG | VM_CALL_FORWARDING);
4899
4900 if (vm_cc_markable(cc)) {
4901 vm_cc_attr_index_set(cc, IVAR_CACHE_INIT);
4902 VM_CALL_METHOD_ATTR(v,
4903 vm_call_attrset_direct(ec, cfp, cc, calling->recv),
4904 CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
4905 }
4906 else {
4907 cc = &((struct rb_callcache) {
4908 .flags = T_IMEMO |
4909 (imemo_callcache << FL_USHIFT) |
4910 VM_CALLCACHE_UNMARKABLE |
4911 VM_CALLCACHE_ON_STACK,
4912 .klass = cc->klass,
4913 .cme_ = cc->cme_,
4914 .call_ = cc->call_,
4915 .aux_ = {
4916 .attr = {
4917 .value = IVAR_CACHE_INIT,
4918 }
4919 },
4920 });
4921
4922 VM_CALL_METHOD_ATTR(v,
4923 vm_call_attrset_direct(ec, cfp, cc, calling->recv),
4924 CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
4925 }
4926 return v;
4927
4928 case VM_METHOD_TYPE_IVAR:
4929 CALLER_SETUP_ARG(cfp, calling, ci, 0);
4930 rb_check_arity(calling->argc, 0, 0);
4931 vm_cc_attr_index_set(cc, rb_getivar_cache_pack(ROOT_SHAPE_ID, ATTR_INDEX_NOT_SET));
4932 const unsigned int ivar_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_FORWARDING);
4933 VM_CALL_METHOD_ATTR(v,
4934 vm_call_ivar(ec, cfp, calling),
4935 CC_SET_FASTPATH(cc, vm_call_ivar, !(vm_ci_flag(ci) & ivar_mask)));
4936 return v;
4937
4938 case VM_METHOD_TYPE_MISSING:
4939 vm_cc_method_missing_reason_set(cc, 0);
4940 CC_SET_FASTPATH(cc, vm_call_method_missing, TRUE);
4941 return vm_call_method_missing(ec, cfp, calling);
4942
4943 case VM_METHOD_TYPE_BMETHOD:
4944 CC_SET_FASTPATH(cc, vm_call_bmethod, TRUE);
4945 return vm_call_bmethod(ec, cfp, calling);
4946
4947 case VM_METHOD_TYPE_ALIAS:
4948 CC_SET_FASTPATH(cc, vm_call_alias, TRUE);
4949 return vm_call_alias(ec, cfp, calling);
4950
4951 case VM_METHOD_TYPE_OPTIMIZED:
4952 return vm_call_optimized(ec, cfp, calling, ci, cc);
4953
4954 case VM_METHOD_TYPE_UNDEF:
4955 break;
4956
4957 case VM_METHOD_TYPE_ZSUPER:
4958 return vm_call_zsuper(ec, cfp, calling, RCLASS_ORIGIN(vm_cc_cme(cc)->defined_class));
4959
4960 case VM_METHOD_TYPE_REFINED:
4961 // CC_SET_FASTPATH(cc, vm_call_refined, TRUE);
4962 // should not set FASTPATH since vm_call_refined assumes cc->call is vm_call_super_method on invokesuper.
4963 return vm_call_refined(ec, cfp, calling);
4964 }
4965
4966 rb_bug("vm_call_method: unsupported method type (%d)", vm_cc_cme(cc)->def->type);
4967}
4968
4969NORETURN(static void vm_raise_method_missing(rb_execution_context_t *ec, int argc, const VALUE *argv, VALUE obj, int call_status));
4970
4971static VALUE
4972vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4973{
4974 /* method missing */
4975 const struct rb_callinfo *ci = calling->cd->ci;
4976 const int stat = ci_missing_reason(ci);
4977
4978 if (vm_ci_mid(ci) == idMethodMissing) {
4979 if (UNLIKELY(calling->heap_argv)) {
4980 vm_raise_method_missing(ec, RARRAY_LENINT(calling->heap_argv), RARRAY_CONST_PTR(calling->heap_argv), calling->recv, stat);
4981 }
4982 else {
4983 rb_control_frame_t *reg_cfp = cfp;
4984 VALUE *argv = STACK_ADDR_FROM_TOP(calling->argc);
4985 vm_raise_method_missing(ec, calling->argc, argv, calling->recv, stat);
4986 }
4987 }
4988 else {
4989 return vm_call_method_missing_body(ec, cfp, calling, ci, stat);
4990 }
4991}
4992
4993/* Protected method calls and super invocations need to check that the receiver
4994 * (self for super) inherits the module on which the method is defined.
4995 * In the case of refinements, it should consider the original class not the
4996 * refinement.
4997 */
4998static VALUE
4999vm_defined_class_for_protected_call(const rb_callable_method_entry_t *me)
5000{
5001 VALUE defined_class = me->defined_class;
5002 VALUE refined_class = RCLASS_REFINED_CLASS(defined_class);
5003 return NIL_P(refined_class) ? defined_class : refined_class;
5004}
5005
5006static inline VALUE
5007vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
5008{
5009 const struct rb_callinfo *ci = calling->cd->ci;
5010 const struct rb_callcache *cc = calling->cc;
5011
5012 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
5013
5014 if (vm_cc_cme(cc) != NULL) {
5015 switch (METHOD_ENTRY_VISI(vm_cc_cme(cc))) {
5016 case METHOD_VISI_PUBLIC: /* likely */
5017 return vm_call_method_each_type(ec, cfp, calling);
5018
5019 case METHOD_VISI_PRIVATE:
5020 if (!(vm_ci_flag(ci) & VM_CALL_FCALL)) {
5021 enum method_missing_reason stat = MISSING_PRIVATE;
5022 if (vm_ci_flag(ci) & VM_CALL_VCALL) stat |= MISSING_VCALL;
5023
5024 vm_cc_method_missing_reason_set(cc, stat);
5025 CC_SET_FASTPATH(cc, vm_call_method_missing, TRUE);
5026 return vm_call_method_missing(ec, cfp, calling);
5027 }
5028 return vm_call_method_each_type(ec, cfp, calling);
5029
5030 case METHOD_VISI_PROTECTED:
5031 if (!(vm_ci_flag(ci) & (VM_CALL_OPT_SEND | VM_CALL_FCALL))) {
5032 VALUE defined_class = vm_defined_class_for_protected_call(vm_cc_cme(cc));
5033 if (!rb_obj_is_kind_of(cfp->self, defined_class)) {
5034 vm_cc_method_missing_reason_set(cc, MISSING_PROTECTED);
5035 return vm_call_method_missing(ec, cfp, calling);
5036 }
5037 else {
5038 /* caching method info to dummy cc */
5039 VM_ASSERT(vm_cc_cme(cc) != NULL);
5040 struct rb_callcache cc_on_stack = *cc;
5041 FL_SET_RAW((VALUE)&cc_on_stack, VM_CALLCACHE_UNMARKABLE);
5042 calling->cc = &cc_on_stack;
5043 return vm_call_method_each_type(ec, cfp, calling);
5044 }
5045 }
5046 return vm_call_method_each_type(ec, cfp, calling);
5047
5048 default:
5049 rb_bug("unreachable");
5050 }
5051 }
5052 else {
5053 return vm_call_method_nome(ec, cfp, calling);
5054 }
5055}
5056
5057static VALUE
5058vm_call_general(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
5059{
5060 RB_DEBUG_COUNTER_INC(ccf_general);
5061 return vm_call_method(ec, reg_cfp, calling);
5062}
5063
5064void
5065rb_vm_cc_general(const struct rb_callcache *cc)
5066{
5067 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
5068 VM_ASSERT(cc != vm_cc_empty());
5069
5070 *(vm_call_handler *)&cc->call_ = vm_call_general;
5071}
5072
5073static VALUE
5074vm_call_super_method(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
5075{
5076 RB_DEBUG_COUNTER_INC(ccf_super_method);
5077
5078 // This line is introduced to make different from `vm_call_general` because some compilers (VC we found)
5079 // can merge the function and the address of the function becomes same.
5080 // The address of `vm_call_super_method` is used in `search_refined_method`, so it should be different.
5081 if (ec == NULL) rb_bug("unreachable");
5082
5083 /* this check is required to distinguish with other functions. */
5084 VM_ASSERT(vm_cc_call(calling->cc) == vm_call_super_method);
5085 return vm_call_method(ec, reg_cfp, calling);
5086}
5087
5088/* super */
5089
5090static inline VALUE
5091vm_search_normal_superclass(VALUE klass)
5092{
5093 if (RICLASS_FOR_REFINEMENT_P(klass)) {
5094 klass = RBASIC(klass)->klass;
5095 }
5096 klass = RCLASS_ORIGIN(klass);
5097 return RCLASS_SUPER(klass);
5098}
5099
5100NORETURN(static void vm_super_outside(void));
5101
5102static void
5103vm_super_outside(void)
5104{
5105 rb_raise(rb_eNoMethodError, "super called outside of method");
5106}
5107
5108static const struct rb_callcache *
5109empty_cc_for_super(void)
5110{
5111 return &vm_empty_cc_for_super;
5112}
5113
5114static const struct rb_callcache *
5115vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *cd, VALUE recv)
5116{
5117 VALUE current_defined_class;
5118 const rb_iseq_t *iseq = CFP_ISEQ(reg_cfp);
5119 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp);
5120
5121 if (!me) {
5122 vm_super_outside();
5123 }
5124
5125 current_defined_class = vm_defined_class_for_protected_call(me);
5126
5127 if (BUILTIN_TYPE(current_defined_class) != T_MODULE &&
5128 iseq != method_entry_iseqptr(me) &&
5129 !rb_obj_is_kind_of(recv, current_defined_class)) {
5130 VALUE m = RB_TYPE_P(current_defined_class, T_ICLASS) ?
5131 RCLASS_INCLUDER(current_defined_class) : current_defined_class;
5132
5133 if (m) { /* not bound UnboundMethod */
5134 rb_raise(rb_eTypeError,
5135 "self has wrong type to call super in this context: "
5136 "%"PRIsVALUE" (expected %"PRIsVALUE")",
5137 rb_obj_class(recv), m);
5138 }
5139 }
5140
5141 if (me->def->type == VM_METHOD_TYPE_BMETHOD && (vm_ci_flag(cd->ci) & VM_CALL_ZSUPER)) {
5142 rb_raise(rb_eRuntimeError,
5143 "implicit argument passing of super from method defined"
5144 " by define_method() is not supported."
5145 " Specify all arguments explicitly.");
5146 }
5147
5148 ID mid = me->def->original_id;
5149
5150 if (!vm_ci_markable(cd->ci)) {
5151 VM_FORCE_WRITE((const VALUE *)&cd->ci->mid, (VALUE)mid);
5152 }
5153 else {
5154 // update iseq. really? (TODO)
5155 cd->ci = vm_ci_new_runtime(mid,
5156 vm_ci_flag(cd->ci),
5157 vm_ci_argc(cd->ci),
5158 vm_ci_kwarg(cd->ci));
5159
5160 RB_OBJ_WRITTEN(iseq, Qundef, cd->ci);
5161 }
5162
5163 const struct rb_callcache *cc;
5164
5165 VALUE klass = vm_search_normal_superclass(me->defined_class);
5166
5167 if (!klass) {
5168 /* bound instance method of module */
5169 cc = vm_cc_new(Qundef, NULL, vm_call_method_missing, cc_type_super);
5170 RB_OBJ_WRITE(iseq, &cd->cc, cc);
5171 }
5172 else if (klass == rb_cBasicObject &&
5173 RB_TYPE_P(me->defined_class, T_ICLASS) &&
5174 RCLASS_INCLUDER(me->defined_class) == 0) {
5175 rb_raise(rb_eNoMethodError,
5176 "super in a method in a module that has been refined and that is called via super"
5177 " from a refinement method is not supported.");
5178 }
5179 else {
5180 cc = vm_search_method_fastpath(reg_cfp, cd, klass);
5181 const rb_callable_method_entry_t *cached_cme = vm_cc_cme(cc);
5182
5183 // define_method can cache for different method id
5184 if (cached_cme == NULL) {
5185 // empty_cc_for_super is not markable object
5186 cd->cc = empty_cc_for_super();
5187 }
5188 else if (cached_cme->called_id != mid) {
5189 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
5190 if (cme) {
5191 cc = vm_cc_new(klass, cme, vm_call_super_method, cc_type_super);
5192 RB_OBJ_WRITE(iseq, &cd->cc, cc);
5193 }
5194 else {
5195 cd->cc = cc = empty_cc_for_super();
5196 }
5197 }
5198 else {
5199 switch (cached_cme->def->type) {
5200 // vm_call_refined (search_refined_method) assumes cc->call is vm_call_super_method on invokesuper
5201 case VM_METHOD_TYPE_REFINED:
5202 // cc->klass is superclass of receiver class. Checking cc->klass is not enough to invalidate IVC for the receiver class.
5203 case VM_METHOD_TYPE_ATTRSET:
5204 case VM_METHOD_TYPE_IVAR:
5205 vm_cc_call_set(cc, vm_call_super_method); // invalidate fastpath
5206 break;
5207 default:
5208 break; // use fastpath
5209 }
5210 }
5211 }
5212
5213 VM_ASSERT((vm_cc_cme(cc), true));
5214
5215 return cc;
5216}
5217
5218/* yield */
5219
5220static inline int
5221block_proc_is_lambda(const VALUE procval)
5222{
5223 rb_proc_t *proc;
5224
5225 if (procval) {
5226 GetProcPtr(procval, proc);
5227 return proc->header.is_lambda;
5228 }
5229 else {
5230 return 0;
5231 }
5232}
5233
5234static VALUE
5235vm_yield_with_cfunc(rb_execution_context_t *ec,
5236 const struct rb_captured_block *captured,
5237 VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE block_handler,
5238 const rb_callable_method_entry_t *me)
5239{
5240 int is_lambda = FALSE; /* TODO */
5241 VALUE val, arg, blockarg;
5242 int frame_flag;
5243 const struct vm_ifunc *ifunc = captured->code.ifunc;
5244
5245 if (is_lambda) {
5246 arg = rb_ary_new4(argc, argv);
5247 }
5248 else if (argc == 0) {
5249 arg = Qnil;
5250 }
5251 else {
5252 arg = argv[0];
5253 }
5254
5255 blockarg = rb_vm_bh_to_procval(ec, block_handler);
5256
5257 frame_flag = VM_FRAME_MAGIC_IFUNC | VM_FRAME_FLAG_CFRAME | (me ? VM_FRAME_FLAG_BMETHOD : 0);
5258 if (kw_splat) {
5259 frame_flag |= VM_FRAME_FLAG_CFRAME_KW;
5260 }
5261
5262 vm_push_frame(ec, (const rb_iseq_t *)captured->code.ifunc,
5263 frame_flag,
5264 self,
5265 VM_GUARDED_PREV_EP(captured->ep),
5266 (VALUE)me,
5267 0, ec->cfp->sp, 0, 0);
5268 val = (*ifunc->func)(arg, (VALUE)ifunc->data, argc, argv, blockarg);
5269 rb_vm_pop_frame(ec);
5270
5271 return val;
5272}
5273
5274VALUE
5275rb_vm_yield_with_cfunc(rb_execution_context_t *ec, const struct rb_captured_block *captured, int argc, const VALUE *argv)
5276{
5277 return vm_yield_with_cfunc(ec, captured, captured->self, argc, argv, 0, VM_BLOCK_HANDLER_NONE, NULL);
5278}
5279
5280static VALUE
5281vm_yield_with_symbol(rb_execution_context_t *ec, VALUE symbol, int argc, const VALUE *argv, int kw_splat, VALUE block_handler)
5282{
5283 VALUE passed_proc = rb_vm_bh_to_procval(ec, block_handler);
5284
5285 if (!rb_box_available()) {
5286 return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc);
5287 }
5288
5289 const rb_control_frame_t *reg_cfp = ec->cfp;
5290 const rb_control_frame_t *ruby_cfp = rb_vm_get_ruby_level_next_cfp(ec, reg_cfp);
5291 const rb_box_t *box = NULL;
5292
5293 /*
5294 * Traverse the frames until a user box (Main or Optional Box) is found.
5295 * Frames for built-in methods like <internal:xxx> run in the Root or Master Box,
5296 */
5297 while (ruby_cfp) {
5298 box = current_box_on_cfp(ec, ruby_cfp);
5299 if (BOX_USER_P(box)) {
5300 break;
5301 }
5302 ruby_cfp = rb_vm_get_ruby_level_next_cfp(ec, RUBY_VM_PREVIOUS_CONTROL_FRAME(ruby_cfp));
5303 }
5304
5305 // Fallback to the normal call if no user box is found
5306 if (!ruby_cfp || !box) {
5307 return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc);
5308 }
5309
5310 /*
5311 * Push a dummy block frame whose outer env is `ruby_cfp` so that the
5312 * method resolves in the caller's box (via the ep chain) and the
5313 * backtrace reads like a usual block invocation at the caller's site.
5314 */
5315 const rb_iseq_t *caller_iseq = CFP_ISEQ(ruby_cfp);
5316 VALUE name = rb_sprintf("block in %"PRIsVALUE, rb_iseq_label(caller_iseq));
5317 const rb_iseq_t *iseq = rb_iseq_new_with_opt(Qnil, name,
5318 rb_iseq_path(caller_iseq), rb_iseq_realpath(caller_iseq),
5319 rb_vm_get_sourceline(ruby_cfp), caller_iseq, 0,
5320 ISEQ_TYPE_BLOCK, NULL, Qnil);
5321 volatile VALUE val = Qnil;
5322 enum ruby_tag_type state;
5323
5324 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_FINISH,
5325 ruby_cfp->self, VM_GUARDED_PREV_EP(ruby_cfp->ep),
5326 Qfalse, /* cref or me */
5327 ISEQ_BODY(iseq)->iseq_encoded, reg_cfp->sp,
5328 ISEQ_BODY(iseq)->local_table_size, ISEQ_BODY(iseq)->stack_max);
5329
5330 /*
5331 * The pushed frame is finished (owned by no vm_exec loop), so catch the
5332 * non-local exit here, pop the frame, and let the caller's handlers see
5333 * the exception.
5334 */
5335 EC_PUSH_TAG(ec);
5336 if ((state = EC_EXEC_TAG()) == TAG_NONE) {
5337 val = rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, passed_proc);
5338 }
5339 EC_POP_TAG();
5340
5341 if (state != TAG_NONE) {
5342 rb_vm_rewind_cfp(ec, (rb_control_frame_t *)reg_cfp);
5343 EC_JUMP_TAG(ec, state);
5344 }
5345
5346 rb_vm_pop_frame(ec);
5347
5348 return val;
5349}
5350
5351static inline int
5352vm_callee_setup_block_arg_arg0_splat(rb_control_frame_t *cfp, const rb_iseq_t *iseq, VALUE *argv, VALUE ary)
5353{
5354 int i;
5355 long len = RARRAY_LEN(ary);
5356
5357 CHECK_VM_STACK_OVERFLOW(cfp, ISEQ_BODY(iseq)->param.lead_num);
5358
5359 for (i=0; i<len && i<ISEQ_BODY(iseq)->param.lead_num; i++) {
5360 argv[i] = RARRAY_AREF(ary, i);
5361 }
5362
5363 return i;
5364}
5365
5366static inline VALUE
5367vm_callee_setup_block_arg_arg0_check(VALUE *argv)
5368{
5369 VALUE ary, arg0 = argv[0];
5370 ary = rb_check_array_type(arg0);
5371#if 0
5372 argv[0] = arg0;
5373#else
5374 VM_ASSERT(argv[0] == arg0);
5375#endif
5376 return ary;
5377}
5378
5379static int
5380vm_callee_setup_block_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const struct rb_callinfo *ci, const rb_iseq_t *iseq, VALUE *argv, const enum arg_setup_type arg_setup_type)
5381{
5382 if (rb_simple_iseq_p(iseq)) {
5383 rb_control_frame_t *cfp = ec->cfp;
5384 VALUE arg0;
5385
5386 CALLER_SETUP_ARG(cfp, calling, ci, ISEQ_BODY(iseq)->param.lead_num);
5387
5388 if (arg_setup_type == arg_setup_block &&
5389 calling->argc == 1 &&
5390 ISEQ_BODY(iseq)->param.flags.has_lead &&
5391 !ISEQ_BODY(iseq)->param.flags.ambiguous_param0 &&
5392 !NIL_P(arg0 = vm_callee_setup_block_arg_arg0_check(argv))) {
5393 calling->argc = vm_callee_setup_block_arg_arg0_splat(cfp, iseq, argv, arg0);
5394 }
5395
5396 if (calling->argc != ISEQ_BODY(iseq)->param.lead_num) {
5397 if (arg_setup_type == arg_setup_block) {
5398 if (calling->argc < ISEQ_BODY(iseq)->param.lead_num) {
5399 int i;
5400 CHECK_VM_STACK_OVERFLOW(cfp, ISEQ_BODY(iseq)->param.lead_num);
5401 for (i=calling->argc; i<ISEQ_BODY(iseq)->param.lead_num; i++) argv[i] = Qnil;
5402 calling->argc = ISEQ_BODY(iseq)->param.lead_num; /* fill rest parameters */
5403 }
5404 else if (calling->argc > ISEQ_BODY(iseq)->param.lead_num) {
5405 calling->argc = ISEQ_BODY(iseq)->param.lead_num; /* simply truncate arguments */
5406 }
5407 }
5408 else {
5409 argument_arity_error(ec, iseq, NULL, calling->argc, ISEQ_BODY(iseq)->param.lead_num, ISEQ_BODY(iseq)->param.lead_num);
5410 }
5411 }
5412
5413 return 0;
5414 }
5415 else {
5416 return setup_parameters_complex(ec, iseq, calling, ci, argv, arg_setup_type);
5417 }
5418}
5419
5420static int
5421vm_yield_setup_args(rb_execution_context_t *ec, const rb_iseq_t *iseq, const int argc, VALUE *argv, int flags, VALUE block_handler, enum arg_setup_type arg_setup_type)
5422{
5423 struct rb_calling_info calling_entry, *calling;
5424
5425 calling = &calling_entry;
5426 calling->argc = argc;
5427 calling->block_handler = block_handler;
5428 calling->kw_splat = (flags & VM_CALL_KW_SPLAT) ? 1 : 0;
5429 calling->recv = Qundef;
5430 calling->heap_argv = 0;
5431 calling->cc = NULL;
5432 struct rb_callinfo dummy_ci = VM_CI_ON_STACK(0, flags, 0, 0);
5433
5434 return vm_callee_setup_block_arg(ec, calling, &dummy_ci, iseq, argv, arg_setup_type);
5435}
5436
5437/* ruby iseq -> ruby block */
5438
5439static VALUE
5440vm_invoke_iseq_block_with_cref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5441 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5442 bool is_lambda, VALUE block_handler, const rb_cref_t *cref)
5443{
5444 const struct rb_captured_block *captured = VM_BH_TO_ISEQ_BLOCK(block_handler);
5445 const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
5446 const int arg_size = ISEQ_BODY(iseq)->param.size;
5447 VALUE * const rsp = GET_SP() - calling->argc;
5448 VALUE * const argv = rsp;
5449 int opt_pc = vm_callee_setup_block_arg(ec, calling, ci, iseq, argv, is_lambda ? arg_setup_method : arg_setup_block);
5450 int frame_flag = VM_FRAME_MAGIC_BLOCK | (is_lambda ? VM_FRAME_FLAG_LAMBDA : 0);
5451
5452 SET_SP(rsp);
5453
5454 /* cref is normally 0; Proc#refined supplies a refinement cref */
5455 vm_push_frame(ec, iseq,
5456 frame_flag,
5457 captured->self,
5458 VM_GUARDED_PREV_EP(captured->ep), (VALUE)cref,
5459 ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
5460 rsp + arg_size,
5461 ISEQ_BODY(iseq)->local_table_size - arg_size, ISEQ_BODY(iseq)->stack_max);
5462
5463 return Qundef;
5464}
5465
5466static VALUE
5467vm_invoke_iseq_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5468 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5469 bool is_lambda, VALUE block_handler)
5470{
5471 return vm_invoke_iseq_block_with_cref(ec, reg_cfp, calling, ci, is_lambda, block_handler, NULL);
5472}
5473
5474static VALUE
5475vm_invoke_symbol_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5476 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5477 MAYBE_UNUSED(bool is_lambda), VALUE block_handler)
5478{
5479 VALUE symbol = VM_BH_TO_SYMBOL(block_handler);
5480 int flags = vm_ci_flag(ci);
5481
5482 if (UNLIKELY(!(flags & VM_CALL_ARGS_SIMPLE) &&
5483 ((calling->argc == 0) ||
5484 (calling->argc == 1 && (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT))) ||
5485 (calling->argc == 2 && (flags & VM_CALL_ARGS_SPLAT) && (flags & VM_CALL_KW_SPLAT)) ||
5486 ((flags & VM_CALL_KWARG) && (vm_ci_kwarg(ci)->keyword_len == calling->argc))))) {
5487 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV);
5488 flags = 0;
5489 if (UNLIKELY(calling->heap_argv)) {
5490#if VM_ARGC_STACK_MAX < 0
5491 if (RARRAY_LEN(calling->heap_argv) < 1) {
5492 rb_raise(rb_eArgError, "no receiver given");
5493 }
5494#endif
5495 calling->recv = rb_ary_shift(calling->heap_argv);
5496 // Modify stack to avoid cfp consistency error
5497 reg_cfp->sp++;
5498 reg_cfp->sp[-1] = reg_cfp->sp[-2];
5499 reg_cfp->sp[-2] = calling->recv;
5500 flags |= VM_CALL_ARGS_SPLAT;
5501 }
5502 else {
5503 if (calling->argc < 1) {
5504 rb_raise(rb_eArgError, "no receiver given");
5505 }
5506 calling->recv = TOPN(--calling->argc);
5507 }
5508 if (calling->kw_splat) {
5509 flags |= VM_CALL_KW_SPLAT;
5510 }
5511 }
5512 else {
5513 if (calling->argc < 1) {
5514 rb_raise(rb_eArgError, "no receiver given");
5515 }
5516 calling->recv = TOPN(--calling->argc);
5517 }
5518
5519 return vm_call_symbol(ec, reg_cfp, calling, ci, symbol, flags);
5520}
5521
5522static VALUE
5523vm_invoke_ifunc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5524 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5525 MAYBE_UNUSED(bool is_lambda), VALUE block_handler)
5526{
5527 VALUE val;
5528 int argc;
5529 const struct rb_captured_block *captured = VM_BH_TO_IFUNC_BLOCK(block_handler);
5530 CALLER_SETUP_ARG(ec->cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT);
5531 argc = calling->argc;
5532 val = vm_yield_with_cfunc(ec, captured, captured->self, CALLING_ARGC(calling), calling->heap_argv ? RARRAY_CONST_PTR(calling->heap_argv) : STACK_ADDR_FROM_TOP(argc), calling->kw_splat, calling->block_handler, NULL);
5533 POPN(argc); /* TODO: should put before C/yield? */
5534 return val;
5535}
5536
5537static inline VALUE
5538vm_block_to_block_handler(const struct rb_block *block)
5539{
5540 switch (vm_block_type(block)) {
5541 case block_type_iseq:
5542 return VM_BH_FROM_ISEQ_BLOCK(&block->as.captured);
5543 case block_type_ifunc:
5544 return VM_BH_FROM_IFUNC_BLOCK(&block->as.captured);
5545 case block_type_symbol:
5546 return VM_BH_FROM_SYMBOL(block->as.symbol);
5547 case block_type_proc:
5548 return VM_BH_FROM_PROC(block->as.proc);
5549 }
5550 VM_UNREACHABLE(vm_yield_with_proc);
5551 return Qundef;
5552}
5553
5554static VALUE
5555vm_proc_to_block_handler(VALUE procval)
5556{
5557 return vm_block_to_block_handler(vm_proc_block(procval));
5558}
5559
5560NOINLINE(static VALUE
5561vm_invoke_proc_block_with_cref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5562 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5563 bool is_lambda, VALUE block_handler, VALUE refined_procval));
5564static VALUE
5565vm_invoke_proc_block_with_cref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5566 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5567 bool is_lambda, VALUE block_handler, VALUE refined_procval)
5568{
5569 const rb_cref_t *cref = rb_proc_refinements_cref_for_call(refined_procval);
5570 return vm_invoke_iseq_block_with_cref(ec, reg_cfp, calling, ci, is_lambda, block_handler, cref);
5571}
5572
5573static VALUE
5574vm_invoke_proc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5575 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5576 bool is_lambda, VALUE block_handler)
5577{
5578 VALUE refined_procval = 0;
5579
5580 while (vm_block_handler_type(block_handler) == block_handler_type_proc) {
5581 VALUE procval = VM_BH_TO_PROC(block_handler);
5582 rb_proc_t *po;
5583 GetProcPtr(procval, po);
5584 if (po->header.is_refined) refined_procval = procval;
5585 is_lambda = po->header.is_lambda;
5586 block_handler = vm_block_to_block_handler(&po->block);
5587 }
5588
5589 if (UNLIKELY(refined_procval) && vm_block_handler_type(block_handler) == block_handler_type_iseq) {
5590 /* should not be inlined so that vm_invoke_proc_block stays leaf/frameless. */
5591 return vm_invoke_proc_block_with_cref(ec, reg_cfp, calling, ci, is_lambda, block_handler,
5592 refined_procval);
5593 }
5594
5595 return vm_invoke_block(ec, reg_cfp, calling, ci, is_lambda, block_handler);
5596}
5597
5598static inline VALUE
5599vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5600 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5601 bool is_lambda, VALUE block_handler)
5602{
5603 VALUE (*func)(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5604 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5605 bool is_lambda, VALUE block_handler);
5606
5607 switch (vm_block_handler_type(block_handler)) {
5608 case block_handler_type_iseq: func = vm_invoke_iseq_block; break;
5609 case block_handler_type_ifunc: func = vm_invoke_ifunc_block; break;
5610 case block_handler_type_proc: func = vm_invoke_proc_block; break;
5611 case block_handler_type_symbol: func = vm_invoke_symbol_block; break;
5612 default: rb_bug("vm_invoke_block: unreachable");
5613 }
5614
5615 return func(ec, reg_cfp, calling, ci, is_lambda, block_handler);
5616}
5617
5618static VALUE
5619vm_make_proc_with_iseq(const rb_iseq_t *blockiseq)
5620{
5621 const rb_execution_context_t *ec = GET_EC();
5622 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
5623 struct rb_captured_block *captured;
5624
5625 if (cfp == 0) {
5626 rb_bug("vm_make_proc_with_iseq: unreachable");
5627 }
5628
5629 captured = VM_CFP_TO_CAPTURED_BLOCK(cfp);
5630 captured->code.iseq = blockiseq;
5631
5632 return rb_vm_make_proc(ec, captured, rb_cProc);
5633}
5634
5635static VALUE
5636vm_once_exec(VALUE iseq)
5637{
5638 VALUE proc = vm_make_proc_with_iseq((rb_iseq_t *)iseq);
5639 return rb_proc_call_with_block(proc, 0, 0, Qnil);
5640}
5641
5642#define RUNNING_THREAD_ONCE_DONE ((rb_thread_t *)0x1)
5643
5644struct vm_once_wait_arg {
5645 rb_vm_t *vm;
5646 union iseq_inline_storage_entry *is;
5647};
5648
5649static void
5650vm_once_broadcast(rb_vm_t *vm)
5651{
5652 rb_native_mutex_lock(&vm->once_lock);
5653 rb_native_cond_broadcast(&vm->once_cond);
5654 rb_native_mutex_unlock(&vm->once_lock);
5655}
5656
5657static void *
5658vm_once_wait_no_gvl(void *ptr)
5659{
5660 struct vm_once_wait_arg *arg = (struct vm_once_wait_arg *)ptr;
5661 rb_vm_t *vm = arg->vm;
5662 rb_thread_t *running_th;
5663
5664 rb_native_mutex_lock(&vm->once_lock);
5665 running_th = rbimpl_atomic_ptr_load((void **)&arg->is->once.running_thread, RBIMPL_ATOMIC_ACQUIRE);
5666 if (running_th != NULL && running_th != RUNNING_THREAD_ONCE_DONE) {
5667 rb_native_cond_wait(&vm->once_cond, &vm->once_lock);
5668 }
5669 rb_native_mutex_unlock(&vm->once_lock);
5670 return NULL;
5671}
5672
5673static void
5674vm_once_wait_ubf(void *ptr)
5675{
5676 vm_once_broadcast((rb_vm_t *)ptr);
5677}
5678
5679static VALUE
5680vm_once_clear(VALUE data)
5681{
5682 union iseq_inline_storage_entry *is = (union iseq_inline_storage_entry *)data;
5683 rbimpl_atomic_ptr_store((volatile void **)&is->once.running_thread, NULL, RBIMPL_ATOMIC_RELEASE);
5684 vm_once_broadcast(GET_VM());
5685 return Qnil;
5686}
5687
5688/* defined insn */
5689
5690static bool
5691check_respond_to_missing(VALUE obj, VALUE v)
5692{
5693 VALUE args[2];
5694 VALUE r;
5695
5696 args[0] = obj; args[1] = Qfalse;
5697 r = rb_check_funcall(v, idRespond_to_missing, 2, args);
5698 if (!UNDEF_P(r) && RTEST(r)) {
5699 return true;
5700 }
5701 else {
5702 return false;
5703 }
5704}
5705
5706static bool
5707vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v)
5708{
5709 VALUE klass;
5710 enum defined_type type = (enum defined_type)op_type;
5711
5712 switch (type) {
5713 case DEFINED_IVAR:
5714 return rb_ivar_defined(GET_SELF(), SYM2ID(obj));
5715 break;
5716 case DEFINED_GVAR:
5717 return rb_gvar_defined(SYM2ID(obj));
5718 break;
5719 case DEFINED_CVAR: {
5720 const rb_cref_t *cref = vm_get_cref(GET_EP());
5721 klass = vm_get_cvar_base(cref, GET_CFP(), 0);
5722 return rb_cvar_defined(klass, SYM2ID(obj));
5723 break;
5724 }
5725 case DEFINED_CONST:
5726 case DEFINED_CONST_FROM: {
5727 bool allow_nil = type == DEFINED_CONST;
5728 klass = v;
5729 return vm_get_ev_const(ec, klass, SYM2ID(obj), allow_nil, true);
5730 break;
5731 }
5732 case DEFINED_FUNC:
5733 klass = CLASS_OF(v);
5734 return rb_ec_obj_respond_to(ec, v, SYM2ID(obj), TRUE);
5735 break;
5736 case DEFINED_METHOD:{
5737 VALUE klass = CLASS_OF(v);
5738 const rb_callable_method_entry_t *cme = rb_callable_method_entry_with_refinements(klass, SYM2ID(obj), NULL);
5739
5740 if (cme) {
5741 switch (METHOD_ENTRY_VISI(cme)) {
5742 case METHOD_VISI_PRIVATE:
5743 break;
5744 case METHOD_VISI_PROTECTED:
5745 if (!rb_obj_is_kind_of(GET_SELF(), vm_defined_class_for_protected_call(cme))) {
5746 break;
5747 }
5748 case METHOD_VISI_PUBLIC:
5749 return true;
5750 break;
5751 default:
5752 rb_bug("vm_defined: unreachable: %u", (unsigned int)METHOD_ENTRY_VISI(cme));
5753 }
5754 }
5755 else {
5756 return check_respond_to_missing(obj, v);
5757 }
5758 break;
5759 }
5760 case DEFINED_YIELD:
5761 if (GET_BLOCK_HANDLER() != VM_BLOCK_HANDLER_NONE) {
5762 return true;
5763 }
5764 break;
5765 case DEFINED_ZSUPER:
5766 {
5767 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(GET_CFP());
5768
5769 if (me) {
5770 VALUE klass = vm_search_normal_superclass(me->defined_class);
5771 if (!klass) return false;
5772
5773 ID id = me->def->original_id;
5774
5775 return rb_method_boundp(klass, id, 0);
5776 }
5777 }
5778 break;
5779 case DEFINED_REF:
5780 return RTEST(vm_backref_defined(ec, GET_LEP(), FIX2INT(obj)));
5781 default:
5782 rb_bug("unimplemented defined? type (VM)");
5783 break;
5784 }
5785
5786 return false;
5787}
5788
5789bool
5790rb_vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v)
5791{
5792 return vm_defined(ec, reg_cfp, op_type, obj, v);
5793}
5794
5795static const VALUE *
5796vm_get_ep(const VALUE *const reg_ep, rb_num_t lv)
5797{
5798 rb_num_t i;
5799 const VALUE *ep = reg_ep;
5800 for (i = 0; i < lv; i++) {
5801 ep = GET_PREV_EP(ep);
5802 }
5803 return ep;
5804}
5805
5806static VALUE
5807vm_get_special_object(const VALUE *const reg_ep,
5808 enum vm_special_object_type type)
5809{
5810 switch (type) {
5811 case VM_SPECIAL_OBJECT_VMCORE:
5812 return rb_mRubyVMFrozenCore;
5813 case VM_SPECIAL_OBJECT_CBASE:
5814 return vm_get_cbase(reg_ep);
5815 case VM_SPECIAL_OBJECT_CONST_BASE:
5816 return vm_get_const_base(reg_ep);
5817 default:
5818 rb_bug("putspecialobject insn: unknown value_type %d", type);
5819 }
5820}
5821
5822// ZJIT implementation is using the C function
5823// and needs to call a non-static function
5824VALUE
5825rb_vm_get_special_object(const VALUE *reg_ep, enum vm_special_object_type type)
5826{
5827 return vm_get_special_object(reg_ep, type);
5828}
5829
5830static VALUE
5831vm_concat_array(VALUE ary1, VALUE ary2st)
5832{
5833 const VALUE ary2 = ary2st;
5834 VALUE tmp1 = rb_check_to_array(ary1);
5835 VALUE tmp2 = rb_check_to_array(ary2);
5836
5837 if (NIL_P(tmp1)) {
5838 tmp1 = rb_ary_new3(1, ary1);
5839 }
5840 if (tmp1 == ary1) {
5841 tmp1 = rb_ary_dup(ary1);
5842 }
5843
5844 if (NIL_P(tmp2)) {
5845 return rb_ary_push(tmp1, ary2);
5846 }
5847 else {
5848 return rb_ary_concat(tmp1, tmp2);
5849 }
5850}
5851
5852static VALUE
5853vm_concat_to_array(VALUE ary1, VALUE ary2st)
5854{
5855 /* ary1 must be a newly created array */
5856 const VALUE ary2 = ary2st;
5857
5858 if (NIL_P(ary2)) return ary1;
5859
5860 VALUE tmp2 = rb_check_to_array(ary2);
5861
5862 if (NIL_P(tmp2)) {
5863 return rb_ary_push(ary1, ary2);
5864 }
5865 else {
5866 return rb_ary_concat(ary1, tmp2);
5867 }
5868}
5869
5870// YJIT implementation is using the C function
5871// and needs to call a non-static function
5872VALUE
5873rb_vm_concat_array(VALUE ary1, VALUE ary2st)
5874{
5875 return vm_concat_array(ary1, ary2st);
5876}
5877
5878VALUE
5879rb_vm_concat_to_array(VALUE ary1, VALUE ary2st)
5880{
5881 return vm_concat_to_array(ary1, ary2st);
5882}
5883
5884static VALUE
5885vm_splat_array(VALUE flag, VALUE ary)
5886{
5887 if (NIL_P(ary)) {
5888 return RTEST(flag) ? rb_ary_new() : rb_cArray_empty_frozen;
5889 }
5890 VALUE tmp = rb_check_to_array(ary);
5891 if (NIL_P(tmp)) {
5892 return rb_ary_new3(1, ary);
5893 }
5894 else if (RTEST(flag)) {
5895 return rb_ary_dup(tmp);
5896 }
5897 else {
5898 return tmp;
5899 }
5900}
5901
5902// YJIT implementation is using the C function
5903// and needs to call a non-static function
5904VALUE
5905rb_vm_splat_array(VALUE flag, VALUE ary)
5906{
5907 return vm_splat_array(flag, ary);
5908}
5909
5910static VALUE
5911vm_check_match(rb_execution_context_t *ec, VALUE target, VALUE pattern, rb_num_t flag)
5912{
5913 enum vm_check_match_type type = ((int)flag) & VM_CHECKMATCH_TYPE_MASK;
5914
5915 if (flag & VM_CHECKMATCH_ARRAY) {
5916 long i;
5917 const long n = RARRAY_LEN(pattern);
5918
5919 for (i = 0; i < n; i++) {
5920 VALUE v = RARRAY_AREF(pattern, i);
5921 VALUE c = check_match(ec, v, target, type);
5922
5923 if (RTEST(c)) {
5924 return c;
5925 }
5926 }
5927 return Qfalse;
5928 }
5929 else {
5930 return check_match(ec, pattern, target, type);
5931 }
5932}
5933
5934VALUE
5935rb_vm_check_match(rb_execution_context_t *ec, VALUE target, VALUE pattern, rb_num_t flag)
5936{
5937 return vm_check_match(ec, target, pattern, flag);
5938}
5939
5940static VALUE
5941vm_check_keyword(lindex_t bits, lindex_t idx, const VALUE *ep)
5942{
5943 const VALUE kw_bits = *(ep - bits);
5944
5945 if (FIXNUM_P(kw_bits)) {
5946 unsigned int b = (unsigned int)FIX2ULONG(kw_bits);
5947 if ((idx < VM_KW_SPECIFIED_BITS_MAX) && (b & (0x01 << idx)))
5948 return Qfalse;
5949 }
5950 else {
5951 VM_ASSERT(RB_TYPE_P(kw_bits, T_HASH));
5952 if (rb_hash_has_key(kw_bits, INT2FIX(idx))) return Qfalse;
5953 }
5954 return Qtrue;
5955}
5956
5957static void
5958vm_dtrace(rb_event_flag_t flag, rb_execution_context_t *ec)
5959{
5960 if (RUBY_DTRACE_METHOD_ENTRY_ENABLED() ||
5961 RUBY_DTRACE_METHOD_RETURN_ENABLED() ||
5962 RUBY_DTRACE_CMETHOD_ENTRY_ENABLED() ||
5963 RUBY_DTRACE_CMETHOD_RETURN_ENABLED()) {
5964
5965 switch (flag) {
5966 case RUBY_EVENT_CALL:
5967 RUBY_DTRACE_METHOD_ENTRY_HOOK(ec, 0, 0);
5968 return;
5969 case RUBY_EVENT_C_CALL:
5970 RUBY_DTRACE_CMETHOD_ENTRY_HOOK(ec, 0, 0);
5971 return;
5972 case RUBY_EVENT_RETURN:
5973 RUBY_DTRACE_METHOD_RETURN_HOOK(ec, 0, 0);
5974 return;
5975 case RUBY_EVENT_C_RETURN:
5976 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, 0, 0);
5977 return;
5978 }
5979 }
5980}
5981
5982static VALUE
5983vm_const_get_under(ID id, rb_num_t flags, VALUE cbase)
5984{
5985 if (!rb_const_defined_at(cbase, id)) {
5986 return 0;
5987 }
5988 else if (VM_DEFINECLASS_SCOPED_P(flags)) {
5989 return rb_public_const_get_at(cbase, id);
5990 }
5991 else {
5992 return rb_const_get_at(cbase, id);
5993 }
5994}
5995
5996static VALUE
5997vm_check_if_class(ID id, rb_num_t flags, VALUE super, VALUE klass)
5998{
5999 if (!RB_TYPE_P(klass, T_CLASS)) {
6000 return 0;
6001 }
6002 else if (VM_DEFINECLASS_HAS_SUPERCLASS_P(flags)) {
6003 VALUE tmp = rb_class_real(RCLASS_SUPER(klass));
6004
6005 if (tmp != super) {
6006 rb_raise(rb_eTypeError,
6007 "superclass mismatch for class %"PRIsVALUE"",
6008 rb_id2str(id));
6009 }
6010 else {
6011 return klass;
6012 }
6013 }
6014 else {
6015 return klass;
6016 }
6017}
6018
6019static VALUE
6020vm_check_if_module(ID id, VALUE mod)
6021{
6022 if (!RB_TYPE_P(mod, T_MODULE)) {
6023 return 0;
6024 }
6025 else {
6026 return mod;
6027 }
6028}
6029
6030static VALUE
6031declare_under(ID id, VALUE cbase, VALUE c)
6032{
6033 rb_set_class_path_string(c, cbase, rb_id2str(id));
6034 rb_const_set(cbase, id, c);
6035 return c;
6036}
6037
6038static VALUE
6039vm_declare_class(ID id, rb_num_t flags, VALUE cbase, VALUE super)
6040{
6041 /* new class declaration */
6042 VALUE s = VM_DEFINECLASS_HAS_SUPERCLASS_P(flags) ? super : rb_cObject;
6043 VALUE c = declare_under(id, cbase, rb_define_class_id(id, s));
6044 rb_class_inherited(s, c);
6045 return c;
6046}
6047
6048static VALUE
6049vm_declare_module(ID id, VALUE cbase)
6050{
6051 /* new module declaration */
6052 return declare_under(id, cbase, rb_module_new());
6053}
6054
6055NORETURN(static void unmatched_redefinition(const char *type, VALUE cbase, ID id, VALUE old));
6056static void
6057unmatched_redefinition(const char *type, VALUE cbase, ID id, VALUE old)
6058{
6059 VALUE name = rb_id2str(id);
6060 VALUE message = rb_sprintf("%"PRIsVALUE" is not a %s",
6061 name, type);
6062 VALUE location = rb_const_source_location_at(cbase, id);
6063 if (!NIL_P(location)) {
6064 rb_str_catf(message, "\n%"PRIsVALUE":%"PRIsVALUE":"
6065 " previous definition of %"PRIsVALUE" was here",
6066 rb_ary_entry(location, 0), rb_ary_entry(location, 1), name);
6067 }
6068 rb_exc_raise(rb_exc_new_str(rb_eTypeError, message));
6069}
6070
6071static VALUE
6072vm_define_class(ID id, rb_num_t flags, VALUE cbase, VALUE super)
6073{
6074 VALUE klass;
6075
6076 if (VM_DEFINECLASS_HAS_SUPERCLASS_P(flags) && !RB_TYPE_P(super, T_CLASS)) {
6077 rb_raise(rb_eTypeError,
6078 "superclass must be an instance of Class (given an instance of %"PRIsVALUE")",
6079 rb_obj_class(super));
6080 }
6081
6082 vm_check_if_namespace(cbase);
6083
6084 /* find klass */
6085 rb_autoload_load(cbase, id);
6086
6087 if ((klass = vm_const_get_under(id, flags, cbase)) != 0) {
6088 if (!vm_check_if_class(id, flags, super, klass))
6089 unmatched_redefinition("class", cbase, id, klass);
6090 return klass;
6091 }
6092 else {
6093 return vm_declare_class(id, flags, cbase, super);
6094 }
6095}
6096
6097static VALUE
6098vm_define_module(ID id, rb_num_t flags, VALUE cbase)
6099{
6100 VALUE mod;
6101
6102 vm_check_if_namespace(cbase);
6103 if ((mod = vm_const_get_under(id, flags, cbase)) != 0) {
6104 if (!vm_check_if_module(id, mod))
6105 unmatched_redefinition("module", cbase, id, mod);
6106 return mod;
6107 }
6108 else {
6109 return vm_declare_module(id, cbase);
6110 }
6111}
6112
6113static VALUE
6114vm_find_or_create_class_by_id(ID id,
6115 rb_num_t flags,
6116 VALUE cbase,
6117 VALUE super)
6118{
6119 rb_vm_defineclass_type_t type = VM_DEFINECLASS_TYPE(flags);
6120
6121 switch (type) {
6122 case VM_DEFINECLASS_TYPE_CLASS:
6123 /* classdef returns class scope value */
6124 return vm_define_class(id, flags, cbase, super);
6125
6126 case VM_DEFINECLASS_TYPE_SINGLETON_CLASS:
6127 /* classdef returns class scope value */
6128 return rb_singleton_class(cbase);
6129
6130 case VM_DEFINECLASS_TYPE_MODULE:
6131 /* classdef returns class scope value */
6132 return vm_define_module(id, flags, cbase);
6133
6134 default:
6135 rb_bug("unknown defineclass type: %d", (int)type);
6136 }
6137}
6138
6139static rb_method_visibility_t
6140vm_scope_visibility_get(const rb_execution_context_t *ec)
6141{
6142 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
6143
6144 if (!vm_env_cref_by_cref(cfp->ep)) {
6145 return METHOD_VISI_PUBLIC;
6146 }
6147 else {
6148 return CREF_SCOPE_VISI(vm_ec_cref(ec))->method_visi;
6149 }
6150}
6151
6152static int
6153vm_scope_module_func_check(const rb_execution_context_t *ec)
6154{
6155 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
6156
6157 if (!vm_env_cref_by_cref(cfp->ep)) {
6158 return FALSE;
6159 }
6160 else {
6161 return CREF_SCOPE_VISI(vm_ec_cref(ec))->module_func;
6162 }
6163}
6164
6165static void
6166vm_define_method(const rb_execution_context_t *ec, VALUE obj, ID id, VALUE iseqval, int is_singleton)
6167{
6168 VALUE klass;
6169 rb_method_visibility_t visi;
6170 rb_cref_t *cref = vm_ec_cref(ec);
6171
6172 if (is_singleton) {
6173 klass = rb_singleton_class(obj); /* class and frozen checked in this API */
6174 visi = METHOD_VISI_PUBLIC;
6175 }
6176 else {
6177 klass = CREF_CLASS_FOR_DEFINITION(cref);
6178 visi = vm_scope_visibility_get(ec);
6179 }
6180
6181 if (NIL_P(klass)) {
6182 rb_raise(rb_eTypeError, "no class/module to add method");
6183 }
6184
6185 rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, visi);
6186 // Set max_iv_count on klasses based on number of ivar sets that are in the initialize method
6187 if (id == idInitialize && RB_TYPE_P(klass, T_CLASS) &&
6188 !RCLASS_SINGLETON_P(klass) && !RCLASS_EXPECT_NO_IVAR(klass) &&
6189 (rb_get_alloc_func(klass) == rb_class_allocate_instance)) {
6190 RCLASS_SET_MAX_IV_COUNT(klass, rb_estimate_iv_count(klass, (const rb_iseq_t *)iseqval));
6191 }
6192
6193 if (!is_singleton && vm_scope_module_func_check(ec)) {
6194 klass = rb_singleton_class(klass);
6195 rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, METHOD_VISI_PUBLIC);
6196 }
6197}
6198
6199// Return the untagged block handler:
6200// * If it's VM_BLOCK_HANDLER_NONE, return nil
6201// * If it's an ISEQ or an IFUNC, fetch it from its rb_captured_block
6202// * If it's a PROC or SYMBOL, return it as is
6203VALUE
6204rb_vm_untag_block_handler(VALUE block_handler)
6205{
6206 if (VM_BLOCK_HANDLER_NONE == block_handler) return Qnil;
6207
6208 switch (vm_block_handler_type(block_handler)) {
6209 case block_handler_type_iseq:
6210 case block_handler_type_ifunc: {
6211 struct rb_captured_block *captured = VM_TAGGED_PTR_REF(block_handler, 0x03);
6212 return captured->code.val;
6213 }
6214 case block_handler_type_proc:
6215 case block_handler_type_symbol:
6216 return block_handler;
6217 default:
6218 rb_bug("rb_vm_untag_block_handler: unreachable");
6219 }
6220}
6221
6222VALUE
6223rb_vm_get_untagged_block_handler(rb_control_frame_t *reg_cfp)
6224{
6225 return rb_vm_untag_block_handler(VM_CF_BLOCK_HANDLER(reg_cfp));
6226}
6227
6228static VALUE
6229vm_invokeblock_i(struct rb_execution_context_struct *ec,
6230 struct rb_control_frame_struct *reg_cfp,
6231 struct rb_calling_info *calling)
6232{
6233 const struct rb_callinfo *ci = calling->cd->ci;
6234 VALUE block_handler = VM_CF_BLOCK_HANDLER(GET_CFP());
6235
6236 if (block_handler == VM_BLOCK_HANDLER_NONE) {
6237 rb_vm_localjump_error("no block given (yield)", Qnil, 0);
6238 }
6239 else {
6240 return vm_invoke_block(ec, GET_CFP(), calling, ci, false, block_handler);
6241 }
6242}
6243
6244enum method_explorer_type {
6245 mexp_search_method,
6246 mexp_search_invokeblock,
6247 mexp_search_super,
6248};
6249
6250ALWAYS_INLINE(static VALUE vm_sendish(struct rb_execution_context_struct *ec,
6251 struct rb_control_frame_struct *reg_cfp,
6252 struct rb_call_data *cd, VALUE block_handler,
6253 enum method_explorer_type method_explorer));
6254static inline VALUE
6255vm_sendish(
6256 struct rb_execution_context_struct *ec,
6257 struct rb_control_frame_struct *reg_cfp,
6258 struct rb_call_data *cd,
6259 VALUE block_handler,
6260 enum method_explorer_type method_explorer
6261) {
6262 VALUE val = Qundef;
6263 const struct rb_callinfo *ci = cd->ci;
6264 const struct rb_callcache *cc;
6265 int argc = vm_ci_argc(ci);
6266 VALUE recv = TOPN(argc);
6267 struct rb_calling_info calling = {
6268 .block_handler = block_handler,
6269 .kw_splat = IS_ARGS_KW_SPLAT(ci) > 0,
6270 .recv = recv,
6271 .argc = argc,
6272 .cd = cd,
6273 };
6274
6275 switch (method_explorer) {
6276 case mexp_search_method:
6277 calling.cc = cc = vm_search_method_fastpath(reg_cfp, cd, CLASS_OF(recv));
6278 val = vm_cc_call(cc)(ec, GET_CFP(), &calling);
6279 break;
6280 case mexp_search_super:
6281 calling.cc = cc = vm_search_super_method(reg_cfp, cd, recv);
6282 val = vm_cc_call(cc)(ec, GET_CFP(), &calling);
6283 break;
6284 case mexp_search_invokeblock:
6285 val = vm_invokeblock_i(ec, GET_CFP(), &calling);
6286 break;
6287 }
6288 return val;
6289}
6290
6291VALUE
6292rb_vm_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6293{
6294 stack_check(ec);
6295 VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), cd->ci, blockiseq, false);
6296 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);
6297 VM_EXEC(ec, val);
6298 return val;
6299}
6300
6301// Fallback for YJIT/ZJIT, not used by the interpreter
6302VALUE
6303rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6304{
6305 stack_check(ec);
6306
6307 struct rb_forwarding_call_data adjusted_cd;
6308 struct rb_callinfo adjusted_ci;
6309
6310 VALUE bh = vm_caller_setup_fwd_args(GET_EC(), GET_CFP(), cd, blockiseq, false, &adjusted_cd, &adjusted_ci);
6311
6312 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method);
6313
6314 if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
6315 RB_OBJ_WRITE(CFP_ISEQ(GET_CFP()), &cd->cc, adjusted_cd.cd.cc);
6316 }
6317
6318 VM_EXEC(ec, val);
6319 return val;
6320}
6321
6322VALUE
6323rb_vm_opt_send_without_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd)
6324{
6325 stack_check(ec);
6326 VALUE bh = VM_BLOCK_HANDLER_NONE;
6327 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);
6328 VM_EXEC(ec, val);
6329 return val;
6330}
6331
6332VALUE
6333rb_vm_invokesuper(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6334{
6335 stack_check(ec);
6336
6337 struct rb_callinfo adjusted_ci = VM_CI_ON_STACK(vm_ci_mid(cd->ci),
6338 vm_ci_flag(cd->ci),
6339 vm_ci_argc(cd->ci),
6340 vm_ci_kwarg(cd->ci));
6341 const struct rb_callcache *original_cc = rbimpl_atomic_ptr_load((void **)&cd->cc, RBIMPL_ATOMIC_ACQUIRE);
6342 struct rb_call_data adjusted_cd = {
6343 .ci = &adjusted_ci,
6344 .cc = original_cc,
6345 };
6346
6347 VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), adjusted_cd.ci, blockiseq, true);
6348 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd, bh, mexp_search_super);
6349
6350 if (original_cc != adjusted_cd.cc && vm_cc_markable(adjusted_cd.cc)) {
6351 rbimpl_atomic_ptr_store((volatile void **)&cd->cc, (void *)adjusted_cd.cc, RBIMPL_ATOMIC_RELEASE);
6352 RB_OBJ_WRITTEN(CFP_ISEQ(GET_CFP()), Qundef, adjusted_cd.cc);
6353 }
6354
6355 VM_EXEC(ec, val);
6356 return val;
6357}
6358
6359// Fallback for YJIT/ZJIT, not used by the interpreter
6360VALUE
6361rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6362{
6363 stack_check(ec);
6364 struct rb_forwarding_call_data adjusted_cd;
6365 struct rb_callinfo adjusted_ci;
6366
6367 VALUE bh = vm_caller_setup_fwd_args(GET_EC(), GET_CFP(), cd, blockiseq, true, &adjusted_cd, &adjusted_ci);
6368
6369 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super);
6370
6371 if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
6372 RB_OBJ_WRITE(CFP_ISEQ(GET_CFP()), &cd->cc, adjusted_cd.cd.cc);
6373 }
6374
6375 VM_EXEC(ec, val);
6376 return val;
6377}
6378
6379VALUE
6380rb_vm_invokeblock(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd)
6381{
6382 stack_check(ec);
6383 VALUE bh = VM_BLOCK_HANDLER_NONE;
6384 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_invokeblock);
6385 VM_EXEC(ec, val);
6386 return val;
6387}
6388
6389/* object.c */
6390VALUE rb_nil_to_s(VALUE);
6391VALUE rb_true_to_s(VALUE);
6392VALUE rb_false_to_s(VALUE);
6393/* numeric.c */
6394VALUE rb_int_to_s(int argc, VALUE *argv, VALUE x);
6395VALUE rb_fix_to_s(VALUE);
6396/* variable.c */
6397VALUE rb_mod_to_s(VALUE);
6398VALUE rb_mod_name(VALUE);
6399
6400static VALUE
6401vm_objtostring(struct rb_control_frame_struct *reg_cfp, VALUE recv, CALL_DATA cd)
6402{
6403 int type = TYPE(recv);
6404 if (type == T_STRING) {
6405 return recv;
6406 }
6407
6408 const struct rb_callable_method_entry_struct *cme = vm_search_method(reg_cfp, cd, recv);
6409
6410 switch (type) {
6411 case T_SYMBOL:
6412 if (check_method_basic_definition(cme)) {
6413 return rb_sym2str(recv);
6414 }
6415 break;
6416 case T_MODULE:
6417 case T_CLASS:
6418 if (check_cfunc(cme, rb_mod_to_s)) {
6419 // rb_mod_to_s() allocates a mutable string, but since we are only
6420 // going to use this string for interpolation, it's fine to use the
6421 // frozen string.
6422 VALUE val = rb_mod_name(recv);
6423 if (NIL_P(val)) {
6424 val = rb_mod_to_s(recv);
6425 }
6426 return val;
6427 }
6428 break;
6429 case T_NIL:
6430 if (check_cfunc(cme, rb_nil_to_s)) {
6431 return rb_nil_to_s(recv);
6432 }
6433 break;
6434 case T_TRUE:
6435 if (check_cfunc(cme, rb_true_to_s)) {
6436 return rb_true_to_s(recv);
6437 }
6438 break;
6439 case T_FALSE:
6440 if (check_cfunc(cme, rb_false_to_s)) {
6441 return rb_false_to_s(recv);
6442 }
6443 break;
6444 case T_FIXNUM:
6445 if (check_cfunc(cme, rb_int_to_s)) {
6446 return rb_fix_to_s(recv);
6447 }
6448 break;
6449 }
6450 return Qundef;
6451}
6452
6453// ZJIT implementation is using the C function
6454// and needs to call a non-static function
6455VALUE
6456rb_vm_objtostring(struct rb_control_frame_struct *reg_cfp, VALUE recv, CALL_DATA cd)
6457{
6458 return vm_objtostring(reg_cfp, recv, cd);
6459}
6460
6461static VALUE
6462vm_opt_ary_freeze(VALUE ary, int bop, ID id)
6463{
6464 if (BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG)) {
6465 return ary;
6466 }
6467 else {
6468 return Qundef;
6469 }
6470}
6471
6472static VALUE
6473vm_opt_hash_freeze(VALUE hash, int bop, ID id)
6474{
6475 if (BASIC_OP_UNREDEFINED_P(bop, HASH_REDEFINED_OP_FLAG)) {
6476 return hash;
6477 }
6478 else {
6479 return Qundef;
6480 }
6481}
6482
6483static VALUE
6484vm_opt_str_freeze(VALUE str, int bop, ID id)
6485{
6486 if (BASIC_OP_UNREDEFINED_P(bop, STRING_REDEFINED_OP_FLAG)) {
6487 return str;
6488 }
6489 else {
6490 return Qundef;
6491 }
6492}
6493
6494/* this macro is mandatory to use OPTIMIZED_CMP. What a design! */
6495#define id_cmp idCmp
6496
6497static VALUE
6498vm_opt_duparray_include_p(rb_execution_context_t *ec, const VALUE ary, VALUE target)
6499{
6500 if (OP_UNREDEFINED_P(INCLUDE_P, ARRAY)) {
6501 return rb_ary_includes(ary, target);
6502 }
6503 else {
6504 VALUE args[1] = {target};
6505
6506 // duparray
6507 RUBY_DTRACE_CREATE_HOOK(ARRAY, RARRAY_LEN(ary));
6508 VALUE dupary = rb_ary_resurrect(ary);
6509
6510 return rb_vm_call_with_refinements(ec, dupary, idIncludeP, 1, args, RB_NO_KEYWORDS);
6511 }
6512}
6513
6514VALUE
6515rb_vm_opt_duparray_include_p(rb_execution_context_t *ec, const VALUE ary, VALUE target)
6516{
6517 return vm_opt_duparray_include_p(ec, ary, target);
6518}
6519
6520static VALUE
6521vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6522{
6523 if (OP_UNREDEFINED_P(MAX, ARRAY)) {
6524 if (array_len == 0) {
6525 return Qnil;
6526 }
6527 else {
6528 VALUE result = *ptr;
6529 rb_snum_t i = array_len - 1;
6530 while (i-- > 0) {
6531 const VALUE v = *++ptr;
6532 if (OPTIMIZED_CMP(v, result) > 0) {
6533 result = v;
6534 }
6535 }
6536 return result;
6537 }
6538 }
6539 else {
6540 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idMax, 0, NULL, RB_NO_KEYWORDS);
6541 }
6542}
6543
6544VALUE
6545rb_vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6546{
6547 return vm_opt_newarray_max(ec, array_len, ptr);
6548}
6549
6550static VALUE
6551vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6552{
6553 if (OP_UNREDEFINED_P(MIN, ARRAY)) {
6554 if (array_len == 0) {
6555 return Qnil;
6556 }
6557 else {
6558 VALUE result = *ptr;
6559 rb_snum_t i = array_len - 1;
6560 while (i-- > 0) {
6561 const VALUE v = *++ptr;
6562 if (OPTIMIZED_CMP(v, result) < 0) {
6563 result = v;
6564 }
6565 }
6566 return result;
6567 }
6568 }
6569 else {
6570 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idMin, 0, NULL, RB_NO_KEYWORDS);
6571 }
6572}
6573
6574VALUE
6575rb_vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6576{
6577 return vm_opt_newarray_min(ec, array_len, ptr);
6578}
6579
6580static VALUE
6581vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6582{
6583 // If Array#hash is _not_ monkeypatched, use the optimized call
6584 if (OP_UNREDEFINED_P(HASH, ARRAY)) {
6585 return rb_ary_hash_values(array_len, ptr);
6586 }
6587 else {
6588 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idHash, 0, NULL, RB_NO_KEYWORDS);
6589 }
6590}
6591
6592VALUE
6593rb_vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6594{
6595 return vm_opt_newarray_hash(ec, array_len, ptr);
6596}
6597
6598VALUE rb_setup_fake_ary(struct RArray *fake_ary, const VALUE *list, long len);
6599VALUE rb_ec_pack_ary(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer);
6600
6601static VALUE
6602vm_opt_newarray_include_p(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE target)
6603{
6604 if (OP_UNREDEFINED_P(INCLUDE_P, ARRAY)) {
6605 struct RArray fake_ary = {RBASIC_INIT};
6606 VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, array_len);
6607 return rb_ary_includes(ary, target);
6608 }
6609 else {
6610 VALUE args[1] = {target};
6611 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idIncludeP, 1, args, RB_NO_KEYWORDS);
6612 }
6613}
6614
6615VALUE
6616rb_vm_opt_newarray_include_p(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE target)
6617{
6618 return vm_opt_newarray_include_p(ec, array_len, ptr, target);
6619}
6620
6621static VALUE
6622vm_opt_newarray_pack_buffer(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt, VALUE buffer)
6623{
6624 if (OP_UNREDEFINED_P(PACK, ARRAY)) {
6625 struct RArray fake_ary = {RBASIC_INIT};
6626 VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, array_len);
6627 return rb_ec_pack_ary(ec, ary, fmt, (UNDEF_P(buffer) ? Qnil : buffer));
6628 }
6629 else {
6630 // The opt_newarray_send insn drops the keyword args so we need to rebuild them.
6631 // Setup an array with room for keyword hash.
6632 VALUE args[2];
6633 args[0] = fmt;
6634 int kw_splat = RB_NO_KEYWORDS;
6635 int argc = 1;
6636
6637 if (!UNDEF_P(buffer)) {
6638 args[1] = rb_hash_new_capa(1);
6639 rb_hash_aset(args[1], ID2SYM(idBuffer), buffer);
6640 kw_splat = RB_PASS_KEYWORDS;
6641 argc++;
6642 }
6643
6644 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idPack, argc, args, kw_splat);
6645 }
6646}
6647
6648VALUE
6649rb_vm_opt_newarray_pack_buffer(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt, VALUE buffer)
6650{
6651 return vm_opt_newarray_pack_buffer(ec, array_len, ptr, fmt, buffer);
6652}
6653
6654VALUE
6655rb_vm_opt_newarray_pack(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt)
6656{
6657 return vm_opt_newarray_pack_buffer(ec, array_len, ptr, fmt, Qundef);
6658}
6659
6660#undef id_cmp
6661
6662static void
6663vm_track_constant_cache(ID id, void *ic)
6664{
6665 rb_vm_t *vm = GET_VM();
6666 struct rb_id_table *const_cache = &vm->constant_cache;
6667 VALUE lookup_result;
6668 set_table *ics;
6669
6670 if (rb_id_table_lookup(const_cache, id, &lookup_result)) {
6671 ics = (set_table *)lookup_result;
6672 }
6673 else {
6674 ics = set_init_numtable();
6675 rb_id_table_insert(const_cache, id, (VALUE)ics);
6676 }
6677
6678 /* The call below to st_insert could allocate which could trigger a GC.
6679 * If it triggers a GC, it may free an iseq that also holds a cache to this
6680 * constant. If that iseq is the last iseq with a cache to this constant, then
6681 * it will free this ST table, which would cause an use-after-free during this
6682 * st_insert.
6683 *
6684 * So to fix this issue, we store the ID that is currently being inserted
6685 * and, in remove_from_constant_cache, we don't free the ST table for ID
6686 * equal to this one.
6687 *
6688 * See [Bug #20921].
6689 */
6690 vm->inserting_constant_cache_id = id;
6691
6692 set_insert(ics, (st_data_t)ic);
6693
6694 vm->inserting_constant_cache_id = (ID)0;
6695}
6696
6697static void
6698vm_ic_track_const_chain(rb_control_frame_t *cfp, IC ic, const ID *segments)
6699{
6700 RB_VM_LOCKING() {
6701 for (int i = 0; segments[i]; i++) {
6702 ID id = segments[i];
6703 if (id == idNULL) continue;
6704 vm_track_constant_cache(id, ic);
6705 }
6706 }
6707}
6708
6709// For JIT inlining
6710static inline bool
6711vm_inlined_ic_hit_p(VALUE flags, VALUE value, const rb_cref_t *ic_cref, const VALUE *reg_ep)
6712{
6713 if ((flags & IMEMO_CONST_CACHE_SHAREABLE) || rb_ractor_main_p()) {
6714 VM_ASSERT(ractor_incidental_shareable_p(flags & IMEMO_CONST_CACHE_SHAREABLE, value));
6715
6716 return (ic_cref == NULL || // no need to check CREF
6717 ic_cref == vm_get_cref(reg_ep));
6718 }
6719 return false;
6720}
6721
6722static bool
6723vm_ic_hit_p(const struct iseq_inline_constant_cache_entry *ice, const VALUE *reg_ep)
6724{
6725 VM_ASSERT(IMEMO_TYPE_P(ice, imemo_constcache));
6726 return vm_inlined_ic_hit_p(ice->flags, ice->value, ice->ic_cref, reg_ep);
6727}
6728
6729// YJIT needs this function to never allocate and never raise
6730bool
6731rb_vm_ic_hit_p(IC ic, const VALUE *reg_ep)
6732{
6733 return ic->entry && vm_ic_hit_p(ic->entry, reg_ep);
6734}
6735
6736static void
6737vm_ic_update(const rb_iseq_t *iseq, IC ic, VALUE val, const VALUE *reg_ep, const VALUE *pc)
6738{
6739 if (ruby_vm_const_missing_count > 0) {
6740 ruby_vm_const_missing_count = 0;
6741 ic->entry = NULL;
6742 return;
6743 }
6744
6745 struct iseq_inline_constant_cache_entry *ice = SHAREABLE_IMEMO_NEW(struct iseq_inline_constant_cache_entry, imemo_constcache, 0);
6746 RB_OBJ_WRITE(ice, &ice->value, val);
6747 ice->ic_cref = vm_get_const_key_cref(reg_ep);
6748
6749 if (rb_ractor_shareable_p(val)) {
6750 RUBY_ASSERT((rb_gc_verify_shareable(val), 1));
6751 ice->flags |= IMEMO_CONST_CACHE_SHAREABLE;
6752 }
6753 RB_OBJ_WRITE(iseq, &ic->entry, ice);
6754 RUBY_ASSERT(pc >= ISEQ_BODY(iseq)->iseq_encoded);
6755 unsigned pos = (unsigned)(pc - ISEQ_BODY(iseq)->iseq_encoded);
6756 rb_yjit_constant_ic_update(iseq, ic, pos);
6757}
6758
6759VALUE
6760rb_vm_opt_getconstant_path(rb_execution_context_t *ec, rb_control_frame_t *const reg_cfp, IC ic)
6761{
6762 VALUE val;
6763 const ID *segments = ic->segments;
6764 struct iseq_inline_constant_cache_entry *ice = ic->entry;
6765
6766 if (ice && vm_ic_hit_p(ice, GET_EP())) {
6767 val = ice->value;
6768
6769 // On a non-main ractor another ractor can concurrently reassign a
6770 // shareable constant (which invalidates ICs asynchronously), so the
6771 // cached-but-still-shareable value may legitimately differ from a
6772 // freshly recomputed chain. Only assert equality when single-ractor.
6773 VM_ASSERT(val == vm_get_ev_const_chain(ec, segments) || rb_multi_ractor_p());
6774 }
6775 else {
6776 ruby_vm_constant_cache_misses++;
6777 val = vm_get_ev_const_chain(ec, segments);
6778 vm_ic_track_const_chain(GET_CFP(), ic, segments);
6779 // Undo the PC increment to get the address to this instruction
6780 // INSN_ATTR(width) == 2
6781 vm_ic_update(CFP_ISEQ(GET_CFP()), ic, val, GET_EP(), CFP_PC(GET_CFP()) - 2);
6782 }
6783 return val;
6784}
6785
6786static VALUE
6787vm_once_dispatch(rb_execution_context_t *ec, ISEQ iseq, ISE is)
6788{
6789 rb_thread_t *th = rb_ec_thread_ptr(ec);
6790 rb_thread_t *running_th;
6791
6792 again:
6793 running_th = rbimpl_atomic_ptr_load((void**)&is->once.running_thread, RBIMPL_ATOMIC_ACQUIRE);
6794 if (running_th == RUNNING_THREAD_ONCE_DONE) {
6795 return is->once.value;
6796 }
6797 else if (running_th == NULL) {
6798 VALUE val = Qundef;
6799 enum ruby_tag_type state;
6800 if (rbimpl_atomic_ptr_cas((void**)&is->once.running_thread, running_th, th, RBIMPL_ATOMIC_RELEASE, RBIMPL_ATOMIC_RELAXED) != running_th) {
6801 goto again;
6802 }
6803 EC_PUSH_TAG(ec);
6804 if ((state = EC_EXEC_TAG()) == TAG_NONE) {
6805 val = vm_once_exec((VALUE)iseq);
6806 }
6807 EC_POP_TAG();
6808 if (state != TAG_NONE) {
6809 vm_once_clear((VALUE)is);
6810 EC_JUMP_TAG(ec, state);
6811 }
6812
6813 // TODO: confirm that it is shareable
6814 if (RB_FL_ABLE(val)) {
6815 RB_OBJ_SET_SHAREABLE(val);
6816 }
6817
6818 RB_OBJ_WRITE(CFP_ISEQ(ec->cfp), &is->once.value, val);
6819
6820 rbimpl_atomic_ptr_store((volatile void**)&is->once.running_thread, RUNNING_THREAD_ONCE_DONE, RBIMPL_ATOMIC_RELEASE); /* success */
6821 vm_once_broadcast(rb_ec_vm_ptr(ec));
6822 return val;
6823 }
6824 else if (running_th == th) {
6825 /* recursive once */
6826 return vm_once_exec((VALUE)iseq);
6827 }
6828 else {
6829 /* wait for the winner to finish */
6830 struct vm_once_wait_arg arg = { .vm = rb_ec_vm_ptr(ec), .is = is };
6831 rb_nogvl(vm_once_wait_no_gvl, &arg, vm_once_wait_ubf, arg.vm,
6832 RB_NOGVL_UBF_ASYNC_SAFE | RB_NOGVL_INTR_FAIL);
6833 RUBY_VM_CHECK_INTS(ec);
6834 goto again;
6835 }
6836}
6837
6838static OFFSET
6839vm_case_dispatch(CDHASH hash, OFFSET else_offset, VALUE key)
6840{
6841 switch (OBJ_BUILTIN_TYPE(key)) {
6842 case -1:
6843 case T_FLOAT:
6844 case T_SYMBOL:
6845 case T_BIGNUM:
6846 case T_STRING:
6847 if (BASIC_OP_UNREDEFINED_P(BOP_EQQ,
6848 SYMBOL_REDEFINED_OP_FLAG |
6849 INTEGER_REDEFINED_OP_FLAG |
6850 FLOAT_REDEFINED_OP_FLAG |
6851 NIL_REDEFINED_OP_FLAG |
6852 TRUE_REDEFINED_OP_FLAG |
6853 FALSE_REDEFINED_OP_FLAG |
6854 STRING_REDEFINED_OP_FLAG)) {
6855 st_data_t val;
6856 if (RB_FLOAT_TYPE_P(key)) {
6857 double kval = RFLOAT_VALUE(key);
6858 if (!isinf(kval) && modf(kval, &kval) == 0.0) {
6859 key = FIXABLE(kval) ? LONG2FIX((long)kval) : rb_dbl2big(kval);
6860 }
6861 }
6862 if (st_lookup(rb_imemo_cdhash_tbl(hash), key, &val)) {
6863 return (VALUE)val;
6864 }
6865 else {
6866 return else_offset;
6867 }
6868 }
6869 }
6870 return 0;
6871}
6872
6873NORETURN(static void
6874 vm_stack_consistency_error(const rb_execution_context_t *ec,
6875 const rb_control_frame_t *,
6876 const VALUE *));
6877static void
6878vm_stack_consistency_error(const rb_execution_context_t *ec,
6879 const rb_control_frame_t *cfp,
6880 const VALUE *bp)
6881{
6882 const ptrdiff_t nsp = VM_SP_CNT(ec, cfp->sp);
6883 const ptrdiff_t nbp = VM_SP_CNT(ec, bp);
6884 static const char stack_consistency_error[] =
6885 "Stack consistency error (sp: %"PRIdPTRDIFF", bp: %"PRIdPTRDIFF")";
6886#if defined RUBY_DEVEL
6887 VALUE mesg = rb_sprintf(stack_consistency_error, nsp, nbp);
6888 rb_str_cat_cstr(mesg, "\n");
6889 rb_str_append(mesg, rb_iseq_disasm(CFP_ISEQ(cfp)));
6890 rb_exc_fatal(rb_exc_new3(rb_eFatal, mesg));
6891#else
6892 rb_bug(stack_consistency_error, nsp, nbp);
6893#endif
6894}
6895
6896static VALUE
6897vm_opt_plus(VALUE recv, VALUE obj)
6898{
6899 if (FIXNUM_2_P(recv, obj) &&
6900 OP_UNREDEFINED_P(PLUS, INTEGER)) {
6901 return rb_fix_plus_fix(recv, obj);
6902 }
6903 else if (FLONUM_2_P(recv, obj) &&
6904 OP_UNREDEFINED_P(PLUS, FLOAT)) {
6905 return DBL2NUM(RFLOAT_VALUE(recv) + RFLOAT_VALUE(obj));
6906 }
6907 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6908 return Qundef;
6909 }
6910 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6911 RBASIC_CLASS(obj) == rb_cFloat &&
6912 OP_UNREDEFINED_P(PLUS, FLOAT)) {
6913 return DBL2NUM(RFLOAT_VALUE(recv) + RFLOAT_VALUE(obj));
6914 }
6915 else if (RBASIC_CLASS(recv) == rb_cString &&
6916 RBASIC_CLASS(obj) == rb_cString &&
6917 OP_UNREDEFINED_P(PLUS, STRING)) {
6918 return rb_str_opt_plus(recv, obj);
6919 }
6920 else if (RBASIC_CLASS(recv) == rb_cArray &&
6921 RBASIC_CLASS(obj) == rb_cArray &&
6922 OP_UNREDEFINED_P(PLUS, ARRAY)) {
6923 return rb_ary_plus(recv, obj);
6924 }
6925 else {
6926 return Qundef;
6927 }
6928}
6929
6930static VALUE
6931vm_opt_minus(VALUE recv, VALUE obj)
6932{
6933 if (FIXNUM_2_P(recv, obj) &&
6934 OP_UNREDEFINED_P(MINUS, INTEGER)) {
6935 return rb_fix_minus_fix(recv, obj);
6936 }
6937 else if (FLONUM_2_P(recv, obj) &&
6938 OP_UNREDEFINED_P(MINUS, FLOAT)) {
6939 return DBL2NUM(RFLOAT_VALUE(recv) - RFLOAT_VALUE(obj));
6940 }
6941 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6942 return Qundef;
6943 }
6944 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6945 RBASIC_CLASS(obj) == rb_cFloat &&
6946 OP_UNREDEFINED_P(MINUS, FLOAT)) {
6947 return DBL2NUM(RFLOAT_VALUE(recv) - RFLOAT_VALUE(obj));
6948 }
6949 else {
6950 return Qundef;
6951 }
6952}
6953
6954static VALUE
6955vm_opt_mult(VALUE recv, VALUE obj)
6956{
6957 if (FIXNUM_2_P(recv, obj) &&
6958 OP_UNREDEFINED_P(MULT, INTEGER)) {
6959 return rb_fix_mul_fix(recv, obj);
6960 }
6961 else if (FLONUM_2_P(recv, obj) &&
6962 OP_UNREDEFINED_P(MULT, FLOAT)) {
6963 return DBL2NUM(RFLOAT_VALUE(recv) * RFLOAT_VALUE(obj));
6964 }
6965 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6966 return Qundef;
6967 }
6968 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6969 RBASIC_CLASS(obj) == rb_cFloat &&
6970 OP_UNREDEFINED_P(MULT, FLOAT)) {
6971 return DBL2NUM(RFLOAT_VALUE(recv) * RFLOAT_VALUE(obj));
6972 }
6973 else {
6974 return Qundef;
6975 }
6976}
6977
6978static VALUE
6979vm_opt_div(VALUE recv, VALUE obj)
6980{
6981 if (FIXNUM_2_P(recv, obj) &&
6982 OP_UNREDEFINED_P(DIV, INTEGER)) {
6983 return (FIX2LONG(obj) == 0) ? Qundef : rb_fix_div_fix(recv, obj);
6984 }
6985 else if (FLONUM_2_P(recv, obj) &&
6986 OP_UNREDEFINED_P(DIV, FLOAT)) {
6987 return rb_flo_div_flo(recv, obj);
6988 }
6989 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6990 return Qundef;
6991 }
6992 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6993 RBASIC_CLASS(obj) == rb_cFloat &&
6994 OP_UNREDEFINED_P(DIV, FLOAT)) {
6995 return rb_flo_div_flo(recv, obj);
6996 }
6997 else {
6998 return Qundef;
6999 }
7000}
7001
7002static VALUE
7003vm_opt_mod(VALUE recv, VALUE obj)
7004{
7005 if (FIXNUM_2_P(recv, obj) &&
7006 OP_UNREDEFINED_P(MOD, INTEGER)) {
7007 return (FIX2LONG(obj) == 0) ? Qundef : rb_fix_mod_fix(recv, obj);
7008 }
7009 else if (FLONUM_2_P(recv, obj) &&
7010 OP_UNREDEFINED_P(MOD, FLOAT)) {
7011 return DBL2NUM(ruby_float_mod(RFLOAT_VALUE(recv), RFLOAT_VALUE(obj)));
7012 }
7013 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7014 return Qundef;
7015 }
7016 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7017 RBASIC_CLASS(obj) == rb_cFloat &&
7018 OP_UNREDEFINED_P(MOD, FLOAT)) {
7019 return DBL2NUM(ruby_float_mod(RFLOAT_VALUE(recv), RFLOAT_VALUE(obj)));
7020 }
7021 else {
7022 return Qundef;
7023 }
7024}
7025
7026static VALUE
7027vm_opt_neq(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, CALL_DATA cd_eq, VALUE recv, VALUE obj)
7028{
7029 if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_not_equal)) {
7030 VALUE val = opt_equality(reg_cfp, recv, obj, cd_eq);
7031
7032 if (!UNDEF_P(val)) {
7033 return RBOOL(!RTEST(val));
7034 }
7035 }
7036
7037 return Qundef;
7038}
7039
7040static VALUE
7041vm_opt_lt(VALUE recv, VALUE obj)
7042{
7043 if (FIXNUM_2_P(recv, obj) &&
7044 OP_UNREDEFINED_P(LT, INTEGER)) {
7045 return RBOOL((SIGNED_VALUE)recv < (SIGNED_VALUE)obj);
7046 }
7047 else if (FLONUM_2_P(recv, obj) &&
7048 OP_UNREDEFINED_P(LT, FLOAT)) {
7049 return RBOOL(RFLOAT_VALUE(recv) < RFLOAT_VALUE(obj));
7050 }
7051 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7052 return Qundef;
7053 }
7054 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7055 RBASIC_CLASS(obj) == rb_cFloat &&
7056 OP_UNREDEFINED_P(LT, FLOAT)) {
7057 return RBOOL(RFLOAT_VALUE(recv) < RFLOAT_VALUE(obj));
7058 }
7059 else {
7060 return Qundef;
7061 }
7062}
7063
7064static VALUE
7065vm_opt_le(VALUE recv, VALUE obj)
7066{
7067 if (FIXNUM_2_P(recv, obj) &&
7068 OP_UNREDEFINED_P(LE, INTEGER)) {
7069 return RBOOL((SIGNED_VALUE)recv <= (SIGNED_VALUE)obj);
7070 }
7071 else if (FLONUM_2_P(recv, obj) &&
7072 OP_UNREDEFINED_P(LE, FLOAT)) {
7073 return RBOOL(RFLOAT_VALUE(recv) <= RFLOAT_VALUE(obj));
7074 }
7075 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7076 return Qundef;
7077 }
7078 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7079 RBASIC_CLASS(obj) == rb_cFloat &&
7080 OP_UNREDEFINED_P(LE, FLOAT)) {
7081 return RBOOL(RFLOAT_VALUE(recv) <= RFLOAT_VALUE(obj));
7082 }
7083 else {
7084 return Qundef;
7085 }
7086}
7087
7088static VALUE
7089vm_opt_gt(VALUE recv, VALUE obj)
7090{
7091 if (FIXNUM_2_P(recv, obj) &&
7092 OP_UNREDEFINED_P(GT, INTEGER)) {
7093 return RBOOL((SIGNED_VALUE)recv > (SIGNED_VALUE)obj);
7094 }
7095 else if (FLONUM_2_P(recv, obj) &&
7096 OP_UNREDEFINED_P(GT, FLOAT)) {
7097 return RBOOL(RFLOAT_VALUE(recv) > RFLOAT_VALUE(obj));
7098 }
7099 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7100 return Qundef;
7101 }
7102 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7103 RBASIC_CLASS(obj) == rb_cFloat &&
7104 OP_UNREDEFINED_P(GT, FLOAT)) {
7105 return RBOOL(RFLOAT_VALUE(recv) > RFLOAT_VALUE(obj));
7106 }
7107 else {
7108 return Qundef;
7109 }
7110}
7111
7112static VALUE
7113vm_opt_ge(VALUE recv, VALUE obj)
7114{
7115 if (FIXNUM_2_P(recv, obj) &&
7116 OP_UNREDEFINED_P(GE, INTEGER)) {
7117 return RBOOL((SIGNED_VALUE)recv >= (SIGNED_VALUE)obj);
7118 }
7119 else if (FLONUM_2_P(recv, obj) &&
7120 OP_UNREDEFINED_P(GE, FLOAT)) {
7121 return RBOOL(RFLOAT_VALUE(recv) >= RFLOAT_VALUE(obj));
7122 }
7123 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
7124 return Qundef;
7125 }
7126 else if (RBASIC_CLASS(recv) == rb_cFloat &&
7127 RBASIC_CLASS(obj) == rb_cFloat &&
7128 OP_UNREDEFINED_P(GE, FLOAT)) {
7129 return RBOOL(RFLOAT_VALUE(recv) >= RFLOAT_VALUE(obj));
7130 }
7131 else {
7132 return Qundef;
7133 }
7134}
7135
7136
7137static VALUE
7138vm_opt_ltlt(VALUE recv, VALUE obj)
7139{
7140 if (SPECIAL_CONST_P(recv)) {
7141 return Qundef;
7142 }
7143 else if (RBASIC_CLASS(recv) == rb_cString &&
7144 OP_UNREDEFINED_P(LTLT, STRING)) {
7145 if (LIKELY(RB_TYPE_P(obj, T_STRING))) {
7146 return rb_str_buf_append(recv, obj);
7147 }
7148 else {
7149 return rb_str_concat(recv, obj);
7150 }
7151 }
7152 else if (RBASIC_CLASS(recv) == rb_cArray &&
7153 OP_UNREDEFINED_P(LTLT, ARRAY)) {
7154 return rb_ary_push(recv, obj);
7155 }
7156 else {
7157 return Qundef;
7158 }
7159}
7160
7161static VALUE
7162vm_opt_and(VALUE recv, VALUE obj)
7163{
7164 // If recv and obj are both fixnums, then the bottom tag bit
7165 // will be 1 on both. 1 & 1 == 1, so the result value will also
7166 // be a fixnum. If either side is *not* a fixnum, then the tag bit
7167 // will be 0, and we return Qundef.
7168 VALUE ret = ((SIGNED_VALUE) recv) & ((SIGNED_VALUE) obj);
7169
7170 if (FIXNUM_P(ret) &&
7171 OP_UNREDEFINED_P(AND, INTEGER)) {
7172 return ret;
7173 }
7174 else {
7175 return Qundef;
7176 }
7177}
7178
7179static VALUE
7180vm_opt_or(VALUE recv, VALUE obj)
7181{
7182 if (FIXNUM_2_P(recv, obj) &&
7183 OP_UNREDEFINED_P(OR, INTEGER)) {
7184 return recv | obj;
7185 }
7186 else {
7187 return Qundef;
7188 }
7189}
7190
7191static VALUE
7192vm_opt_aref(VALUE recv, VALUE obj)
7193{
7194 if (SPECIAL_CONST_P(recv)) {
7195 if (FIXNUM_2_P(recv, obj) &&
7196 OP_UNREDEFINED_P(AREF, INTEGER)) {
7197 return rb_fix_aref(recv, obj);
7198 }
7199 return Qundef;
7200 }
7201 else if (RBASIC_CLASS(recv) == rb_cArray &&
7202 OP_UNREDEFINED_P(AREF, ARRAY)) {
7203 if (FIXNUM_P(obj)) {
7204 return rb_ary_entry_internal(recv, FIX2LONG(obj));
7205 }
7206 else {
7207 return rb_ary_aref1(recv, obj);
7208 }
7209 }
7210 else if (RBASIC_CLASS(recv) == rb_cHash &&
7211 OP_UNREDEFINED_P(AREF, HASH)) {
7212 return rb_hash_aref(recv, obj);
7213 }
7214 else {
7215 return Qundef;
7216 }
7217}
7218
7219static VALUE
7220vm_opt_aset(VALUE recv, VALUE obj, VALUE set)
7221{
7222 if (SPECIAL_CONST_P(recv)) {
7223 return Qundef;
7224 }
7225 else if (RBASIC_CLASS(recv) == rb_cArray &&
7226 OP_UNREDEFINED_P(ASET, ARRAY) &&
7227 FIXNUM_P(obj)) {
7228 rb_ary_store(recv, FIX2LONG(obj), set);
7229 return set;
7230 }
7231 else if (RBASIC_CLASS(recv) == rb_cHash &&
7232 OP_UNREDEFINED_P(ASET, HASH)) {
7233 rb_hash_aset(recv, obj, set);
7234 return set;
7235 }
7236 else {
7237 return Qundef;
7238 }
7239}
7240
7241static VALUE
7242vm_opt_length(VALUE recv, int bop)
7243{
7244 if (SPECIAL_CONST_P(recv)) {
7245 return Qundef;
7246 }
7247 else if (RBASIC_CLASS(recv) == rb_cString &&
7248 BASIC_OP_UNREDEFINED_P(bop, STRING_REDEFINED_OP_FLAG)) {
7249 if (bop == BOP_EMPTY_P) {
7250 return LONG2NUM(RSTRING_LEN(recv));
7251 }
7252 else {
7253 return rb_str_length(recv);
7254 }
7255 }
7256 else if (RBASIC_CLASS(recv) == rb_cArray &&
7257 BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG)) {
7258 return LONG2NUM(RARRAY_LEN(recv));
7259 }
7260 else if (RBASIC_CLASS(recv) == rb_cHash &&
7261 BASIC_OP_UNREDEFINED_P(bop, HASH_REDEFINED_OP_FLAG)) {
7262 return INT2FIX(RHASH_SIZE(recv));
7263 }
7264 else {
7265 return Qundef;
7266 }
7267}
7268
7269static VALUE
7270vm_opt_empty_p(VALUE recv)
7271{
7272 switch (vm_opt_length(recv, BOP_EMPTY_P)) {
7273 case Qundef: return Qundef;
7274 case INT2FIX(0): return Qtrue;
7275 default: return Qfalse;
7276 }
7277}
7278
7279VALUE rb_false(VALUE obj);
7280
7281static VALUE
7282vm_opt_nil_p(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv)
7283{
7284 if (NIL_P(recv) &&
7285 OP_UNREDEFINED_P(NIL_P, NIL)) {
7286 return Qtrue;
7287 }
7288 else if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_false)) {
7289 return Qfalse;
7290 }
7291 else {
7292 return Qundef;
7293 }
7294}
7295
7296static VALUE
7297fix_succ(VALUE x)
7298{
7299 switch (x) {
7300 case ~0UL:
7301 /* 0xFFFF_FFFF == INT2FIX(-1)
7302 * `-1.succ` is of course 0. */
7303 return INT2FIX(0);
7304 case RSHIFT(~0UL, 1):
7305 /* 0x7FFF_FFFF == LONG2FIX(0x3FFF_FFFF)
7306 * 0x3FFF_FFFF + 1 == 0x4000_0000, which is a Bignum. */
7307 return rb_uint2big(1UL << (SIZEOF_LONG * CHAR_BIT - 2));
7308 default:
7309 /* LONG2FIX(FIX2LONG(x)+FIX2LONG(y))
7310 * == ((lx*2+1)/2 + (ly*2+1)/2)*2+1
7311 * == lx*2 + ly*2 + 1
7312 * == (lx*2+1) + (ly*2+1) - 1
7313 * == x + y - 1
7314 *
7315 * Here, if we put y := INT2FIX(1):
7316 *
7317 * == x + INT2FIX(1) - 1
7318 * == x + 2 .
7319 */
7320 return x + 2;
7321 }
7322}
7323
7324static VALUE
7325vm_opt_succ(VALUE recv)
7326{
7327 if (FIXNUM_P(recv) &&
7328 OP_UNREDEFINED_P(SUCC, INTEGER)) {
7329 return fix_succ(recv);
7330 }
7331 else if (SPECIAL_CONST_P(recv)) {
7332 return Qundef;
7333 }
7334 else if (RBASIC_CLASS(recv) == rb_cString &&
7335 OP_UNREDEFINED_P(SUCC, STRING)) {
7336 return rb_str_succ(recv);
7337 }
7338 else {
7339 return Qundef;
7340 }
7341}
7342
7343static VALUE
7344vm_opt_not(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv)
7345{
7346 if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_not)) {
7347 return RBOOL(!RTEST(recv));
7348 }
7349 else {
7350 return Qundef;
7351 }
7352}
7353
7354static VALUE
7355vm_opt_regexpmatch2(VALUE recv, VALUE obj)
7356{
7357 if (SPECIAL_CONST_P(recv)) {
7358 return Qundef;
7359 }
7360 else if (RBASIC_CLASS(recv) == rb_cString &&
7361 CLASS_OF(obj) == rb_cRegexp &&
7362 OP_UNREDEFINED_P(MATCH, STRING)) {
7363 return rb_reg_match(obj, recv);
7364 }
7365 else if (RBASIC_CLASS(recv) == rb_cRegexp &&
7366 OP_UNREDEFINED_P(MATCH, REGEXP)) {
7367 return rb_reg_match(recv, obj);
7368 }
7369 else {
7370 return Qundef;
7371 }
7372}
7373
7374#undef EQ_UNREDEFINED_P
7375#undef OP_UNREDEFINED_P
7376
7377rb_event_flag_t rb_iseq_event_flags(const rb_iseq_t *iseq, size_t pos);
7378
7379NOINLINE(static void vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp));
7380
7381static inline void
7382vm_trace_hook(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE *pc,
7383 rb_event_flag_t pc_events, rb_event_flag_t target_event,
7384 rb_hook_list_t *global_hooks, rb_hook_list_t *local_hooks, VALUE val)
7385{
7386 rb_event_flag_t event = pc_events & target_event;
7387 VALUE self = GET_SELF();
7388
7389 VM_ASSERT(rb_popcount64((uint64_t)event) == 1);
7390
7391 if (local_hooks) local_hooks->running++; // make sure they don't get deleted while global hooks run
7392
7393 if (event & global_hooks->events) {
7394 /* increment PC because source line is calculated with PC-1 */
7395 reg_cfp->pc++;
7396 vm_dtrace(event, ec);
7397 rb_exec_event_hook_orig(ec, global_hooks, event, self, 0, 0, 0 , val, 0);
7398 reg_cfp->pc--;
7399 }
7400
7401 if (local_hooks) local_hooks->running--;
7402 if (local_hooks != NULL) {
7403 if (event & local_hooks->events) {
7404 /* increment PC because source line is calculated with PC-1 */
7405 reg_cfp->pc++;
7406 rb_exec_event_hook_orig(ec, local_hooks, event, self, 0, 0, 0 , val, 0);
7407 reg_cfp->pc--;
7408 }
7409 }
7410}
7411
7412/* Re-fetch the iseq-local hook list before each event: a hook fired by an
7413 * earlier event at this pc can disable the last targeted TracePoint on this
7414 * iseq, which frees local_hooks. Reload from its stable source (the ractor's
7415 * targeted-hooks table) so we never dereference a dangling pointer. */
7416#define VM_TRACE_HOOK(target_event, val) do { \
7417 if ((pc_events & (target_event)) & enabled_flags) { \
7418 if (local_hooks_cnt > 0) local_hooks = rb_iseq_local_hooks(iseq, r, false); \
7419 vm_trace_hook(ec, reg_cfp, pc, pc_events, (target_event), global_hooks, local_hooks, (val)); \
7420 } \
7421} while (0)
7422
7423static VALUE
7424rescue_errinfo(rb_execution_context_t *ec, rb_control_frame_t *cfp)
7425{
7426 VM_ASSERT(VM_FRAME_RUBYFRAME_P(cfp));
7427 VM_ASSERT(ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_RESCUE);
7428 return cfp->ep[VM_ENV_INDEX_LAST_LVAR];
7429}
7430
7431static void
7432vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp)
7433{
7434 const VALUE *pc = reg_cfp->pc;
7435 rb_ractor_t *r = rb_ec_ractor_ptr(ec);
7436 rb_event_flag_t enabled_flags = r->pub.hooks.events & ISEQ_TRACE_EVENTS;
7437 rb_event_flag_t ractor_events = enabled_flags;
7438
7439 if (enabled_flags == 0 && rb_ractor_targeted_hooks_cnt(r) == 0) {
7440 return;
7441 }
7442 else {
7443 const rb_iseq_t *iseq = CFP_ISEQ(reg_cfp);
7444 size_t pos = pc - ISEQ_BODY(iseq)->iseq_encoded;
7445 rb_event_flag_t pc_events = rb_iseq_event_flags(iseq, pos);
7446 unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt;
7447 rb_hook_list_t *local_hooks = NULL;
7448 if (RB_UNLIKELY(local_hooks_cnt > 0)) {
7449 st_data_t val;
7450 if (st_lookup(rb_ractor_targeted_hooks(r), (st_data_t)iseq, &val)) {
7451 local_hooks = (rb_hook_list_t*)val;
7452 }
7453 }
7454 rb_event_flag_t iseq_local_events = local_hooks != NULL ? local_hooks->events : 0;
7455
7456 rb_hook_list_t *bmethod_local_hooks = NULL;
7457 rb_event_flag_t bmethod_local_events = 0;
7458 unsigned int bmethod_hooks_cnt = 0;
7459 rb_method_definition_t *bmethod_def = NULL;
7460 const bool bmethod_frame = VM_FRAME_BMETHOD_P(reg_cfp);
7461 enabled_flags |= iseq_local_events;
7462
7463 VM_ASSERT((iseq_local_events & ~ISEQ_TRACE_EVENTS) == 0);
7464
7465 if (bmethod_frame) {
7466 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp);
7467 VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD);
7468 bmethod_def = me->def;
7469 bmethod_hooks_cnt = me->def->body.bmethod.local_hooks_cnt;
7470 if (RB_UNLIKELY(bmethod_hooks_cnt > 0)) {
7471 st_data_t val;
7472 if (st_lookup(rb_ractor_targeted_hooks(r), (st_data_t)me->def, &val)) {
7473 bmethod_local_hooks = (rb_hook_list_t*)val;
7474 }
7475 if (bmethod_local_hooks) {
7476 bmethod_local_events = bmethod_local_hooks->events;
7477 }
7478 }
7479 }
7480
7481 if ((pc_events & enabled_flags) == 0 && !bmethod_frame) {
7482#if 0
7483 /* disable trace */
7484 /* TODO: incomplete */
7485 rb_iseq_trace_set(iseq, vm_event_flags & ISEQ_TRACE_EVENTS);
7486#else
7487 /* do not disable trace because of performance problem
7488 * (re-enable overhead)
7489 */
7490#endif
7491 return;
7492 }
7493 else if (ec->trace_arg != NULL) {
7494 /* already tracing */
7495 return;
7496 }
7497 else {
7498 rb_hook_list_t *global_hooks = rb_ec_ractor_hooks(ec);
7499 /* Note, not considering iseq local events here since the same
7500 * iseq could be used in multiple bmethods. */
7501 rb_event_flag_t bmethod_events = ractor_events | bmethod_local_events;
7502
7503 if (0) {
7504 VALUE path = rb_iseq_path(iseq);
7505 VALUE label = rb_iseq_label(iseq);
7506 ruby_debug_printf("vm_trace>>%4d (%4x) - %.*s:%d %.*s\n",
7507 (int)pos,
7508 (int)pc_events,
7509 RSTRING_LENINT(path), RSTRING_PTR(path),
7510 (int)rb_iseq_line_no(iseq, pos),
7511 RSTRING_LENINT(label), RSTRING_PTR(label));
7512 }
7513 VM_ASSERT(reg_cfp->pc == pc);
7514 VM_ASSERT(pc_events != 0);
7515
7516 /* check traces */
7517 if ((pc_events & RUBY_EVENT_B_CALL) && bmethod_frame && (bmethod_events & RUBY_EVENT_CALL)) {
7518 /* b_call instruction running as a method. Fire call event. */
7519 vm_trace_hook(ec, reg_cfp, pc, RUBY_EVENT_CALL, RUBY_EVENT_CALL, global_hooks, bmethod_local_hooks, Qundef);
7520 }
7521 VM_TRACE_HOOK(RUBY_EVENT_CLASS | RUBY_EVENT_CALL | RUBY_EVENT_B_CALL, Qundef);
7522 VM_TRACE_HOOK(RUBY_EVENT_RESCUE, rescue_errinfo(ec, reg_cfp));
7523 VM_TRACE_HOOK(RUBY_EVENT_LINE, Qundef);
7524 VM_TRACE_HOOK(RUBY_EVENT_COVERAGE_LINE, Qundef);
7525 VM_TRACE_HOOK(RUBY_EVENT_COVERAGE_BRANCH, Qundef);
7526 VM_TRACE_HOOK(RUBY_EVENT_END | RUBY_EVENT_RETURN | RUBY_EVENT_B_RETURN, TOPN(0));
7527 if ((pc_events & RUBY_EVENT_B_RETURN) && bmethod_frame && (bmethod_events & RUBY_EVENT_RETURN)) {
7528 /* b_return instruction running as a method. Fire return event. */
7529 /* An earlier event's hook at this pc may have freed bmethod_local_hooks
7530 * (see VM_TRACE_HOOK); reload it from its stable source. */
7531 if (bmethod_hooks_cnt > 0) bmethod_local_hooks = rb_method_def_local_hooks(bmethod_def, r, false);
7532 vm_trace_hook(ec, reg_cfp, pc, RUBY_EVENT_RETURN, RUBY_EVENT_RETURN, global_hooks, bmethod_local_hooks, TOPN(0));
7533 }
7534 }
7535 }
7536}
7537#undef VM_TRACE_HOOK
7538
7539#if VM_CHECK_MODE > 0
7540NORETURN( NOINLINE( COLDFUNC
7541void rb_vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)));
7542
7543void
7544Init_vm_stack_canary(void)
7545{
7546 /* This has to be called _after_ our PRNG is properly set up. */
7547 int n = ruby_fill_random_bytes(&vm_stack_canary, sizeof vm_stack_canary, false);
7548 vm_stack_canary |= 0x01; // valid VALUE (Fixnum)
7549
7550 vm_stack_canary_was_born = true;
7551 VM_ASSERT(n == 0);
7552}
7553
7554void
7555rb_vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)
7556{
7557 /* Because a method has already been called, why not call
7558 * another one. */
7559 const char *insn = rb_insns_name(i);
7560 VALUE inspection = rb_inspect(c);
7561 const char *str = StringValueCStr(inspection);
7562
7563 rb_bug("dead canary found at %s: %s", insn, str);
7564}
7565
7566#else
7567void Init_vm_stack_canary(void) { /* nothing to do */ }
7568#endif
7569
7570
7571/* a part of the following code is generated by this ruby script:
7572
757316.times{|i|
7574 typedef_args = (0...i).map{|j| "VALUE v#{j+1}"}.join(", ")
7575 typedef_args.prepend(", ") if i != 0
7576 call_args = (0...i).map{|j| "argv[#{j}]"}.join(", ")
7577 call_args.prepend(", ") if i != 0
7578 puts %Q{
7579static VALUE
7580builtin_invoker#{i}(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7581{
7582 typedef VALUE (*rb_invoke_funcptr#{i}_t)(rb_execution_context_t *ec, VALUE self#{typedef_args});
7583 return (*(rb_invoke_funcptr#{i}_t)funcptr)(ec, self#{call_args});
7584}}
7585}
7586
7587puts
7588puts "static VALUE (* const cfunc_invokers[])(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr) = {"
758916.times{|i|
7590 puts " builtin_invoker#{i},"
7591}
7592puts "};"
7593*/
7594
7595static VALUE
7596builtin_invoker0(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7597{
7598 typedef VALUE (*rb_invoke_funcptr0_t)(rb_execution_context_t *ec, VALUE self);
7599 return (*(rb_invoke_funcptr0_t)funcptr)(ec, self);
7600}
7601
7602static VALUE
7603builtin_invoker1(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7604{
7605 typedef VALUE (*rb_invoke_funcptr1_t)(rb_execution_context_t *ec, VALUE self, VALUE v1);
7606 return (*(rb_invoke_funcptr1_t)funcptr)(ec, self, argv[0]);
7607}
7608
7609static VALUE
7610builtin_invoker2(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7611{
7612 typedef VALUE (*rb_invoke_funcptr2_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2);
7613 return (*(rb_invoke_funcptr2_t)funcptr)(ec, self, argv[0], argv[1]);
7614}
7615
7616static VALUE
7617builtin_invoker3(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7618{
7619 typedef VALUE (*rb_invoke_funcptr3_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3);
7620 return (*(rb_invoke_funcptr3_t)funcptr)(ec, self, argv[0], argv[1], argv[2]);
7621}
7622
7623static VALUE
7624builtin_invoker4(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7625{
7626 typedef VALUE (*rb_invoke_funcptr4_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4);
7627 return (*(rb_invoke_funcptr4_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3]);
7628}
7629
7630static VALUE
7631builtin_invoker5(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7632{
7633 typedef VALUE (*rb_invoke_funcptr5_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5);
7634 return (*(rb_invoke_funcptr5_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4]);
7635}
7636
7637static VALUE
7638builtin_invoker6(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7639{
7640 typedef VALUE (*rb_invoke_funcptr6_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6);
7641 return (*(rb_invoke_funcptr6_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
7642}
7643
7644static VALUE
7645builtin_invoker7(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7646{
7647 typedef VALUE (*rb_invoke_funcptr7_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7);
7648 return (*(rb_invoke_funcptr7_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
7649}
7650
7651static VALUE
7652builtin_invoker8(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7653{
7654 typedef VALUE (*rb_invoke_funcptr8_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8);
7655 return (*(rb_invoke_funcptr8_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
7656}
7657
7658static VALUE
7659builtin_invoker9(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7660{
7661 typedef VALUE (*rb_invoke_funcptr9_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9);
7662 return (*(rb_invoke_funcptr9_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
7663}
7664
7665static VALUE
7666builtin_invoker10(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7667{
7668 typedef VALUE (*rb_invoke_funcptr10_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10);
7669 return (*(rb_invoke_funcptr10_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
7670}
7671
7672static VALUE
7673builtin_invoker11(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7674{
7675 typedef VALUE (*rb_invoke_funcptr11_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11);
7676 return (*(rb_invoke_funcptr11_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
7677}
7678
7679static VALUE
7680builtin_invoker12(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7681{
7682 typedef VALUE (*rb_invoke_funcptr12_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12);
7683 return (*(rb_invoke_funcptr12_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
7684}
7685
7686static VALUE
7687builtin_invoker13(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7688{
7689 typedef VALUE (*rb_invoke_funcptr13_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13);
7690 return (*(rb_invoke_funcptr13_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
7691}
7692
7693static VALUE
7694builtin_invoker14(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7695{
7696 typedef VALUE (*rb_invoke_funcptr14_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13, VALUE v14);
7697 return (*(rb_invoke_funcptr14_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
7698}
7699
7700static VALUE
7701builtin_invoker15(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7702{
7703 typedef VALUE (*rb_invoke_funcptr15_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13, VALUE v14, VALUE v15);
7704 return (*(rb_invoke_funcptr15_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
7705}
7706
7707typedef VALUE (*builtin_invoker)(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr);
7708
7709static builtin_invoker
7710lookup_builtin_invoker(int argc)
7711{
7712 static const builtin_invoker invokers[] = {
7713 builtin_invoker0,
7714 builtin_invoker1,
7715 builtin_invoker2,
7716 builtin_invoker3,
7717 builtin_invoker4,
7718 builtin_invoker5,
7719 builtin_invoker6,
7720 builtin_invoker7,
7721 builtin_invoker8,
7722 builtin_invoker9,
7723 builtin_invoker10,
7724 builtin_invoker11,
7725 builtin_invoker12,
7726 builtin_invoker13,
7727 builtin_invoker14,
7728 builtin_invoker15,
7729 };
7730
7731 return invokers[argc];
7732}
7733
7734static inline VALUE
7735invoke_bf(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const struct rb_builtin_function* bf, const VALUE *argv)
7736{
7737 const bool canary_p = ISEQ_BODY(CFP_ISEQ(reg_cfp))->builtin_attrs & BUILTIN_ATTR_LEAF; // Verify an assumption of `Primitive.attr! :leaf`
7738 SETUP_CANARY(canary_p);
7739 rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr;
7740 VALUE ret = (*lookup_builtin_invoker(bf->argc))(ec, reg_cfp->self, argv, func_ptr);
7741 CHECK_CANARY(canary_p, BIN(invokebuiltin));
7742 return ret;
7743}
7744
7745static VALUE
7746vm_invoke_builtin(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function* bf, const VALUE *argv)
7747{
7748 return invoke_bf(ec, cfp, bf, argv);
7749}
7750
7751static VALUE
7752vm_invoke_builtin_delegate(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function *bf, unsigned int start_index)
7753{
7754 if (0) { // debug print
7755 fputs("vm_invoke_builtin_delegate: passing -> ", stderr);
7756 for (int i=0; i<bf->argc; i++) {
7757 ruby_debug_printf(":%s ", rb_id2name(ISEQ_BODY(CFP_ISEQ(cfp))->local_table[i+start_index]));
7758 }
7759 ruby_debug_printf("\n" "%s %s(%d):%p\n", RUBY_FUNCTION_NAME_STRING, bf->name, bf->argc,
7760 (void *)(uintptr_t)bf->func_ptr);
7761 }
7762
7763 if (bf->argc == 0) {
7764 return invoke_bf(ec, cfp, bf, NULL);
7765 }
7766 else {
7767 const VALUE *argv = cfp->ep - ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size - VM_ENV_DATA_SIZE + 1 + start_index;
7768 return invoke_bf(ec, cfp, bf, argv);
7769 }
7770}
7771
7772// for __builtin_inline!()
7773
7774VALUE
7775rb_vm_lvar_exposed(rb_execution_context_t *ec, int index)
7776{
7777 const rb_control_frame_t *cfp = ec->cfp;
7778 return cfp->ep[index];
7779}
RUBY_ASSERT
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:219
RUBY_EVENT_END
#define RUBY_EVENT_END
Encountered an end of a class clause.
Definition event.h:40
RUBY_EVENT_C_CALL
#define RUBY_EVENT_C_CALL
A method, written in C, is called.
Definition event.h:43
RUBY_EVENT_B_RETURN
#define RUBY_EVENT_B_RETURN
Encountered a next statement.
Definition event.h:56
RUBY_EVENT_CLASS
#define RUBY_EVENT_CLASS
Encountered a new class.
Definition event.h:39
RUBY_EVENT_LINE
#define RUBY_EVENT_LINE
Encountered a new line.
Definition event.h:38
RUBY_EVENT_RETURN
#define RUBY_EVENT_RETURN
Encountered a return statement.
Definition event.h:42
RUBY_EVENT_C_RETURN
#define RUBY_EVENT_C_RETURN
Return from a method, written in C.
Definition event.h:44
RUBY_EVENT_B_CALL
#define RUBY_EVENT_B_CALL
Encountered an yield statement.
Definition event.h:55
rb_event_flag_t
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
RUBY_EVENT_CALL
#define RUBY_EVENT_CALL
A method, written in Ruby, is called.
Definition event.h:41
RUBY_EVENT_RESCUE
#define RUBY_EVENT_RESCUE
Encountered a rescue statement.
Definition event.h:61
RB_FL_TEST_RAW
static VALUE RB_FL_TEST_RAW(VALUE obj, VALUE flags)
This is an implementation detail of RB_FL_TEST().
Definition fl_type.h:404
RB_FL_ABLE
static bool RB_FL_ABLE(VALUE obj)
Checks if the object is flaggable.
Definition fl_type.h:381
RUBY_FL_SHAREABLE
@ RUBY_FL_SHAREABLE
This flag has something to do with Ractor.
Definition fl_type.h:253
rb_singleton_class
VALUE rb_singleton_class(VALUE obj)
Finds or creates the singleton class of the passed object.
Definition class.c:2870
rb_module_new
VALUE rb_module_new(void)
Creates a new, anonymous module.
Definition class.c:1501
rb_class_inherited
VALUE rb_class_inherited(VALUE super, VALUE klass)
Calls Class::inherited.
Definition class.c:1401
rb_define_class_id
VALUE rb_define_class_id(ID id, VALUE super)
This is a very badly designed API that creates an anonymous class.
Definition class.c:1380
TYPE
#define TYPE(_)
Old name of rb_type.
Definition value_type.h:108
RFLOAT_VALUE
#define RFLOAT_VALUE
Old name of rb_float_value.
Definition double.h:28
T_STRING
#define T_STRING
Old name of RUBY_T_STRING.
Definition value_type.h:78
Qundef
#define Qundef
Old name of RUBY_Qundef.
Definition special_consts.h:62
INT2FIX
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
T_NIL
#define T_NIL
Old name of RUBY_T_NIL.
Definition value_type.h:72
T_FLOAT
#define T_FLOAT
Old name of RUBY_T_FLOAT.
Definition value_type.h:64
T_IMEMO
#define T_IMEMO
Old name of RUBY_T_IMEMO.
Definition value_type.h:67
ID2SYM
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
T_BIGNUM
#define T_BIGNUM
Old name of RUBY_T_BIGNUM.
Definition value_type.h:57
SPECIAL_CONST_P
#define SPECIAL_CONST_P
Old name of RB_SPECIAL_CONST_P.
Definition special_consts.h:56
T_STRUCT
#define T_STRUCT
Old name of RUBY_T_STRUCT.
Definition value_type.h:79
T_FIXNUM
#define T_FIXNUM
Old name of RUBY_T_FIXNUM.
Definition value_type.h:63
UNREACHABLE_RETURN
#define UNREACHABLE_RETURN
Old name of RBIMPL_UNREACHABLE_RETURN.
Definition assume.h:29
SYM2ID
#define SYM2ID
Old name of RB_SYM2ID.
Definition symbol.h:45
CLASS_OF
#define CLASS_OF
Old name of rb_class_of.
Definition globals.h:205
T_NONE
#define T_NONE
Old name of RUBY_T_NONE.
Definition value_type.h:74
rb_ary_new4
#define rb_ary_new4
Old name of rb_ary_new_from_values.
Definition array.h:659
FIXABLE
#define FIXABLE
Old name of RB_FIXABLE.
Definition fixnum.h:25
LONG2FIX
#define LONG2FIX
Old name of RB_INT2FIX.
Definition long.h:49
FIX2INT
#define FIX2INT
Old name of RB_FIX2INT.
Definition int.h:41
T_MODULE
#define T_MODULE
Old name of RUBY_T_MODULE.
Definition value_type.h:70
STATIC_SYM_P
#define STATIC_SYM_P
Old name of RB_STATIC_SYM_P.
Definition special_consts.h:57
ASSUME
#define ASSUME
Old name of RBIMPL_ASSUME.
Definition assume.h:27
FIX2ULONG
#define FIX2ULONG
Old name of RB_FIX2ULONG.
Definition long.h:47
T_TRUE
#define T_TRUE
Old name of RUBY_T_TRUE.
Definition value_type.h:81
T_ICLASS
#define T_ICLASS
Old name of RUBY_T_ICLASS.
Definition value_type.h:66
T_HASH
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
rb_ary_new3
#define rb_ary_new3
Old name of rb_ary_new_from_args.
Definition array.h:658
LONG2NUM
#define LONG2NUM
Old name of RB_LONG2NUM.
Definition long.h:50
rb_exc_new3
#define rb_exc_new3
Old name of rb_exc_new_str.
Definition error.h:38
T_FALSE
#define T_FALSE
Old name of RUBY_T_FALSE.
Definition value_type.h:61
Qtrue
#define Qtrue
Old name of RUBY_Qtrue.
Definition special_consts.h:61
Qnil
#define Qnil
Old name of RUBY_Qnil.
Definition special_consts.h:60
Qfalse
#define Qfalse
Old name of RUBY_Qfalse.
Definition special_consts.h:59
FIX2LONG
#define FIX2LONG
Old name of RB_FIX2LONG.
Definition long.h:46
T_ARRAY
#define T_ARRAY
Old name of RUBY_T_ARRAY.
Definition value_type.h:56
T_OBJECT
#define T_OBJECT
Old name of RUBY_T_OBJECT.
Definition value_type.h:75
NIL_P
#define NIL_P
Old name of RB_NIL_P.
Definition special_consts.h:55
T_SYMBOL
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
Definition value_type.h:80
DBL2NUM
#define DBL2NUM
Old name of rb_float_new.
Definition double.h:29
T_CLASS
#define T_CLASS
Old name of RUBY_T_CLASS.
Definition value_type.h:58
BUILTIN_TYPE
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
Definition value_type.h:85
FL_TEST
#define FL_TEST
Old name of RB_FL_TEST.
Definition fl_type.h:127
FIXNUM_P
#define FIXNUM_P
Old name of RB_FIXNUM_P.
Definition special_consts.h:53
FL_USHIFT
#define FL_USHIFT
Old name of RUBY_FL_USHIFT.
Definition fl_type.h:67
FL_SET_RAW
#define FL_SET_RAW
Old name of RB_FL_SET_RAW.
Definition fl_type.h:126
SYMBOL_P
#define SYMBOL_P
Old name of RB_SYMBOL_P.
Definition value_type.h:88
rb_exc_raise
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
Definition eval.c:676
ruby_verbose
#define ruby_verbose
This variable controls whether the interpreter is in debug mode.
Definition error.h:476
rb_eTypeError
VALUE rb_eTypeError
TypeError exception.
Definition error.c:1431
rb_eFatal
VALUE rb_eFatal
fatal exception.
Definition error.c:1427
rb_eNoMethodError
VALUE rb_eNoMethodError
NoMethodError exception.
Definition error.c:1439
rb_exc_fatal
void rb_exc_fatal(VALUE mesg)
Raises a fatal error in the current thread.
Definition eval.c:689
rb_eRuntimeError
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1429
rb_warn
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:468
rb_error_frozen_object
void rb_error_frozen_object(VALUE frozen_obj)
Identical to rb_error_frozen(), except it takes arbitrary Ruby object instead of C's string.
Definition error.c:4294
rb_exc_new_str
VALUE rb_exc_new_str(VALUE etype, VALUE str)
Identical to rb_exc_new_cstr(), except it takes a Ruby's string instead of C's.
Definition error.c:1482
RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK
@ RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK
Warning is for checking unused block strictly.
Definition error.h:57
rb_cClass
VALUE rb_cClass
Class class.
Definition object.c:62
rb_cArray
VALUE rb_cArray
Array class.
rb_cObject
VALUE rb_cObject
Object class.
Definition object.c:60
rb_obj_alloc
VALUE rb_obj_alloc(VALUE klass)
Allocates an instance of the given class.
Definition object.c:2250
rb_cRegexp
VALUE rb_cRegexp
Regexp class.
Definition re.c:2828
rb_obj_frozen_p
VALUE rb_obj_frozen_p(VALUE obj)
Just calls RB_OBJ_FROZEN() inside.
Definition object.c:1320
rb_cHash
VALUE rb_cHash
Hash class.
Definition hash.c:122
rb_obj_class
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
Definition object.c:234
rb_inspect
VALUE rb_inspect(VALUE obj)
Generates a human-readable textual representation of the given object.
Definition object.c:668
rb_cBasicObject
VALUE rb_cBasicObject
BasicObject class.
Definition object.c:58
rb_cModule
VALUE rb_cModule
Module class.
Definition object.c:61
rb_class_real
VALUE rb_class_real(VALUE klass)
Finds a "real" class.
Definition object.c:225
rb_obj_is_kind_of
VALUE rb_obj_is_kind_of(VALUE obj, VALUE klass)
Queries if the given object is an instance (of possibly descendants) of the given class.
Definition object.c:905
rb_cFloat
VALUE rb_cFloat
Float class.
Definition numeric.c:201
rb_cProc
VALUE rb_cProc
Proc class.
Definition proc.c:46
rb_cString
VALUE rb_cString
String class.
Definition string.c:85
RB_OBJ_WRITTEN
#define RB_OBJ_WRITTEN(old, oldv, young)
Identical to RB_OBJ_WRITE(), except it doesn't write any values, but only a WB declaration.
Definition gc.h:468
RB_OBJ_WRITE
#define RB_OBJ_WRITE(old, slot, young)
Declaration of a "back" pointer.
Definition gc.h:456
rb_ary_concat
VALUE rb_ary_concat(VALUE lhs, VALUE rhs)
Destructively appends the contents of latter into the end of former.
rb_ary_shift
VALUE rb_ary_shift(VALUE ary)
Destructively deletes an element from the beginning of the passed array and returns what was deleted.
rb_ary_resurrect
VALUE rb_ary_resurrect(VALUE ary)
I guess there is no use case of this function in extension libraries, but this is a routine identical...
rb_ary_dup
VALUE rb_ary_dup(VALUE ary)
Duplicates an array.
rb_ary_includes
VALUE rb_ary_includes(VALUE ary, VALUE elem)
Queries if the passed array has the passed entry.
rb_ary_plus
VALUE rb_ary_plus(VALUE lhs, VALUE rhs)
Creates a new array, concatenating the former to the latter.
rb_ary_cat
VALUE rb_ary_cat(VALUE ary, const VALUE *train, long len)
Destructively appends multiple elements at the end of the array.
rb_check_array_type
VALUE rb_check_array_type(VALUE obj)
Try converting an object to its array representation using its to_ary method, if any.
rb_ary_new
VALUE rb_ary_new(void)
Allocates a new, empty array.
rb_ary_pop
VALUE rb_ary_pop(VALUE ary)
Destructively deletes an element from the end of the passed array and returns what was deleted.
rb_ary_hidden_new
VALUE rb_ary_hidden_new(long capa)
Allocates a hidden (no class) empty array.
rb_ary_push
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
rb_ary_entry
VALUE rb_ary_entry(VALUE ary, long off)
Queries an element of an array.
rb_ary_store
void rb_ary_store(VALUE ary, long key, VALUE val)
Destructively stores the passed value to the passed array's passed index.
UNLIMITED_ARGUMENTS
#define UNLIMITED_ARGUMENTS
This macro is used in conjunction with rb_check_arity().
Definition error.h:35
rb_check_arity
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
Definition error.h:284
rb_proc_call_with_block
VALUE rb_proc_call_with_block(VALUE recv, int argc, const VALUE *argv, VALUE proc)
Identical to rb_proc_call(), except you can additionally pass another proc object,...
Definition proc.c:1763
rb_reg_last_match
VALUE rb_reg_last_match(VALUE md)
This just returns the argument, stringified.
Definition re.c:2095
rb_reg_match
VALUE rb_reg_match(VALUE re, VALUE str)
This is the match operator.
Definition re.c:3970
rb_reg_nth_match
VALUE rb_reg_nth_match(int n, VALUE md)
Queries the nth captured substring.
Definition re.c:2071
rb_reg_match_post
VALUE rb_reg_match_post(VALUE md)
The portion of the original string after the given match.
Definition re.c:2150
rb_reg_nth_defined
VALUE rb_reg_nth_defined(int n, VALUE md)
Identical to rb_reg_nth_match(), except it just returns Boolean.
Definition re.c:2055
rb_reg_match_pre
VALUE rb_reg_match_pre(VALUE md)
The portion of the original string before the given match.
Definition re.c:2119
rb_reg_match_last
VALUE rb_reg_match_last(VALUE md)
The portion of the original string that captured at the very last.
Definition re.c:2179
rb_str_append
VALUE rb_str_append(VALUE dst, VALUE src)
Identical to rb_str_buf_append(), except it converts the right hand side before concatenating.
Definition string.c:3898
rb_str_succ
VALUE rb_str_succ(VALUE orig)
Searches for the "successor" of a string.
Definition string.c:5437
rb_str_buf_append
VALUE rb_str_buf_append(VALUE dst, VALUE src)
Identical to rb_str_cat_cstr(), except it takes Ruby's string instead of C's.
Definition string.c:3864
rb_str_concat
VALUE rb_str_concat(VALUE dst, VALUE src)
Identical to rb_str_append(), except it also accepts an integer as a codepoint.
Definition string.c:4135
rb_str_cat_cstr
#define rb_str_cat_cstr(buf, str)
Identical to rb_str_cat(), except it assumes the passed pointer is a pointer to a C string.
Definition string.h:1657
rb_str_length
VALUE rb_str_length(VALUE)
Identical to rb_str_strlen(), except it returns the value in rb_cInteger.
Definition string.c:2491
rb_str_intern
VALUE rb_str_intern(VALUE str)
Identical to rb_to_symbol(), except it assumes the receiver being an instance of RString.
Definition symbol.c:1085
rb_const_get
VALUE rb_const_get(VALUE space, ID name)
Identical to rb_const_defined(), except it returns the actual defined value.
Definition variable.c:3430
rb_ivar_set
VALUE rb_ivar_set(VALUE obj, ID name, VALUE val)
Identical to rb_iv_set(), except it accepts the name as an ID instead of a C string.
Definition variable.c:2060
rb_cvar_set
void rb_cvar_set(VALUE klass, ID name, VALUE val)
Assigns a value to a class variable.
Definition variable.c:4217
rb_cvar_find
VALUE rb_cvar_find(VALUE klass, ID name, VALUE *front)
Identical to rb_cvar_get(), except it takes additional "front" pointer.
Definition variable.c:4278
rb_ivar_get
VALUE rb_ivar_get(VALUE obj, ID name)
Identical to rb_iv_get(), except it accepts the name as an ID instead of a C string.
Definition variable.c:1579
rb_const_set
void rb_const_set(VALUE space, ID name, VALUE val)
Names a constant.
Definition variable.c:3908
rb_autoload_load
VALUE rb_autoload_load(VALUE space, ID name)
Kicks the autoload procedure as if it was "touched".
Definition variable.c:3260
rb_mod_name
VALUE rb_mod_name(VALUE mod)
Queries the name of a module.
Definition variable.c:151
rb_const_get_at
VALUE rb_const_get_at(VALUE space, ID name)
Identical to rb_const_defined_at(), except it returns the actual defined value.
Definition variable.c:3436
rb_set_class_path_string
void rb_set_class_path_string(VALUE klass, VALUE space, VALUE name)
Identical to rb_set_class_path(), except it accepts the name as Ruby's string instead of C's.
Definition variable.c:438
rb_ivar_defined
VALUE rb_ivar_defined(VALUE obj, ID name)
Queries if the instance variable is defined at the object.
Definition variable.c:2120
rb_const_defined_at
int rb_const_defined_at(VALUE space, ID name)
Identical to rb_const_defined(), except it doesn't look for parent classes.
Definition variable.c:3768
rb_cvar_defined
VALUE rb_cvar_defined(VALUE klass, ID name)
Queries if the given class has the given class variable.
Definition variable.c:4301
rb_class_path
VALUE rb_class_path(VALUE mod)
Identical to rb_mod_name(), except it returns #<Class: ...> style inspection for anonymous modules.
Definition variable.c:395
rb_const_defined
int rb_const_defined(VALUE space, ID name)
Queries if the constant is defined at the namespace.
Definition variable.c:3762
rb_method_basic_definition_p
int rb_method_basic_definition_p(VALUE klass, ID mid)
Well... Let us hesitate from describing what a "basic definition" is.
Definition vm_method.c:3468
rb_check_funcall
VALUE rb_check_funcall(VALUE recv, ID mid, int argc, const VALUE *argv)
Identical to rb_funcallv(), except it returns RUBY_Qundef instead of raising rb_eNoMethodError.
Definition vm_eval.c:691
rb_get_alloc_func
rb_alloc_func_t rb_get_alloc_func(VALUE klass)
Queries the allocator function of a class.
Definition vm_method.c:1818
rb_method_boundp
int rb_method_boundp(VALUE klass, ID id, int ex)
Queries if the klass has this method.
Definition vm_method.c:2428
rb_check_id
ID rb_check_id(volatile VALUE *namep)
Detects if the given name is already interned or not.
Definition symbol.c:1288
rb_sym2str
VALUE rb_sym2str(VALUE symbol)
Obtain a frozen string representation of a symbol (not including the leading colon).
Definition symbol.c:1148
off
int off
Offset inside of ptr.
Definition io.h:5
len
int len
Length of the buffer.
Definition io.h:8
rb_ractor_shareable_p
static bool rb_ractor_shareable_p(VALUE obj)
Queries if multiple Ractors can share the passed object or not.
Definition ractor.h:249
RB_OBJ_SHAREABLE_P
#define RB_OBJ_SHAREABLE_P(obj)
Queries if the passed object has previously classified as shareable or not.
Definition ractor.h:235
RB_NOGVL_UBF_ASYNC_SAFE
#define RB_NOGVL_UBF_ASYNC_SAFE
Passing this flag to rb_nogvl() indicates that the passed UBF is async-signal-safe.
Definition thread.h:71
RB_NOGVL_INTR_FAIL
#define RB_NOGVL_INTR_FAIL
Passing this flag to rb_nogvl() prevents it from processing interrupts after the given function retur...
Definition thread.h:50
rb_nogvl
void * rb_nogvl(void *(*func)(void *), void *data1, rb_unblock_function_t *ubf, void *data2, int flags)
Identical to rb_thread_call_without_gvl(), except it additionally takes "flags" that change the behav...
Definition thread.c:1767
MEMCPY
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition memory.h:372
ALLOCA_N
#define ALLOCA_N(type, n)
Definition memory.h:292
RB_GC_GUARD
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:167
MEMMOVE
#define MEMMOVE(p1, p2, type, n)
Handy macro to call memmove.
Definition memory.h:384
ruby::backward::cxxanyargs::type
VALUE type(ANYARGS)
ANYARGS-ed function type.
Definition cxxanyargs.hpp:56
RARRAY_LEN
#define RARRAY_LEN
Just another name of rb_array_len.
Definition rarray.h:50
RARRAY_LENINT
static int RARRAY_LENINT(VALUE ary)
Identical to rb_array_len(), except it differs for the return type.
Definition rarray.h:280
RARRAY_PTR
static VALUE * RARRAY_PTR(VALUE ary)
Wild use of a C pointer.
Definition rarray.h:365
RARRAY_AREF
#define RARRAY_AREF(a, i)
Definition rarray.h:402
RARRAY_CONST_PTR
#define RARRAY_CONST_PTR
Just another name of rb_array_const_ptr.
Definition rarray.h:51
RBASIC_CLASS
static VALUE RBASIC_CLASS(VALUE obj)
Queries the class of an object.
Definition rbasic.h:166
RBASIC
#define RBASIC(obj)
Convenient casting macro.
Definition rbasic.h:40
RCLASS_SUPER
#define RCLASS_SUPER
Just another name of rb_class_get_superclass.
Definition rclass.h:44
RHASH_SIZE
#define RHASH_SIZE(h)
Queries the size of the hash.
Definition rhash.h:69
RHASH_EMPTY_P
#define RHASH_EMPTY_P(h)
Checks if the hash is empty.
Definition rhash.h:79
RSTRING_LENINT
static int RSTRING_LENINT(VALUE str)
Identical to RSTRING_LEN(), except it differs for the return type.
Definition rstring.h:438
StringValueCStr
#define StringValueCStr(v)
Identical to StringValuePtr, except it additionally checks for the contents for viability as a C stri...
Definition rstring.h:89
RB_PASS_KEYWORDS
#define RB_PASS_KEYWORDS
Pass keywords, final argument must be a hash of keywords.
Definition scan_args.h:72
RB_NO_KEYWORDS
#define RB_NO_KEYWORDS
Do not pass keywords.
Definition scan_args.h:69
RB_SPECIAL_CONST_P
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
Definition special_consts.h:327
RTEST
#define RTEST
This is an old name of RB_TEST.
Definition special_consts.h:51
ANYARGS
#define ANYARGS
Functions declared using this macro take arbitrary arguments, including void.
Definition stdarg.h:64
RArray
Ruby's array.
Definition rarray.h:127
RArray::ary
const VALUE ary[1]
Embedded elements.
Definition rarray.h:187
RArray::ptr
const VALUE * ptr
Pointer to the C array that holds the elements of the array.
Definition rarray.h:174
RHash
Definition hash.h:54
iseq_catch_table_entry
Definition iseq.h:368
iseq_catch_table
Definition iseq.h:379
iseq_inline_constant_cache_entry
Definition vm_core.h:261
iseq_inline_constant_cache
Definition vm_core.h:271
iseq_inline_constant_cache::segments
const ID * segments
A null-terminated list of ids, used to represent a constant's path idNULL is used to represent the ::...
Definition vm_core.h:285
iseq_inline_cvar_cache_entry
Definition vm_core.h:293
iseq_inline_iv_cache_entry
Definition vm_core.h:288
rb_block
Definition vm_core.h:943
rb_box_struct
Internal header for Ruby Box.
Definition box.h:14
rb_builtin_function
Definition builtin.h:6
rb_call_data
Definition vm_callinfo.h:606
rb_callable_method_entry_struct
Definition method.h:63
rb_callcache
Definition vm_callinfo.h:292
rb_callinfo_kwarg
Definition vm_callinfo.h:48
rb_callinfo
Definition vm_callinfo.h:79
rb_calling_info
Definition vm_core.h:306
rb_captured_block
Definition vm_core.h:919
rb_class_cc_entries
Definition vm_callinfo.h:611
rb_const_entry_struct
Definition constant.h:33
rb_control_frame_struct
Definition vm_core.h:952
rb_cref_struct
CREF (Class REFerence)
Definition method.h:45
rb_cvar_class_tbl_entry
Internal header for Class.
Definition class.h:30
rb_env_t
Definition vm_core.h:1400
rb_execution_context_struct
Definition vm_core.h:1086
rb_forwarding_call_data
Definition vm_insnhelper.h:69
rb_getivar_cache
Definition shape.h:594
rb_hook_list_struct
Definition vm_core.h:682
rb_id_table
Definition id_table.h:7
rb_iseq_constant_body
Definition vm_core.h:428
rb_iseq_struct
Definition vm_core.h:593
rb_method_cfunc_struct
Definition method.h:148
rb_method_definition_struct
Definition method.h:187
rb_method_entry_struct
Definition method.h:55
rb_method_iseq_struct::cref
rb_cref_t * cref
class reference, should be marked
Definition method.h:144
rb_method_iseq_struct::iseqptr
const rb_iseq_t * iseqptr
iseq pointer, should be separated from iseqval
Definition method.h:143
rb_ractor_struct
Definition ractor_core.h:76
rb_setivar_cache
Definition shape.h:636
rb_thread_struct
Definition vm_core.h:1183
rb_vm_struct
Definition vm_core.h:707
set_table
Definition set_table.h:10
st_table
Definition st.h:79
vm_ifunc
IFUNC (Internal FUNCtion)
Definition imemo.h:87
vm_once_wait_arg
Definition vm_insnhelper.c:5644
vm_svar
SVAR (Special VARiable)
Definition imemo.h:52
vm_svar::cref_or_me
const VALUE cref_or_me
class reference or rb_method_entry_t
Definition imemo.h:54
vm_throw_data
THROW_DATA.
Definition imemo.h:61
rb_native_mutex_lock
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
rb_native_cond_broadcast
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
Signals a condition variable.
rb_native_mutex_unlock
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
rb_native_cond_wait
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
Waits for the passed condition variable to be signalled.
__attribute__
Definition vm_insnhelper.c:2394
iseq_inline_storage_entry
Definition vm_core.h:297
rb_proc_t
Definition vm_core.h:1370
SIGNED_VALUE
intptr_t SIGNED_VALUE
A signed integer type that has the same width with VALUE.
Definition value.h:63
ID
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
SIZEOF_VALUE
#define SIZEOF_VALUE
Identical to sizeof(VALUE), except it is a macro that can also be used inside of preprocessor directi...
Definition value.h:69
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
RB_BUILTIN_TYPE
static enum ruby_value_type RB_BUILTIN_TYPE(VALUE obj)
Queries the type of the object.
Definition value_type.h:182
RB_FLOAT_TYPE_P
static bool RB_FLOAT_TYPE_P(VALUE obj)
Queries if the object is an instance of rb_cFloat.
Definition value_type.h:264
RB_TYPE_P
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition value_type.h:376
Generated by doxygen 1.9.8