1#ifndef RUBY_VM_CALLINFO_H
2#define RUBY_VM_CALLINFO_H
11#include "debug_counter.h"
12#include "internal/class.h"
15enum vm_call_flag_bits {
16 VM_CALL_ARGS_SPLAT_bit,
17 VM_CALL_ARGS_BLOCKARG_bit,
20 VM_CALL_ARGS_SIMPLE_bit,
27 VM_CALL_KW_SPLAT_MUT_bit,
28 VM_CALL_ARGS_SPLAT_MUT_bit,
29 VM_CALL_FORWARDING_bit,
30 VM_CALL_IGNORE_REFINEMENTS_bit,
34#define VM_CALL_ARGS_SPLAT (0x01 << VM_CALL_ARGS_SPLAT_bit)
35#define VM_CALL_ARGS_BLOCKARG (0x01 << VM_CALL_ARGS_BLOCKARG_bit)
36#define VM_CALL_FCALL (0x01 << VM_CALL_FCALL_bit)
37#define VM_CALL_VCALL (0x01 << VM_CALL_VCALL_bit)
38#define VM_CALL_ARGS_SIMPLE (0x01 << VM_CALL_ARGS_SIMPLE_bit)
39#define VM_CALL_KWARG (0x01 << VM_CALL_KWARG_bit)
40#define VM_CALL_KW_SPLAT (0x01 << VM_CALL_KW_SPLAT_bit)
41#define VM_CALL_TAILCALL (0x01 << VM_CALL_TAILCALL_bit)
42#define VM_CALL_SUPER (0x01 << VM_CALL_SUPER_bit)
43#define VM_CALL_ZSUPER (0x01 << VM_CALL_ZSUPER_bit)
44#define VM_CALL_OPT_SEND (0x01 << VM_CALL_OPT_SEND_bit)
45#define VM_CALL_KW_SPLAT_MUT (0x01 << VM_CALL_KW_SPLAT_MUT_bit)
46#define VM_CALL_ARGS_SPLAT_MUT (0x01 << VM_CALL_ARGS_SPLAT_MUT_bit)
47#define VM_CALL_FORWARDING (0x01 << VM_CALL_FORWARDING_bit)
48#define VM_CALL_IGNORE_REFINEMENTS (0x01 << VM_CALL_IGNORE_REFINEMENTS_bit)
57rb_callinfo_kwarg_bytes(
int keyword_len)
59 return rb_size_mul_add_or_raise(
76 ruby_xfree_sized(kwarg, rb_callinfo_kwarg_bytes(kwarg->keyword_len));
89#if !defined(USE_EMBED_CI) || (USE_EMBED_CI+0)
98#define CI_EMBED_TAG_bits 1
99#define CI_EMBED_ARGC_bits 15
100#define CI_EMBED_FLAG_bits 16
101#define CI_EMBED_ID_bits 32
102#elif SIZEOF_VALUE == 4
103#define CI_EMBED_TAG_bits 1
104#define CI_EMBED_ARGC_bits 3
105#define CI_EMBED_FLAG_bits 13
106#define CI_EMBED_ID_bits 15
109#if (CI_EMBED_TAG_bits + CI_EMBED_ARGC_bits + CI_EMBED_FLAG_bits + CI_EMBED_ID_bits) != (SIZEOF_VALUE * 8)
113#define CI_EMBED_FLAG 0x01
114#define CI_EMBED_ARGC_SHFT (CI_EMBED_TAG_bits)
115#define CI_EMBED_ARGC_MASK ((((VALUE)1)<<CI_EMBED_ARGC_bits) - 1)
116#define CI_EMBED_FLAG_SHFT (CI_EMBED_TAG_bits + CI_EMBED_ARGC_bits)
117#define CI_EMBED_FLAG_MASK ((((VALUE)1)<<CI_EMBED_FLAG_bits) - 1)
118#define CI_EMBED_ID_SHFT (CI_EMBED_TAG_bits + CI_EMBED_ARGC_bits + CI_EMBED_FLAG_bits)
119#define CI_EMBED_ID_MASK ((((VALUE)1)<<CI_EMBED_ID_bits) - 1)
127 if (LIKELY(((
VALUE)ci) & 0x01)) {
131 VM_ASSERT(IMEMO_TYPE_P(ci, imemo_callinfo));
139 if (vm_ci_packed_p(ci) || IMEMO_TYPE_P(ci, imemo_callinfo)) {
150 if (vm_ci_packed_p(ci)) {
151 return (((
VALUE)ci) >> CI_EMBED_ID_SHFT) & CI_EMBED_ID_MASK;
158static inline unsigned int
161 if (vm_ci_packed_p(ci)) {
162 return (
unsigned int)((((
VALUE)ci) >> CI_EMBED_FLAG_SHFT) & CI_EMBED_FLAG_MASK);
169static inline unsigned int
172 if (vm_ci_packed_p(ci)) {
173 return (
unsigned int)((((
VALUE)ci) >> CI_EMBED_ARGC_SHFT) & CI_EMBED_ARGC_MASK);
183 if (vm_ci_packed_p(ci)) {
194 if (vm_ci_packed_p(ci)) {
195 ruby_debug_printf(
"packed_ci ID:%s flag:%x argc:%u\n",
196 rb_id2name(vm_ci_mid(ci)), vm_ci_flag(ci), vm_ci_argc(ci));
203#define vm_ci_new(mid, flag, argc, kwarg) vm_ci_new_(mid, flag, argc, kwarg, __FILE__, __LINE__)
204#define vm_ci_new_runtime(mid, flag, argc, kwarg) vm_ci_new_runtime_(mid, flag, argc, kwarg, __FILE__, __LINE__)
207#define VM_CI_EMBEDDABLE_P(mid, flag, argc, kwarg) \
208 (((mid ) & ~CI_EMBED_ID_MASK) ? false : \
209 ((flag) & ~CI_EMBED_FLAG_MASK) ? false : \
210 ((argc) & ~CI_EMBED_ARGC_MASK) ? false : \
211 (kwarg) ? false : true)
213#define vm_ci_new_id(mid, flag, argc, must_zero) \
214 ((const struct rb_callinfo *) \
215 ((((VALUE)(mid )) << CI_EMBED_ID_SHFT) | \
216 (((VALUE)(flag)) << CI_EMBED_FLAG_SHFT) | \
217 (((VALUE)(argc)) << CI_EMBED_ARGC_SHFT) | \
225vm_ci_new_(
ID mid,
unsigned int flag,
unsigned int argc,
const struct rb_callinfo_kwarg *kwarg,
const char *file,
int line)
227 if (USE_EMBED_CI && VM_CI_EMBEDDABLE_P(mid, flag, argc, kwarg)) {
228 RB_DEBUG_COUNTER_INC(ci_packed);
229 return vm_ci_new_id(mid, flag, argc, kwarg);
232 const bool debug = 0;
233 if (debug) ruby_debug_printf(
"%s:%d ", file, line);
235 const struct rb_callinfo *ci = rb_vm_ci_lookup(mid, flag, argc, kwarg);
239 RB_DEBUG_COUNTER_INC(ci_kw);
242 RB_DEBUG_COUNTER_INC(ci_nokw);
245 VM_ASSERT(vm_ci_flag(ci) == flag);
246 VM_ASSERT(vm_ci_argc(ci) == argc);
253vm_ci_new_runtime_(
ID mid,
unsigned int flag,
unsigned int argc,
const struct rb_callinfo_kwarg *kwarg,
const char *file,
int line)
255 RB_DEBUG_COUNTER_INC(ci_runtime);
256 return vm_ci_new_(mid, flag, argc, kwarg, file, line);
259#define VM_CALLINFO_NOT_UNDER_GC IMEMO_FL_USER0
267 else if (vm_ci_packed_p(ci)) {
271 VM_ASSERT(IMEMO_TYPE_P(ci, imemo_callinfo));
276#define VM_CI_ON_STACK(mid_, flags_, argc_, kwarg_) \
277 (struct rb_callinfo) { \
279 (imemo_callinfo << FL_USHIFT) | \
280 VM_CALLINFO_NOT_UNDER_GC, \
287typedef VALUE (*vm_call_handler)(
302 const vm_call_handler call_;
308 const enum method_missing_reason method_missing_reason;
315#define VM_CALLCACHE_IVAR IMEMO_FL_USER0
316#define VM_CALLCACHE_BF IMEMO_FL_USER1
317#define VM_CALLCACHE_SUPER IMEMO_FL_USER2
318#define VM_CALLCACHE_REFINEMENT IMEMO_FL_USER3
319#define VM_CALLCACHE_UNMARKABLE IMEMO_FL_USER4
320#define VM_CALLCACHE_ON_STACK IMEMO_FL_USER5
321#define VM_CALLCACHE_INVALID_SUPER IMEMO_FL_USER6
330extern const struct rb_callcache rb_vm_empty_cc_for_super;
333#define vm_cc_empty() (&rb_vm_empty_cc)
336cc_check_class(
VALUE klass)
344void rb_vm_cc_table_delete(
VALUE table,
ID mid);
346static inline void vm_cc_attr_index_set(
const struct rb_callcache *cc, uint64_t packed_cache);
349vm_cc_new(
VALUE klass,
351 vm_call_handler call,
352 enum vm_cc_type
type)
354 cc_check_class(klass);
356 rb_gc_declare_weak_references((
VALUE)cc);
359 *((vm_call_handler *)&cc->call_) = call;
365 *(
VALUE *)&cc->flags |= VM_CALLCACHE_SUPER;
367 case cc_type_refinement:
368 *(
VALUE *)&cc->flags |= VM_CALLCACHE_REFINEMENT;
369 rb_vm_insert_cc_refinement(cc);
374 if (cme->def->type == VM_METHOD_TYPE_ATTRSET) {
375 vm_cc_attr_index_set(cc, IVAR_CACHE_INIT);
377 else if (cme->def->type == VM_METHOD_TYPE_IVAR) {
378 vm_cc_attr_index_set(cc, rb_getivar_cache_pack(ROOT_SHAPE_ID, ATTR_INDEX_NOT_SET));
382 *(
VALUE *)&cc->flags |= VM_CALLCACHE_INVALID_SUPER;
385 RB_DEBUG_COUNTER_INC(cc_new);
392 return (cc->flags & VM_CALLCACHE_SUPER) != 0;
398 return (cc->flags & VM_CALLCACHE_REFINEMENT) != 0;
401#define VM_CC_ON_STACK(clazz, call, aux, cme) \
402 (struct rb_callcache) { \
404 (imemo_callcache << FL_USHIFT) | \
405 VM_CALLCACHE_UNMARKABLE | \
406 VM_CALLCACHE_ON_STACK, \
407 .klass = cc_check_class(clazz), \
416 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
417 VM_ASSERT(cc_check_class(cc->klass));
418 return cc->klass == klass;
424 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
431 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
439 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
440 VM_ASSERT(cc_check_class(cc->klass));
442 return !UNDEF_P(cc->klass);
452# define VM_CC_RACED_P() rb_multi_ractor_p()
454# define VM_CC_RACED_P() true
460 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
464 VM_ASSERT(cc->klass !=
Qundef || !vm_cc_markable(cc) || vm_cc_invalid_super(cc) || VM_CC_RACED_P());
465 VM_ASSERT(cc_check_class(cc->klass));
466 VM_ASSERT(cc->call_ == NULL ||
467 !vm_cc_markable(cc) ||
468 vm_cc_invalid_super(cc) ||
474static inline vm_call_handler
477 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
478 VM_ASSERT(cc->call_ != NULL);
479 VM_ASSERT(cc->klass !=
Qundef || !vm_cc_markable(cc) || vm_cc_invalid_super(cc) || VM_CC_RACED_P());
480 VM_ASSERT(cc_check_class(cc->klass));
484static inline uint64_t
487 return ATOMIC_U64_LOAD_RELAXED(cc->aux_.attr.value);
490static inline uint64_t
493 return ATOMIC_U64_LOAD_RELAXED(ic->value);
496static inline uint64_t
500 return vm_cc_atomic_cache_read(cc);
503 return vm_ic_atomic_cache_read(ic);
507static inline unsigned int
508vm_cc_cmethod_missing_reason(
const struct rb_callcache *cc)
510 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
511 return cc->aux_.method_missing_reason;
517 if (vm_cc_valid(cc) && !METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc))) {
528vm_cc_call_set(
const struct rb_callcache *cc, vm_call_handler call)
530 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
531 VM_ASSERT(cc != vm_cc_empty());
532 *(vm_call_handler *)&cc->call_ = call;
538 *(
VALUE *)&cc->flags |= VM_CALLCACHE_IVAR;
544 return (cc->flags & VM_CALLCACHE_IVAR) != 0;
548vm_cc_attr_index_set(
const struct rb_callcache *cc, uint64_t packed_cache)
550 uint64_t *attr_value = (uint64_t *)&cc->aux_.attr.value;
551 if (!vm_cc_markable(cc)) {
552 *attr_value = IVAR_CACHE_INIT;
555 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
556 VM_ASSERT(cc != vm_cc_empty());
557 *attr_value = packed_cache;
565 vm_cc_attr_index_set(cc, packed_cache);
568 ATOMIC_U64_SET_RELAXED(ic->value, packed_cache);
573vm_cc_method_missing_reason_set(
const struct rb_callcache *cc,
enum method_missing_reason reason)
575 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
576 VM_ASSERT(cc != vm_cc_empty());
577 *(
enum method_missing_reason *)&cc->aux_.method_missing_reason = reason;
583 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
584 VM_ASSERT(cc != vm_cc_empty());
586 *(
VALUE *)&cc->flags |= VM_CALLCACHE_BF;
592 return (cc->flags & VM_CALLCACHE_BF) != 0;
598 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
599 VM_ASSERT(cc != vm_cc_empty());
601 VM_ASSERT(cc->klass !=
Qundef || rb_multi_ractor_p());
604 RB_DEBUG_COUNTER_INC(cc_ent_invalidate);
625 unsigned short kw_len;
626 } entries[FLEX_ARY_LEN];
630STATIC_ASSERT(cc_entries_flag_fits_in_short, VM_CALL__END <= 16);
634#define VM_CALL_KW_LEN_MAX UINT16_MAX
637vm_ccs_alloc_size(
size_t capa)
644const rb_callable_method_entry_t *rb_vm_lookup_overloaded_cme(
const rb_callable_method_entry_t *cme);
645void rb_vm_dump_overloaded_cme_table(
void);
650 return ccs->debug_sig == ~(
VALUE)ccs;
654vm_cc_check_cme_unlocked(
const struct rb_callcache *cc,
const rb_callable_method_entry_t *cme)
656 return vm_cc_cme(cc) == cme ||
657 (cme->def->iseq_overload && vm_cc_cme(cc) == rb_vm_lookup_overloaded_cme(cme));
661vm_cc_check_cme(
const struct rb_callcache *cc,
const rb_callable_method_entry_t *cme)
664 if (rb_current_execution_context(
false) == NULL) {
665 valid = vm_cc_check_cme_unlocked(cc, cme);
668 RB_VM_LOCKING_NO_BARRIER() {
669 valid = vm_cc_check_cme_unlocked(cc, cme);
678 fprintf(stderr,
"iseq_overload:%d, cme:%p (def:%p), cm_cc_cme(cc):%p (def:%p)\n",
679 (
int)cme->def->iseq_overload,
680 (
void *)cme, (
void *)cme->def,
681 (
void *)vm_cc_cme(cc), (
void *)vm_cc_cme(cc)->def);
684 rp(rb_vm_lookup_overloaded_cme(cme));
#define RUBY_ATOMIC_INC(var)
Atomically increments the value pointed by var.
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
#define RUBY_ATOMIC_FETCH_SUB(var, val)
Atomically replaces the value pointed by var with the result of subtraction of val to the old value o...
#define Qundef
Old name of RUBY_Qundef.
#define T_ICLASS
Old name of RUBY_T_ICLASS.
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
#define FL_ANY_RAW
Old name of RB_FL_ANY_RAW.
#define T_CLASS
Old name of RUBY_T_CLASS.
VALUE rb_eRuntimeError
RuntimeError exception.
int capa
Designed capacity of the buffer.
VALUE type(ANYARGS)
ANYARGS-ed function type.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.