Ruby  3.4.0dev (2024-12-06 revision 892c46283a5ea4179500d951c9d4866c0051f27b)
cont.c (892c46283a5ea4179500d951c9d4866c0051f27b)
1 /**********************************************************************
2 
3  cont.c -
4 
5  $Author$
6  created at: Thu May 23 09:03:43 2007
7 
8  Copyright (C) 2007 Koichi Sasada
9 
10 **********************************************************************/
11 
12 #include "ruby/internal/config.h"
13 
14 #ifndef _WIN32
15 #include <unistd.h>
16 #include <sys/mman.h>
17 #endif
18 
19 // On Solaris, madvise() is NOT declared for SUS (XPG4v2) or later,
20 // but MADV_* macros are defined when __EXTENSIONS__ is defined.
21 #ifdef NEED_MADVICE_PROTOTYPE_USING_CADDR_T
22 #include <sys/types.h>
23 extern int madvise(caddr_t, size_t, int);
24 #endif
25 
26 #include COROUTINE_H
27 
28 #include "eval_intern.h"
29 #include "internal.h"
30 #include "internal/cont.h"
31 #include "internal/thread.h"
32 #include "internal/error.h"
33 #include "internal/gc.h"
34 #include "internal/proc.h"
35 #include "internal/sanitizers.h"
36 #include "internal/warnings.h"
37 #include "ruby/fiber/scheduler.h"
38 #include "rjit.h"
39 #include "yjit.h"
40 #include "vm_core.h"
41 #include "vm_sync.h"
42 #include "id_table.h"
43 #include "ractor_core.h"
44 
45 static const int DEBUG = 0;
46 
47 #define RB_PAGE_SIZE (pagesize)
48 #define RB_PAGE_MASK (~(RB_PAGE_SIZE - 1))
49 static long pagesize;
50 
51 static const rb_data_type_t cont_data_type, fiber_data_type;
52 static VALUE rb_cContinuation;
53 static VALUE rb_cFiber;
54 static VALUE rb_eFiberError;
55 #ifdef RB_EXPERIMENTAL_FIBER_POOL
56 static VALUE rb_cFiberPool;
57 #endif
58 
59 #define CAPTURE_JUST_VALID_VM_STACK 1
60 
61 // Defined in `coroutine/$arch/Context.h`:
62 #ifdef COROUTINE_LIMITED_ADDRESS_SPACE
63 #define FIBER_POOL_ALLOCATION_FREE
64 #define FIBER_POOL_INITIAL_SIZE 8
65 #define FIBER_POOL_ALLOCATION_MAXIMUM_SIZE 32
66 #else
67 #define FIBER_POOL_INITIAL_SIZE 32
68 #define FIBER_POOL_ALLOCATION_MAXIMUM_SIZE 1024
69 #endif
70 #ifdef RB_EXPERIMENTAL_FIBER_POOL
71 #define FIBER_POOL_ALLOCATION_FREE
72 #endif
73 
74 enum context_type {
75  CONTINUATION_CONTEXT = 0,
76  FIBER_CONTEXT = 1
77 };
78 
80  VALUE *ptr;
81 #ifdef CAPTURE_JUST_VALID_VM_STACK
82  size_t slen; /* length of stack (head of ec->vm_stack) */
83  size_t clen; /* length of control frames (tail of ec->vm_stack) */
84 #endif
85 };
86 
87 struct fiber_pool;
88 
89 // Represents a single stack.
91  // A pointer to the memory allocation (lowest address) for the stack.
92  void * base;
93 
94  // The current stack pointer, taking into account the direction of the stack.
95  void * current;
96 
97  // The size of the stack excluding any guard pages.
98  size_t size;
99 
100  // The available stack capacity w.r.t. the current stack offset.
101  size_t available;
102 
103  // The pool this stack should be allocated from.
104  struct fiber_pool * pool;
105 
106  // If the stack is allocated, the allocation it came from.
107  struct fiber_pool_allocation * allocation;
108 };
109 
110 // A linked list of vacant (unused) stacks.
111 // This structure is stored in the first page of a stack if it is not in use.
112 // @sa fiber_pool_vacancy_pointer
114  // Details about the vacant stack:
115  struct fiber_pool_stack stack;
116 
117  // The vacancy linked list.
118 #ifdef FIBER_POOL_ALLOCATION_FREE
119  struct fiber_pool_vacancy * previous;
120 #endif
121  struct fiber_pool_vacancy * next;
122 };
123 
124 // Manages singly linked list of mapped regions of memory which contains 1 more more stack:
125 //
126 // base = +-------------------------------+-----------------------+ +
127 // |VM Stack |VM Stack | | |
128 // | | | | |
129 // | | | | |
130 // +-------------------------------+ | |
131 // |Machine Stack |Machine Stack | | |
132 // | | | | |
133 // | | | | |
134 // | | | . . . . | | size
135 // | | | | |
136 // | | | | |
137 // | | | | |
138 // | | | | |
139 // | | | | |
140 // +-------------------------------+ | |
141 // |Guard Page |Guard Page | | |
142 // +-------------------------------+-----------------------+ v
143 //
144 // +------------------------------------------------------->
145 //
146 // count
147 //
149  // A pointer to the memory mapped region.
150  void * base;
151 
152  // The size of the individual stacks.
153  size_t size;
154 
155  // The stride of individual stacks (including any guard pages or other accounting details).
156  size_t stride;
157 
158  // The number of stacks that were allocated.
159  size_t count;
160 
161 #ifdef FIBER_POOL_ALLOCATION_FREE
162  // The number of stacks used in this allocation.
163  size_t used;
164 #endif
165 
166  struct fiber_pool * pool;
167 
168  // The allocation linked list.
169 #ifdef FIBER_POOL_ALLOCATION_FREE
170  struct fiber_pool_allocation * previous;
171 #endif
172  struct fiber_pool_allocation * next;
173 };
174 
175 // A fiber pool manages vacant stacks to reduce the overhead of creating fibers.
176 struct fiber_pool {
177  // A singly-linked list of allocations which contain 1 or more stacks each.
178  struct fiber_pool_allocation * allocations;
179 
180  // Free list that provides O(1) stack "allocation".
181  struct fiber_pool_vacancy * vacancies;
182 
183  // The size of the stack allocations (excluding any guard page).
184  size_t size;
185 
186  // The total number of stacks that have been allocated in this pool.
187  size_t count;
188 
189  // The initial number of stacks to allocate.
190  size_t initial_count;
191 
192  // Whether to madvise(free) the stack or not.
193  // If this value is set to 1, the stack will be madvise(free)ed
194  // (or equivalent), where possible, when it is returned to the pool.
195  int free_stacks;
196 
197  // The number of stacks that have been used in this pool.
198  size_t used;
199 
200  // The amount to allocate for the vm_stack.
201  size_t vm_stack_size;
202 };
203 
204 // Continuation contexts used by JITs
205 struct rb_jit_cont {
206  rb_execution_context_t *ec; // continuation ec
207  struct rb_jit_cont *prev, *next; // used to form lists
208 };
209 
210 // Doubly linked list for enumerating all on-stack ISEQs.
211 static struct rb_jit_cont *first_jit_cont;
212 
213 typedef struct rb_context_struct {
214  enum context_type type;
215  int argc;
216  int kw_splat;
217  VALUE self;
218  VALUE value;
219 
220  struct cont_saved_vm_stack saved_vm_stack;
221 
222  struct {
223  VALUE *stack;
224  VALUE *stack_src;
225  size_t stack_size;
226  } machine;
227  rb_execution_context_t saved_ec;
228  rb_jmpbuf_t jmpbuf;
229  struct rb_jit_cont *jit_cont; // Continuation contexts for JITs
230 } rb_context_t;
231 
232 /*
233  * Fiber status:
234  * [Fiber.new] ------> FIBER_CREATED ----> [Fiber#kill] --> |
235  * | [Fiber#resume] |
236  * v |
237  * +--> FIBER_RESUMED ----> [return] ------> |
238  * [Fiber#resume] | | [Fiber.yield/transfer] |
239  * [Fiber#transfer] | v |
240  * +--- FIBER_SUSPENDED --> [Fiber#kill] --> |
241  * |
242  * |
243  * FIBER_TERMINATED <-------------------+
244  */
245 enum fiber_status {
246  FIBER_CREATED,
247  FIBER_RESUMED,
248  FIBER_SUSPENDED,
249  FIBER_TERMINATED
250 };
251 
252 #define FIBER_CREATED_P(fiber) ((fiber)->status == FIBER_CREATED)
253 #define FIBER_RESUMED_P(fiber) ((fiber)->status == FIBER_RESUMED)
254 #define FIBER_SUSPENDED_P(fiber) ((fiber)->status == FIBER_SUSPENDED)
255 #define FIBER_TERMINATED_P(fiber) ((fiber)->status == FIBER_TERMINATED)
256 #define FIBER_RUNNABLE_P(fiber) (FIBER_CREATED_P(fiber) || FIBER_SUSPENDED_P(fiber))
257 
259  rb_context_t cont;
260  VALUE first_proc;
261  struct rb_fiber_struct *prev;
262  struct rb_fiber_struct *resuming_fiber;
263 
264  BITFIELD(enum fiber_status, status, 2);
265  /* Whether the fiber is allowed to implicitly yield. */
266  unsigned int yielding : 1;
267  unsigned int blocking : 1;
268 
269  unsigned int killed : 1;
270 
271  struct coroutine_context context;
272  struct fiber_pool_stack stack;
273 };
274 
275 static struct fiber_pool shared_fiber_pool = {NULL, NULL, 0, 0, 0, 0};
276 
277 void
278 rb_free_shared_fiber_pool(void)
279 {
280  xfree(shared_fiber_pool.allocations);
281 }
282 
283 static ID fiber_initialize_keywords[3] = {0};
284 
285 /*
286  * FreeBSD require a first (i.e. addr) argument of mmap(2) is not NULL
287  * if MAP_STACK is passed.
288  * https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=158755
289  */
290 #if defined(MAP_STACK) && !defined(__FreeBSD__) && !defined(__FreeBSD_kernel__)
291 #define FIBER_STACK_FLAGS (MAP_PRIVATE | MAP_ANON | MAP_STACK)
292 #else
293 #define FIBER_STACK_FLAGS (MAP_PRIVATE | MAP_ANON)
294 #endif
295 
296 #define ERRNOMSG strerror(errno)
297 
298 // Locates the stack vacancy details for the given stack.
299 inline static struct fiber_pool_vacancy *
300 fiber_pool_vacancy_pointer(void * base, size_t size)
301 {
302  STACK_GROW_DIR_DETECTION;
303 
304  return (struct fiber_pool_vacancy *)(
305  (char*)base + STACK_DIR_UPPER(0, size - RB_PAGE_SIZE)
306  );
307 }
308 
309 #if defined(COROUTINE_SANITIZE_ADDRESS)
310 // Compute the base pointer for a vacant stack, for the area which can be poisoned.
311 inline static void *
312 fiber_pool_stack_poison_base(struct fiber_pool_stack * stack)
313 {
314  STACK_GROW_DIR_DETECTION;
315 
316  return (char*)stack->base + STACK_DIR_UPPER(RB_PAGE_SIZE, 0);
317 }
318 
319 // Compute the size of the vacant stack, for the area that can be poisoned.
320 inline static size_t
321 fiber_pool_stack_poison_size(struct fiber_pool_stack * stack)
322 {
323  return stack->size - RB_PAGE_SIZE;
324 }
325 #endif
326 
327 // Reset the current stack pointer and available size of the given stack.
328 inline static void
329 fiber_pool_stack_reset(struct fiber_pool_stack * stack)
330 {
331  STACK_GROW_DIR_DETECTION;
332 
333  stack->current = (char*)stack->base + STACK_DIR_UPPER(0, stack->size);
334  stack->available = stack->size;
335 }
336 
337 // A pointer to the base of the current unused portion of the stack.
338 inline static void *
339 fiber_pool_stack_base(struct fiber_pool_stack * stack)
340 {
341  STACK_GROW_DIR_DETECTION;
342 
343  VM_ASSERT(stack->current);
344 
345  return STACK_DIR_UPPER(stack->current, (char*)stack->current - stack->available);
346 }
347 
348 // Allocate some memory from the stack. Used to allocate vm_stack inline with machine stack.
349 // @sa fiber_initialize_coroutine
350 inline static void *
351 fiber_pool_stack_alloca(struct fiber_pool_stack * stack, size_t offset)
352 {
353  STACK_GROW_DIR_DETECTION;
354 
355  if (DEBUG) fprintf(stderr, "fiber_pool_stack_alloca(%p): %"PRIuSIZE"/%"PRIuSIZE"\n", (void*)stack, offset, stack->available);
356  VM_ASSERT(stack->available >= offset);
357 
358  // The pointer to the memory being allocated:
359  void * pointer = STACK_DIR_UPPER(stack->current, (char*)stack->current - offset);
360 
361  // Move the stack pointer:
362  stack->current = STACK_DIR_UPPER((char*)stack->current + offset, (char*)stack->current - offset);
363  stack->available -= offset;
364 
365  return pointer;
366 }
367 
368 // Reset the current stack pointer and available size of the given stack.
369 inline static void
370 fiber_pool_vacancy_reset(struct fiber_pool_vacancy * vacancy)
371 {
372  fiber_pool_stack_reset(&vacancy->stack);
373 
374  // Consume one page of the stack because it's used for the vacancy list:
375  fiber_pool_stack_alloca(&vacancy->stack, RB_PAGE_SIZE);
376 }
377 
378 inline static struct fiber_pool_vacancy *
379 fiber_pool_vacancy_push(struct fiber_pool_vacancy * vacancy, struct fiber_pool_vacancy * head)
380 {
381  vacancy->next = head;
382 
383 #ifdef FIBER_POOL_ALLOCATION_FREE
384  if (head) {
385  head->previous = vacancy;
386  vacancy->previous = NULL;
387  }
388 #endif
389 
390  return vacancy;
391 }
392 
393 #ifdef FIBER_POOL_ALLOCATION_FREE
394 static void
395 fiber_pool_vacancy_remove(struct fiber_pool_vacancy * vacancy)
396 {
397  if (vacancy->next) {
398  vacancy->next->previous = vacancy->previous;
399  }
400 
401  if (vacancy->previous) {
402  vacancy->previous->next = vacancy->next;
403  }
404  else {
405  // It's the head of the list:
406  vacancy->stack.pool->vacancies = vacancy->next;
407  }
408 }
409 
410 inline static struct fiber_pool_vacancy *
411 fiber_pool_vacancy_pop(struct fiber_pool * pool)
412 {
413  struct fiber_pool_vacancy * vacancy = pool->vacancies;
414 
415  if (vacancy) {
416  fiber_pool_vacancy_remove(vacancy);
417  }
418 
419  return vacancy;
420 }
421 #else
422 inline static struct fiber_pool_vacancy *
423 fiber_pool_vacancy_pop(struct fiber_pool * pool)
424 {
425  struct fiber_pool_vacancy * vacancy = pool->vacancies;
426 
427  if (vacancy) {
428  pool->vacancies = vacancy->next;
429  }
430 
431  return vacancy;
432 }
433 #endif
434 
435 // Initialize the vacant stack. The [base, size] allocation should not include the guard page.
436 // @param base The pointer to the lowest address of the allocated memory.
437 // @param size The size of the allocated memory.
438 inline static struct fiber_pool_vacancy *
439 fiber_pool_vacancy_initialize(struct fiber_pool * fiber_pool, struct fiber_pool_vacancy * vacancies, void * base, size_t size)
440 {
441  struct fiber_pool_vacancy * vacancy = fiber_pool_vacancy_pointer(base, size);
442 
443  vacancy->stack.base = base;
444  vacancy->stack.size = size;
445 
446  fiber_pool_vacancy_reset(vacancy);
447 
448  vacancy->stack.pool = fiber_pool;
449 
450  return fiber_pool_vacancy_push(vacancy, vacancies);
451 }
452 
453 // Allocate a maximum of count stacks, size given by stride.
454 // @param count the number of stacks to allocate / were allocated.
455 // @param stride the size of the individual stacks.
456 // @return [void *] the allocated memory or NULL if allocation failed.
457 inline static void *
458 fiber_pool_allocate_memory(size_t * count, size_t stride)
459 {
460  // We use a divide-by-2 strategy to try and allocate memory. We are trying
461  // to allocate `count` stacks. In normal situation, this won't fail. But
462  // if we ran out of address space, or we are allocating more memory than
463  // the system would allow (e.g. overcommit * physical memory + swap), we
464  // divide count by two and try again. This condition should only be
465  // encountered in edge cases, but we handle it here gracefully.
466  while (*count > 1) {
467 #if defined(_WIN32)
468  void * base = VirtualAlloc(0, (*count)*stride, MEM_COMMIT, PAGE_READWRITE);
469 
470  if (!base) {
471  *count = (*count) >> 1;
472  }
473  else {
474  return base;
475  }
476 #else
477  errno = 0;
478  size_t mmap_size = (*count)*stride;
479  void * base = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, FIBER_STACK_FLAGS, -1, 0);
480 
481  if (base == MAP_FAILED) {
482  // If the allocation fails, count = count / 2, and try again.
483  *count = (*count) >> 1;
484  }
485  else {
486  ruby_annotate_mmap(base, mmap_size, "Ruby:fiber_pool_allocate_memory");
487 #if defined(MADV_FREE_REUSE)
488  // On Mac MADV_FREE_REUSE is necessary for the task_info api
489  // to keep the accounting accurate as possible when a page is marked as reusable
490  // it can possibly not occurring at first call thus re-iterating if necessary.
491  while (madvise(base, mmap_size, MADV_FREE_REUSE) == -1 && errno == EAGAIN);
492 #endif
493  return base;
494  }
495 #endif
496  }
497 
498  return NULL;
499 }
500 
501 // Given an existing fiber pool, expand it by the specified number of stacks.
502 // @param count the maximum number of stacks to allocate.
503 // @return the allocated fiber pool.
504 // @sa fiber_pool_allocation_free
505 static struct fiber_pool_allocation *
506 fiber_pool_expand(struct fiber_pool * fiber_pool, size_t count)
507 {
508  STACK_GROW_DIR_DETECTION;
509 
510  size_t size = fiber_pool->size;
511  size_t stride = size + RB_PAGE_SIZE;
512 
513  // Allocate the memory required for the stacks:
514  void * base = fiber_pool_allocate_memory(&count, stride);
515 
516  if (base == NULL) {
517  rb_raise(rb_eFiberError, "can't alloc machine stack to fiber (%"PRIuSIZE" x %"PRIuSIZE" bytes): %s", count, size, ERRNOMSG);
518  }
519 
520  struct fiber_pool_vacancy * vacancies = fiber_pool->vacancies;
521  struct fiber_pool_allocation * allocation = RB_ALLOC(struct fiber_pool_allocation);
522 
523  // Initialize fiber pool allocation:
524  allocation->base = base;
525  allocation->size = size;
526  allocation->stride = stride;
527  allocation->count = count;
528 #ifdef FIBER_POOL_ALLOCATION_FREE
529  allocation->used = 0;
530 #endif
531  allocation->pool = fiber_pool;
532 
533  if (DEBUG) {
534  fprintf(stderr, "fiber_pool_expand(%"PRIuSIZE"): %p, %"PRIuSIZE"/%"PRIuSIZE" x [%"PRIuSIZE":%"PRIuSIZE"]\n",
535  count, (void*)fiber_pool, fiber_pool->used, fiber_pool->count, size, fiber_pool->vm_stack_size);
536  }
537 
538  // Iterate over all stacks, initializing the vacancy list:
539  for (size_t i = 0; i < count; i += 1) {
540  void * base = (char*)allocation->base + (stride * i);
541  void * page = (char*)base + STACK_DIR_UPPER(size, 0);
542 
543 #if defined(_WIN32)
544  DWORD old_protect;
545 
546  if (!VirtualProtect(page, RB_PAGE_SIZE, PAGE_READWRITE | PAGE_GUARD, &old_protect)) {
547  VirtualFree(allocation->base, 0, MEM_RELEASE);
548  rb_raise(rb_eFiberError, "can't set a guard page: %s", ERRNOMSG);
549  }
550 #else
551  if (mprotect(page, RB_PAGE_SIZE, PROT_NONE) < 0) {
552  munmap(allocation->base, count*stride);
553  rb_raise(rb_eFiberError, "can't set a guard page: %s", ERRNOMSG);
554  }
555 #endif
556 
557  vacancies = fiber_pool_vacancy_initialize(
558  fiber_pool, vacancies,
559  (char*)base + STACK_DIR_UPPER(0, RB_PAGE_SIZE),
560  size
561  );
562 
563 #ifdef FIBER_POOL_ALLOCATION_FREE
564  vacancies->stack.allocation = allocation;
565 #endif
566  }
567 
568  // Insert the allocation into the head of the pool:
569  allocation->next = fiber_pool->allocations;
570 
571 #ifdef FIBER_POOL_ALLOCATION_FREE
572  if (allocation->next) {
573  allocation->next->previous = allocation;
574  }
575 
576  allocation->previous = NULL;
577 #endif
578 
579  fiber_pool->allocations = allocation;
580  fiber_pool->vacancies = vacancies;
581  fiber_pool->count += count;
582 
583  return allocation;
584 }
585 
586 // Initialize the specified fiber pool with the given number of stacks.
587 // @param vm_stack_size The size of the vm stack to allocate.
588 static void
589 fiber_pool_initialize(struct fiber_pool * fiber_pool, size_t size, size_t count, size_t vm_stack_size)
590 {
591  VM_ASSERT(vm_stack_size < size);
592 
593  fiber_pool->allocations = NULL;
594  fiber_pool->vacancies = NULL;
595  fiber_pool->size = ((size / RB_PAGE_SIZE) + 1) * RB_PAGE_SIZE;
596  fiber_pool->count = 0;
597  fiber_pool->initial_count = count;
598  fiber_pool->free_stacks = 1;
599  fiber_pool->used = 0;
600 
601  fiber_pool->vm_stack_size = vm_stack_size;
602 
603  fiber_pool_expand(fiber_pool, count);
604 }
605 
606 #ifdef FIBER_POOL_ALLOCATION_FREE
607 // Free the list of fiber pool allocations.
608 static void
609 fiber_pool_allocation_free(struct fiber_pool_allocation * allocation)
610 {
611  STACK_GROW_DIR_DETECTION;
612 
613  VM_ASSERT(allocation->used == 0);
614 
615  if (DEBUG) fprintf(stderr, "fiber_pool_allocation_free: %p base=%p count=%"PRIuSIZE"\n", (void*)allocation, allocation->base, allocation->count);
616 
617  size_t i;
618  for (i = 0; i < allocation->count; i += 1) {
619  void * base = (char*)allocation->base + (allocation->stride * i) + STACK_DIR_UPPER(0, RB_PAGE_SIZE);
620 
621  struct fiber_pool_vacancy * vacancy = fiber_pool_vacancy_pointer(base, allocation->size);
622 
623  // Pop the vacant stack off the free list:
624  fiber_pool_vacancy_remove(vacancy);
625  }
626 
627 #ifdef _WIN32
628  VirtualFree(allocation->base, 0, MEM_RELEASE);
629 #else
630  munmap(allocation->base, allocation->stride * allocation->count);
631 #endif
632 
633  if (allocation->previous) {
634  allocation->previous->next = allocation->next;
635  }
636  else {
637  // We are the head of the list, so update the pool:
638  allocation->pool->allocations = allocation->next;
639  }
640 
641  if (allocation->next) {
642  allocation->next->previous = allocation->previous;
643  }
644 
645  allocation->pool->count -= allocation->count;
646 
647  ruby_xfree(allocation);
648 }
649 #endif
650 
651 // Acquire a stack from the given fiber pool. If none are available, allocate more.
652 static struct fiber_pool_stack
653 fiber_pool_stack_acquire(struct fiber_pool * fiber_pool)
654 {
655  struct fiber_pool_vacancy * vacancy = fiber_pool_vacancy_pop(fiber_pool);
656 
657  if (DEBUG) fprintf(stderr, "fiber_pool_stack_acquire: %p used=%"PRIuSIZE"\n", (void*)fiber_pool->vacancies, fiber_pool->used);
658 
659  if (!vacancy) {
660  const size_t maximum = FIBER_POOL_ALLOCATION_MAXIMUM_SIZE;
661  const size_t minimum = fiber_pool->initial_count;
662 
663  size_t count = fiber_pool->count;
664  if (count > maximum) count = maximum;
665  if (count < minimum) count = minimum;
666 
667  fiber_pool_expand(fiber_pool, count);
668 
669  // The free list should now contain some stacks:
670  VM_ASSERT(fiber_pool->vacancies);
671 
672  vacancy = fiber_pool_vacancy_pop(fiber_pool);
673  }
674 
675  VM_ASSERT(vacancy);
676  VM_ASSERT(vacancy->stack.base);
677 
678 #if defined(COROUTINE_SANITIZE_ADDRESS)
679  __asan_unpoison_memory_region(fiber_pool_stack_poison_base(&vacancy->stack), fiber_pool_stack_poison_size(&vacancy->stack));
680 #endif
681 
682  // Take the top item from the free list:
683  fiber_pool->used += 1;
684 
685 #ifdef FIBER_POOL_ALLOCATION_FREE
686  vacancy->stack.allocation->used += 1;
687 #endif
688 
689  fiber_pool_stack_reset(&vacancy->stack);
690 
691  return vacancy->stack;
692 }
693 
694 // We advise the operating system that the stack memory pages are no longer being used.
695 // This introduce some performance overhead but allows system to relaim memory when there is pressure.
696 static inline void
697 fiber_pool_stack_free(struct fiber_pool_stack * stack)
698 {
699  void * base = fiber_pool_stack_base(stack);
700  size_t size = stack->available;
701 
702  // If this is not true, the vacancy information will almost certainly be destroyed:
703  VM_ASSERT(size <= (stack->size - RB_PAGE_SIZE));
704 
705  int advice = stack->pool->free_stacks >> 1;
706 
707  if (DEBUG) fprintf(stderr, "fiber_pool_stack_free: %p+%"PRIuSIZE" [base=%p, size=%"PRIuSIZE"] advice=%d\n", base, size, stack->base, stack->size, advice);
708 
709  // The pages being used by the stack can be returned back to the system.
710  // That doesn't change the page mapping, but it does allow the system to
711  // reclaim the physical memory.
712  // Since we no longer care about the data itself, we don't need to page
713  // out to disk, since that is costly. Not all systems support that, so
714  // we try our best to select the most efficient implementation.
715  // In addition, it's actually slightly desirable to not do anything here,
716  // but that results in higher memory usage.
717 
718 #ifdef __wasi__
719  // WebAssembly doesn't support madvise, so we just don't do anything.
720 #elif VM_CHECK_MODE > 0 && defined(MADV_DONTNEED)
721  if (!advice) advice = MADV_DONTNEED;
722  // This immediately discards the pages and the memory is reset to zero.
723  madvise(base, size, advice);
724 #elif defined(MADV_FREE_REUSABLE)
725  if (!advice) advice = MADV_FREE_REUSABLE;
726  // Darwin / macOS / iOS.
727  // Acknowledge the kernel down to the task info api we make this
728  // page reusable for future use.
729  // As for MADV_FREE_REUSABLE below we ensure in the rare occasions the task was not
730  // completed at the time of the call to re-iterate.
731  while (madvise(base, size, advice) == -1 && errno == EAGAIN);
732 #elif defined(MADV_FREE)
733  if (!advice) advice = MADV_FREE;
734  // Recent Linux.
735  madvise(base, size, advice);
736 #elif defined(MADV_DONTNEED)
737  if (!advice) advice = MADV_DONTNEED;
738  // Old Linux.
739  madvise(base, size, advice);
740 #elif defined(POSIX_MADV_DONTNEED)
741  if (!advice) advice = POSIX_MADV_DONTNEED;
742  // Solaris?
743  posix_madvise(base, size, advice);
744 #elif defined(_WIN32)
745  VirtualAlloc(base, size, MEM_RESET, PAGE_READWRITE);
746  // Not available in all versions of Windows.
747  //DiscardVirtualMemory(base, size);
748 #endif
749 
750 #if defined(COROUTINE_SANITIZE_ADDRESS)
751  __asan_poison_memory_region(fiber_pool_stack_poison_base(stack), fiber_pool_stack_poison_size(stack));
752 #endif
753 }
754 
755 // Release and return a stack to the vacancy list.
756 static void
757 fiber_pool_stack_release(struct fiber_pool_stack * stack)
758 {
759  struct fiber_pool * pool = stack->pool;
760  struct fiber_pool_vacancy * vacancy = fiber_pool_vacancy_pointer(stack->base, stack->size);
761 
762  if (DEBUG) fprintf(stderr, "fiber_pool_stack_release: %p used=%"PRIuSIZE"\n", stack->base, stack->pool->used);
763 
764  // Copy the stack details into the vacancy area:
765  vacancy->stack = *stack;
766  // After this point, be careful about updating/using state in stack, since it's copied to the vacancy area.
767 
768  // Reset the stack pointers and reserve space for the vacancy data:
769  fiber_pool_vacancy_reset(vacancy);
770 
771  // Push the vacancy into the vancancies list:
772  pool->vacancies = fiber_pool_vacancy_push(vacancy, pool->vacancies);
773  pool->used -= 1;
774 
775 #ifdef FIBER_POOL_ALLOCATION_FREE
776  struct fiber_pool_allocation * allocation = stack->allocation;
777 
778  allocation->used -= 1;
779 
780  // Release address space and/or dirty memory:
781  if (allocation->used == 0) {
782  fiber_pool_allocation_free(allocation);
783  }
784  else if (stack->pool->free_stacks) {
785  fiber_pool_stack_free(&vacancy->stack);
786  }
787 #else
788  // This is entirely optional, but clears the dirty flag from the stack
789  // memory, so it won't get swapped to disk when there is memory pressure:
790  if (stack->pool->free_stacks) {
791  fiber_pool_stack_free(&vacancy->stack);
792  }
793 #endif
794 }
795 
796 static inline void
797 ec_switch(rb_thread_t *th, rb_fiber_t *fiber)
798 {
799  rb_execution_context_t *ec = &fiber->cont.saved_ec;
800 #ifdef RUBY_ASAN_ENABLED
801  ec->machine.asan_fake_stack_handle = asan_get_thread_fake_stack_handle();
802 #endif
803  rb_ractor_set_current_ec(th->ractor, th->ec = ec);
804  // ruby_current_execution_context_ptr = th->ec = ec;
805 
806  /*
807  * timer-thread may set trap interrupt on previous th->ec at any time;
808  * ensure we do not delay (or lose) the trap interrupt handling.
809  */
810  if (th->vm->ractor.main_thread == th &&
811  rb_signal_buff_size() > 0) {
812  RUBY_VM_SET_TRAP_INTERRUPT(ec);
813  }
814 
815  VM_ASSERT(ec->fiber_ptr->cont.self == 0 || ec->vm_stack != NULL);
816 }
817 
818 static inline void
819 fiber_restore_thread(rb_thread_t *th, rb_fiber_t *fiber)
820 {
821  ec_switch(th, fiber);
822  VM_ASSERT(th->ec->fiber_ptr == fiber);
823 }
824 
825 NORETURN(static COROUTINE fiber_entry(struct coroutine_context * from, struct coroutine_context * to));
826 static COROUTINE
827 fiber_entry(struct coroutine_context * from, struct coroutine_context * to)
828 {
829  rb_fiber_t *fiber = to->argument;
830 
831 #if defined(COROUTINE_SANITIZE_ADDRESS)
832  // Address sanitizer will copy the previous stack base and stack size into
833  // the "from" fiber. `coroutine_initialize_main` doesn't generally know the
834  // stack bounds (base + size). Therefore, the main fiber `stack_base` and
835  // `stack_size` will be NULL/0. It's specifically important in that case to
836  // get the (base+size) of the previous fiber and save it, so that later when
837  // we return to the main coroutine, we don't supply (NULL, 0) to
838  // __sanitizer_start_switch_fiber which royally messes up the internal state
839  // of ASAN and causes (sometimes) the following message:
840  // "WARNING: ASan is ignoring requested __asan_handle_no_return"
841  __sanitizer_finish_switch_fiber(to->fake_stack, (const void**)&from->stack_base, &from->stack_size);
842 #endif
843 
844  rb_thread_t *thread = fiber->cont.saved_ec.thread_ptr;
845 
846 #ifdef COROUTINE_PTHREAD_CONTEXT
847  ruby_thread_set_native(thread);
848 #endif
849 
850  fiber_restore_thread(thread, fiber);
851 
852  rb_fiber_start(fiber);
853 
854 #ifndef COROUTINE_PTHREAD_CONTEXT
855  VM_UNREACHABLE(fiber_entry);
856 #endif
857 }
858 
859 // Initialize a fiber's coroutine's machine stack and vm stack.
860 static VALUE *
861 fiber_initialize_coroutine(rb_fiber_t *fiber, size_t * vm_stack_size)
862 {
863  struct fiber_pool * fiber_pool = fiber->stack.pool;
864  rb_execution_context_t *sec = &fiber->cont.saved_ec;
865  void * vm_stack = NULL;
866 
867  VM_ASSERT(fiber_pool != NULL);
868 
869  fiber->stack = fiber_pool_stack_acquire(fiber_pool);
870  vm_stack = fiber_pool_stack_alloca(&fiber->stack, fiber_pool->vm_stack_size);
871  *vm_stack_size = fiber_pool->vm_stack_size;
872 
873  coroutine_initialize(&fiber->context, fiber_entry, fiber_pool_stack_base(&fiber->stack), fiber->stack.available);
874 
875  // The stack for this execution context is the one we allocated:
876  sec->machine.stack_start = fiber->stack.current;
877  sec->machine.stack_maxsize = fiber->stack.available;
878 
879  fiber->context.argument = (void*)fiber;
880 
881  return vm_stack;
882 }
883 
884 // Release the stack from the fiber, it's execution context, and return it to
885 // the fiber pool.
886 static void
887 fiber_stack_release(rb_fiber_t * fiber)
888 {
889  rb_execution_context_t *ec = &fiber->cont.saved_ec;
890 
891  if (DEBUG) fprintf(stderr, "fiber_stack_release: %p, stack.base=%p\n", (void*)fiber, fiber->stack.base);
892 
893  // Return the stack back to the fiber pool if it wasn't already:
894  if (fiber->stack.base) {
895  fiber_pool_stack_release(&fiber->stack);
896  fiber->stack.base = NULL;
897  }
898 
899  // The stack is no longer associated with this execution context:
900  rb_ec_clear_vm_stack(ec);
901 }
902 
903 static const char *
904 fiber_status_name(enum fiber_status s)
905 {
906  switch (s) {
907  case FIBER_CREATED: return "created";
908  case FIBER_RESUMED: return "resumed";
909  case FIBER_SUSPENDED: return "suspended";
910  case FIBER_TERMINATED: return "terminated";
911  }
912  VM_UNREACHABLE(fiber_status_name);
913  return NULL;
914 }
915 
916 static void
917 fiber_verify(const rb_fiber_t *fiber)
918 {
919 #if VM_CHECK_MODE > 0
920  VM_ASSERT(fiber->cont.saved_ec.fiber_ptr == fiber);
921 
922  switch (fiber->status) {
923  case FIBER_RESUMED:
924  VM_ASSERT(fiber->cont.saved_ec.vm_stack != NULL);
925  break;
926  case FIBER_SUSPENDED:
927  VM_ASSERT(fiber->cont.saved_ec.vm_stack != NULL);
928  break;
929  case FIBER_CREATED:
930  case FIBER_TERMINATED:
931  /* TODO */
932  break;
933  default:
934  VM_UNREACHABLE(fiber_verify);
935  }
936 #endif
937 }
938 
939 inline static void
940 fiber_status_set(rb_fiber_t *fiber, enum fiber_status s)
941 {
942  // if (DEBUG) fprintf(stderr, "fiber: %p, status: %s -> %s\n", (void *)fiber, fiber_status_name(fiber->status), fiber_status_name(s));
943  VM_ASSERT(!FIBER_TERMINATED_P(fiber));
944  VM_ASSERT(fiber->status != s);
945  fiber_verify(fiber);
946  fiber->status = s;
947 }
948 
949 static rb_context_t *
950 cont_ptr(VALUE obj)
951 {
952  rb_context_t *cont;
953 
954  TypedData_Get_Struct(obj, rb_context_t, &cont_data_type, cont);
955 
956  return cont;
957 }
958 
959 static rb_fiber_t *
960 fiber_ptr(VALUE obj)
961 {
962  rb_fiber_t *fiber;
963 
964  TypedData_Get_Struct(obj, rb_fiber_t, &fiber_data_type, fiber);
965  if (!fiber) rb_raise(rb_eFiberError, "uninitialized fiber");
966 
967  return fiber;
968 }
969 
970 NOINLINE(static VALUE cont_capture(volatile int *volatile stat));
971 
972 #define THREAD_MUST_BE_RUNNING(th) do { \
973  if (!(th)->ec->tag) rb_raise(rb_eThreadError, "not running thread"); \
974  } while (0)
975 
977 rb_fiber_threadptr(const rb_fiber_t *fiber)
978 {
979  return fiber->cont.saved_ec.thread_ptr;
980 }
981 
982 static VALUE
983 cont_thread_value(const rb_context_t *cont)
984 {
985  return cont->saved_ec.thread_ptr->self;
986 }
987 
988 static void
989 cont_compact(void *ptr)
990 {
991  rb_context_t *cont = ptr;
992 
993  if (cont->self) {
994  cont->self = rb_gc_location(cont->self);
995  }
996  cont->value = rb_gc_location(cont->value);
997  rb_execution_context_update(&cont->saved_ec);
998 }
999 
1000 static void
1001 cont_mark(void *ptr)
1002 {
1003  rb_context_t *cont = ptr;
1004 
1005  RUBY_MARK_ENTER("cont");
1006  if (cont->self) {
1007  rb_gc_mark_movable(cont->self);
1008  }
1009  rb_gc_mark_movable(cont->value);
1010 
1011  rb_execution_context_mark(&cont->saved_ec);
1012  rb_gc_mark(cont_thread_value(cont));
1013 
1014  if (cont->saved_vm_stack.ptr) {
1015 #ifdef CAPTURE_JUST_VALID_VM_STACK
1016  rb_gc_mark_locations(cont->saved_vm_stack.ptr,
1017  cont->saved_vm_stack.ptr + cont->saved_vm_stack.slen + cont->saved_vm_stack.clen);
1018 #else
1019  rb_gc_mark_locations(cont->saved_vm_stack.ptr,
1020  cont->saved_vm_stack.ptr, cont->saved_ec.stack_size);
1021 #endif
1022  }
1023 
1024  if (cont->machine.stack) {
1025  if (cont->type == CONTINUATION_CONTEXT) {
1026  /* cont */
1027  rb_gc_mark_locations(cont->machine.stack,
1028  cont->machine.stack + cont->machine.stack_size);
1029  }
1030  else {
1031  /* fiber machine context is marked as part of rb_execution_context_mark, no need to
1032  * do anything here. */
1033  }
1034  }
1035 
1036  RUBY_MARK_LEAVE("cont");
1037 }
1038 
1039 #if 0
1040 static int
1041 fiber_is_root_p(const rb_fiber_t *fiber)
1042 {
1043  return fiber == fiber->cont.saved_ec.thread_ptr->root_fiber;
1044 }
1045 #endif
1046 
1047 static void jit_cont_free(struct rb_jit_cont *cont);
1048 
1049 static void
1050 cont_free(void *ptr)
1051 {
1052  rb_context_t *cont = ptr;
1053 
1054  RUBY_FREE_ENTER("cont");
1055 
1056  if (cont->type == CONTINUATION_CONTEXT) {
1057  ruby_xfree(cont->saved_ec.vm_stack);
1058  RUBY_FREE_UNLESS_NULL(cont->machine.stack);
1059  }
1060  else {
1061  rb_fiber_t *fiber = (rb_fiber_t*)cont;
1062  coroutine_destroy(&fiber->context);
1063  fiber_stack_release(fiber);
1064  }
1065 
1066  RUBY_FREE_UNLESS_NULL(cont->saved_vm_stack.ptr);
1067 
1068  VM_ASSERT(cont->jit_cont != NULL);
1069  jit_cont_free(cont->jit_cont);
1070  /* free rb_cont_t or rb_fiber_t */
1071  ruby_xfree(ptr);
1072  RUBY_FREE_LEAVE("cont");
1073 }
1074 
1075 static size_t
1076 cont_memsize(const void *ptr)
1077 {
1078  const rb_context_t *cont = ptr;
1079  size_t size = 0;
1080 
1081  size = sizeof(*cont);
1082  if (cont->saved_vm_stack.ptr) {
1083 #ifdef CAPTURE_JUST_VALID_VM_STACK
1084  size_t n = (cont->saved_vm_stack.slen + cont->saved_vm_stack.clen);
1085 #else
1086  size_t n = cont->saved_ec.vm_stack_size;
1087 #endif
1088  size += n * sizeof(*cont->saved_vm_stack.ptr);
1089  }
1090 
1091  if (cont->machine.stack) {
1092  size += cont->machine.stack_size * sizeof(*cont->machine.stack);
1093  }
1094 
1095  return size;
1096 }
1097 
1098 void
1099 rb_fiber_update_self(rb_fiber_t *fiber)
1100 {
1101  if (fiber->cont.self) {
1102  fiber->cont.self = rb_gc_location(fiber->cont.self);
1103  }
1104  else {
1105  rb_execution_context_update(&fiber->cont.saved_ec);
1106  }
1107 }
1108 
1109 void
1110 rb_fiber_mark_self(const rb_fiber_t *fiber)
1111 {
1112  if (fiber->cont.self) {
1113  rb_gc_mark_movable(fiber->cont.self);
1114  }
1115  else {
1116  rb_execution_context_mark(&fiber->cont.saved_ec);
1117  }
1118 }
1119 
1120 static void
1121 fiber_compact(void *ptr)
1122 {
1123  rb_fiber_t *fiber = ptr;
1124  fiber->first_proc = rb_gc_location(fiber->first_proc);
1125 
1126  if (fiber->prev) rb_fiber_update_self(fiber->prev);
1127 
1128  cont_compact(&fiber->cont);
1129  fiber_verify(fiber);
1130 }
1131 
1132 static void
1133 fiber_mark(void *ptr)
1134 {
1135  rb_fiber_t *fiber = ptr;
1136  RUBY_MARK_ENTER("cont");
1137  fiber_verify(fiber);
1138  rb_gc_mark_movable(fiber->first_proc);
1139  if (fiber->prev) rb_fiber_mark_self(fiber->prev);
1140  cont_mark(&fiber->cont);
1141  RUBY_MARK_LEAVE("cont");
1142 }
1143 
1144 static void
1145 fiber_free(void *ptr)
1146 {
1147  rb_fiber_t *fiber = ptr;
1148  RUBY_FREE_ENTER("fiber");
1149 
1150  if (DEBUG) fprintf(stderr, "fiber_free: %p[%p]\n", (void *)fiber, fiber->stack.base);
1151 
1152  if (fiber->cont.saved_ec.local_storage) {
1153  rb_id_table_free(fiber->cont.saved_ec.local_storage);
1154  }
1155 
1156  cont_free(&fiber->cont);
1157  RUBY_FREE_LEAVE("fiber");
1158 }
1159 
1160 static size_t
1161 fiber_memsize(const void *ptr)
1162 {
1163  const rb_fiber_t *fiber = ptr;
1164  size_t size = sizeof(*fiber);
1165  const rb_execution_context_t *saved_ec = &fiber->cont.saved_ec;
1166  const rb_thread_t *th = rb_ec_thread_ptr(saved_ec);
1167 
1168  /*
1169  * vm.c::thread_memsize already counts th->ec->local_storage
1170  */
1171  if (saved_ec->local_storage && fiber != th->root_fiber) {
1172  size += rb_id_table_memsize(saved_ec->local_storage);
1173  size += rb_obj_memsize_of(saved_ec->storage);
1174  }
1175 
1176  size += cont_memsize(&fiber->cont);
1177  return size;
1178 }
1179 
1180 VALUE
1182 {
1183  return RBOOL(rb_typeddata_is_kind_of(obj, &fiber_data_type));
1184 }
1185 
1186 static void
1187 cont_save_machine_stack(rb_thread_t *th, rb_context_t *cont)
1188 {
1189  size_t size;
1190 
1191  SET_MACHINE_STACK_END(&th->ec->machine.stack_end);
1192 
1193  if (th->ec->machine.stack_start > th->ec->machine.stack_end) {
1194  size = cont->machine.stack_size = th->ec->machine.stack_start - th->ec->machine.stack_end;
1195  cont->machine.stack_src = th->ec->machine.stack_end;
1196  }
1197  else {
1198  size = cont->machine.stack_size = th->ec->machine.stack_end - th->ec->machine.stack_start;
1199  cont->machine.stack_src = th->ec->machine.stack_start;
1200  }
1201 
1202  if (cont->machine.stack) {
1203  REALLOC_N(cont->machine.stack, VALUE, size);
1204  }
1205  else {
1206  cont->machine.stack = ALLOC_N(VALUE, size);
1207  }
1208 
1209  FLUSH_REGISTER_WINDOWS;
1210  asan_unpoison_memory_region(cont->machine.stack_src, size, false);
1211  MEMCPY(cont->machine.stack, cont->machine.stack_src, VALUE, size);
1212 }
1213 
1214 static const rb_data_type_t cont_data_type = {
1215  "continuation",
1216  {cont_mark, cont_free, cont_memsize, cont_compact},
1217  0, 0, RUBY_TYPED_FREE_IMMEDIATELY
1218 };
1219 
1220 static inline void
1221 cont_save_thread(rb_context_t *cont, rb_thread_t *th)
1222 {
1223  rb_execution_context_t *sec = &cont->saved_ec;
1224 
1225  VM_ASSERT(th->status == THREAD_RUNNABLE);
1226 
1227  /* save thread context */
1228  *sec = *th->ec;
1229 
1230  /* saved_ec->machine.stack_end should be NULL */
1231  /* because it may happen GC afterward */
1232  sec->machine.stack_end = NULL;
1233 }
1234 
1235 static rb_nativethread_lock_t jit_cont_lock;
1236 
1237 // Register a new continuation with execution context `ec`. Return JIT info about
1238 // the continuation.
1239 static struct rb_jit_cont *
1240 jit_cont_new(rb_execution_context_t *ec)
1241 {
1242  struct rb_jit_cont *cont;
1243 
1244  // We need to use calloc instead of something like ZALLOC to avoid triggering GC here.
1245  // When this function is called from rb_thread_alloc through rb_threadptr_root_fiber_setup,
1246  // the thread is still being prepared and marking it causes SEGV.
1247  cont = calloc(1, sizeof(struct rb_jit_cont));
1248  if (cont == NULL)
1249  rb_memerror();
1250  cont->ec = ec;
1251 
1252  rb_native_mutex_lock(&jit_cont_lock);
1253  if (first_jit_cont == NULL) {
1254  cont->next = cont->prev = NULL;
1255  }
1256  else {
1257  cont->prev = NULL;
1258  cont->next = first_jit_cont;
1259  first_jit_cont->prev = cont;
1260  }
1261  first_jit_cont = cont;
1262  rb_native_mutex_unlock(&jit_cont_lock);
1263 
1264  return cont;
1265 }
1266 
1267 // Unregister continuation `cont`.
1268 static void
1269 jit_cont_free(struct rb_jit_cont *cont)
1270 {
1271  if (!cont) return;
1272 
1273  rb_native_mutex_lock(&jit_cont_lock);
1274  if (cont == first_jit_cont) {
1275  first_jit_cont = cont->next;
1276  if (first_jit_cont != NULL)
1277  first_jit_cont->prev = NULL;
1278  }
1279  else {
1280  cont->prev->next = cont->next;
1281  if (cont->next != NULL)
1282  cont->next->prev = cont->prev;
1283  }
1284  rb_native_mutex_unlock(&jit_cont_lock);
1285 
1286  free(cont);
1287 }
1288 
1289 // Call a given callback against all on-stack ISEQs.
1290 void
1291 rb_jit_cont_each_iseq(rb_iseq_callback callback, void *data)
1292 {
1293  struct rb_jit_cont *cont;
1294  for (cont = first_jit_cont; cont != NULL; cont = cont->next) {
1295  if (cont->ec->vm_stack == NULL)
1296  continue;
1297 
1298  const rb_control_frame_t *cfp = cont->ec->cfp;
1299  while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) {
1300  if (cfp->pc && cfp->iseq && imemo_type((VALUE)cfp->iseq) == imemo_iseq) {
1301  callback(cfp->iseq, data);
1302  }
1303  cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1304  }
1305  }
1306 }
1307 
1308 #if USE_YJIT
1309 // Update the jit_return of all CFPs to leave_exit unless it's leave_exception or not set.
1310 // This prevents jit_exec_exception from jumping to the caller after invalidation.
1311 void
1312 rb_yjit_cancel_jit_return(void *leave_exit, void *leave_exception)
1313 {
1314  struct rb_jit_cont *cont;
1315  for (cont = first_jit_cont; cont != NULL; cont = cont->next) {
1316  if (cont->ec->vm_stack == NULL)
1317  continue;
1318 
1319  const rb_control_frame_t *cfp = cont->ec->cfp;
1320  while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) {
1321  if (cfp->jit_return && cfp->jit_return != leave_exception) {
1322  ((rb_control_frame_t *)cfp)->jit_return = leave_exit;
1323  }
1324  cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1325  }
1326  }
1327 }
1328 #endif
1329 
1330 // Finish working with jit_cont.
1331 void
1332 rb_jit_cont_finish(void)
1333 {
1334  struct rb_jit_cont *cont, *next;
1335  for (cont = first_jit_cont; cont != NULL; cont = next) {
1336  next = cont->next;
1337  free(cont); // Don't use xfree because it's allocated by calloc.
1338  }
1339  rb_native_mutex_destroy(&jit_cont_lock);
1340 }
1341 
1342 static void
1343 cont_init_jit_cont(rb_context_t *cont)
1344 {
1345  VM_ASSERT(cont->jit_cont == NULL);
1346  // We always allocate this since YJIT may be enabled later
1347  cont->jit_cont = jit_cont_new(&(cont->saved_ec));
1348 }
1349 
1351 rb_fiberptr_get_ec(struct rb_fiber_struct *fiber)
1352 {
1353  return &fiber->cont.saved_ec;
1354 }
1355 
1356 static void
1357 cont_init(rb_context_t *cont, rb_thread_t *th)
1358 {
1359  /* save thread context */
1360  cont_save_thread(cont, th);
1361  cont->saved_ec.thread_ptr = th;
1362  cont->saved_ec.local_storage = NULL;
1363  cont->saved_ec.local_storage_recursive_hash = Qnil;
1364  cont->saved_ec.local_storage_recursive_hash_for_trace = Qnil;
1365  cont_init_jit_cont(cont);
1366 }
1367 
1368 static rb_context_t *
1369 cont_new(VALUE klass)
1370 {
1371  rb_context_t *cont;
1372  volatile VALUE contval;
1373  rb_thread_t *th = GET_THREAD();
1374 
1375  THREAD_MUST_BE_RUNNING(th);
1376  contval = TypedData_Make_Struct(klass, rb_context_t, &cont_data_type, cont);
1377  cont->self = contval;
1378  cont_init(cont, th);
1379  return cont;
1380 }
1381 
1382 VALUE
1383 rb_fiberptr_self(struct rb_fiber_struct *fiber)
1384 {
1385  return fiber->cont.self;
1386 }
1387 
1388 unsigned int
1389 rb_fiberptr_blocking(struct rb_fiber_struct *fiber)
1390 {
1391  return fiber->blocking;
1392 }
1393 
1394 // Initialize the jit_cont_lock
1395 void
1396 rb_jit_cont_init(void)
1397 {
1398  rb_native_mutex_initialize(&jit_cont_lock);
1399 }
1400 
1401 #if 0
1402 void
1403 show_vm_stack(const rb_execution_context_t *ec)
1404 {
1405  VALUE *p = ec->vm_stack;
1406  while (p < ec->cfp->sp) {
1407  fprintf(stderr, "%3d ", (int)(p - ec->vm_stack));
1408  rb_obj_info_dump(*p);
1409  p++;
1410  }
1411 }
1412 
1413 void
1414 show_vm_pcs(const rb_control_frame_t *cfp,
1415  const rb_control_frame_t *end_of_cfp)
1416 {
1417  int i=0;
1418  while (cfp != end_of_cfp) {
1419  int pc = 0;
1420  if (cfp->iseq) {
1421  pc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded;
1422  }
1423  fprintf(stderr, "%2d pc: %d\n", i++, pc);
1424  cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1425  }
1426 }
1427 #endif
1428 
1429 static VALUE
1430 cont_capture(volatile int *volatile stat)
1431 {
1432  rb_context_t *volatile cont;
1433  rb_thread_t *th = GET_THREAD();
1434  volatile VALUE contval;
1435  const rb_execution_context_t *ec = th->ec;
1436 
1437  THREAD_MUST_BE_RUNNING(th);
1438  rb_vm_stack_to_heap(th->ec);
1439  cont = cont_new(rb_cContinuation);
1440  contval = cont->self;
1441 
1442 #ifdef CAPTURE_JUST_VALID_VM_STACK
1443  cont->saved_vm_stack.slen = ec->cfp->sp - ec->vm_stack;
1444  cont->saved_vm_stack.clen = ec->vm_stack + ec->vm_stack_size - (VALUE*)ec->cfp;
1445  cont->saved_vm_stack.ptr = ALLOC_N(VALUE, cont->saved_vm_stack.slen + cont->saved_vm_stack.clen);
1446  MEMCPY(cont->saved_vm_stack.ptr,
1447  ec->vm_stack,
1448  VALUE, cont->saved_vm_stack.slen);
1449  MEMCPY(cont->saved_vm_stack.ptr + cont->saved_vm_stack.slen,
1450  (VALUE*)ec->cfp,
1451  VALUE,
1452  cont->saved_vm_stack.clen);
1453 #else
1454  cont->saved_vm_stack.ptr = ALLOC_N(VALUE, ec->vm_stack_size);
1455  MEMCPY(cont->saved_vm_stack.ptr, ec->vm_stack, VALUE, ec->vm_stack_size);
1456 #endif
1457  // At this point, `cfp` is valid but `vm_stack` should be cleared:
1458  rb_ec_set_vm_stack(&cont->saved_ec, NULL, 0);
1459  VM_ASSERT(cont->saved_ec.cfp != NULL);
1460  cont_save_machine_stack(th, cont);
1461 
1462  if (ruby_setjmp(cont->jmpbuf)) {
1463  VALUE value;
1464 
1465  VAR_INITIALIZED(cont);
1466  value = cont->value;
1467  if (cont->argc == -1) rb_exc_raise(value);
1468  cont->value = Qnil;
1469  *stat = 1;
1470  return value;
1471  }
1472  else {
1473  *stat = 0;
1474  return contval;
1475  }
1476 }
1477 
1478 static inline void
1479 cont_restore_thread(rb_context_t *cont)
1480 {
1481  rb_thread_t *th = GET_THREAD();
1482 
1483  /* restore thread context */
1484  if (cont->type == CONTINUATION_CONTEXT) {
1485  /* continuation */
1486  rb_execution_context_t *sec = &cont->saved_ec;
1487  rb_fiber_t *fiber = NULL;
1488 
1489  if (sec->fiber_ptr != NULL) {
1490  fiber = sec->fiber_ptr;
1491  }
1492  else if (th->root_fiber) {
1493  fiber = th->root_fiber;
1494  }
1495 
1496  if (fiber && th->ec != &fiber->cont.saved_ec) {
1497  ec_switch(th, fiber);
1498  }
1499 
1500  if (th->ec->trace_arg != sec->trace_arg) {
1501  rb_raise(rb_eRuntimeError, "can't call across trace_func");
1502  }
1503 
1504  /* copy vm stack */
1505 #ifdef CAPTURE_JUST_VALID_VM_STACK
1506  MEMCPY(th->ec->vm_stack,
1507  cont->saved_vm_stack.ptr,
1508  VALUE, cont->saved_vm_stack.slen);
1509  MEMCPY(th->ec->vm_stack + th->ec->vm_stack_size - cont->saved_vm_stack.clen,
1510  cont->saved_vm_stack.ptr + cont->saved_vm_stack.slen,
1511  VALUE, cont->saved_vm_stack.clen);
1512 #else
1513  MEMCPY(th->ec->vm_stack, cont->saved_vm_stack.ptr, VALUE, sec->vm_stack_size);
1514 #endif
1515  /* other members of ec */
1516 
1517  th->ec->cfp = sec->cfp;
1518  th->ec->raised_flag = sec->raised_flag;
1519  th->ec->tag = sec->tag;
1520  th->ec->root_lep = sec->root_lep;
1521  th->ec->root_svar = sec->root_svar;
1522  th->ec->errinfo = sec->errinfo;
1523 
1524  VM_ASSERT(th->ec->vm_stack != NULL);
1525  }
1526  else {
1527  /* fiber */
1528  fiber_restore_thread(th, (rb_fiber_t*)cont);
1529  }
1530 }
1531 
1532 NOINLINE(static void fiber_setcontext(rb_fiber_t *new_fiber, rb_fiber_t *old_fiber));
1533 
1534 static void
1535 fiber_setcontext(rb_fiber_t *new_fiber, rb_fiber_t *old_fiber)
1536 {
1537  rb_thread_t *th = GET_THREAD();
1538 
1539  /* save old_fiber's machine stack - to ensure efficient garbage collection */
1540  if (!FIBER_TERMINATED_P(old_fiber)) {
1541  STACK_GROW_DIR_DETECTION;
1542  SET_MACHINE_STACK_END(&th->ec->machine.stack_end);
1543  if (STACK_DIR_UPPER(0, 1)) {
1544  old_fiber->cont.machine.stack_size = th->ec->machine.stack_start - th->ec->machine.stack_end;
1545  old_fiber->cont.machine.stack = th->ec->machine.stack_end;
1546  }
1547  else {
1548  old_fiber->cont.machine.stack_size = th->ec->machine.stack_end - th->ec->machine.stack_start;
1549  old_fiber->cont.machine.stack = th->ec->machine.stack_start;
1550  }
1551  }
1552 
1553  /* these values are used in rb_gc_mark_machine_context to mark the fiber's stack. */
1554  old_fiber->cont.saved_ec.machine.stack_start = th->ec->machine.stack_start;
1555  old_fiber->cont.saved_ec.machine.stack_end = FIBER_TERMINATED_P(old_fiber) ? NULL : th->ec->machine.stack_end;
1556 
1557 
1558  // if (DEBUG) fprintf(stderr, "fiber_setcontext: %p[%p] -> %p[%p]\n", (void*)old_fiber, old_fiber->stack.base, (void*)new_fiber, new_fiber->stack.base);
1559 
1560 #if defined(COROUTINE_SANITIZE_ADDRESS)
1561  __sanitizer_start_switch_fiber(FIBER_TERMINATED_P(old_fiber) ? NULL : &old_fiber->context.fake_stack, new_fiber->context.stack_base, new_fiber->context.stack_size);
1562 #endif
1563 
1564  /* swap machine context */
1565  struct coroutine_context * from = coroutine_transfer(&old_fiber->context, &new_fiber->context);
1566 
1567 #if defined(COROUTINE_SANITIZE_ADDRESS)
1568  __sanitizer_finish_switch_fiber(old_fiber->context.fake_stack, NULL, NULL);
1569 #endif
1570 
1571  if (from == NULL) {
1572  rb_syserr_fail(errno, "coroutine_transfer");
1573  }
1574 
1575  /* restore thread context */
1576  fiber_restore_thread(th, old_fiber);
1577 
1578  // It's possible to get here, and new_fiber is already freed.
1579  // if (DEBUG) fprintf(stderr, "fiber_setcontext: %p[%p] <- %p[%p]\n", (void*)old_fiber, old_fiber->stack.base, (void*)new_fiber, new_fiber->stack.base);
1580 }
1581 
1582 NOINLINE(NORETURN(static void cont_restore_1(rb_context_t *)));
1583 
1584 static void
1585 cont_restore_1(rb_context_t *cont)
1586 {
1587  cont_restore_thread(cont);
1588 
1589  /* restore machine stack */
1590 #if defined(_M_AMD64) && !defined(__MINGW64__)
1591  {
1592  /* workaround for x64 SEH */
1593  jmp_buf buf;
1594  setjmp(buf);
1595  _JUMP_BUFFER *bp = (void*)&cont->jmpbuf;
1596  bp->Frame = ((_JUMP_BUFFER*)((void*)&buf))->Frame;
1597  }
1598 #endif
1599  if (cont->machine.stack_src) {
1600  FLUSH_REGISTER_WINDOWS;
1601  MEMCPY(cont->machine.stack_src, cont->machine.stack,
1602  VALUE, cont->machine.stack_size);
1603  }
1604 
1605  ruby_longjmp(cont->jmpbuf, 1);
1606 }
1607 
1608 NORETURN(NOINLINE(static void cont_restore_0(rb_context_t *, VALUE *)));
1609 
1610 static void
1611 cont_restore_0(rb_context_t *cont, VALUE *addr_in_prev_frame)
1612 {
1613  if (cont->machine.stack_src) {
1614 #ifdef HAVE_ALLOCA
1615 #define STACK_PAD_SIZE 1
1616 #else
1617 #define STACK_PAD_SIZE 1024
1618 #endif
1619  VALUE space[STACK_PAD_SIZE];
1620 
1621 #if !STACK_GROW_DIRECTION
1622  if (addr_in_prev_frame > &space[0]) {
1623  /* Stack grows downward */
1624 #endif
1625 #if STACK_GROW_DIRECTION <= 0
1626  volatile VALUE *const end = cont->machine.stack_src;
1627  if (&space[0] > end) {
1628 # ifdef HAVE_ALLOCA
1629  volatile VALUE *sp = ALLOCA_N(VALUE, &space[0] - end);
1630  // We need to make sure that the stack pointer is moved,
1631  // but some compilers may remove the allocation by optimization.
1632  // We hope that the following read/write will prevent such an optimization.
1633  *sp = Qfalse;
1634  space[0] = *sp;
1635 # else
1636  cont_restore_0(cont, &space[0]);
1637 # endif
1638  }
1639 #endif
1640 #if !STACK_GROW_DIRECTION
1641  }
1642  else {
1643  /* Stack grows upward */
1644 #endif
1645 #if STACK_GROW_DIRECTION >= 0
1646  volatile VALUE *const end = cont->machine.stack_src + cont->machine.stack_size;
1647  if (&space[STACK_PAD_SIZE] < end) {
1648 # ifdef HAVE_ALLOCA
1649  volatile VALUE *sp = ALLOCA_N(VALUE, end - &space[STACK_PAD_SIZE]);
1650  space[0] = *sp;
1651 # else
1652  cont_restore_0(cont, &space[STACK_PAD_SIZE-1]);
1653 # endif
1654  }
1655 #endif
1656 #if !STACK_GROW_DIRECTION
1657  }
1658 #endif
1659  }
1660  cont_restore_1(cont);
1661 }
1662 
1663 /*
1664  * Document-class: Continuation
1665  *
1666  * Continuation objects are generated by Kernel#callcc,
1667  * after having +require+d <i>continuation</i>. They hold
1668  * a return address and execution context, allowing a nonlocal return
1669  * to the end of the #callcc block from anywhere within a
1670  * program. Continuations are somewhat analogous to a structured
1671  * version of C's <code>setjmp/longjmp</code> (although they contain
1672  * more state, so you might consider them closer to threads).
1673  *
1674  * For instance:
1675  *
1676  * require "continuation"
1677  * arr = [ "Freddie", "Herbie", "Ron", "Max", "Ringo" ]
1678  * callcc{|cc| $cc = cc}
1679  * puts(message = arr.shift)
1680  * $cc.call unless message =~ /Max/
1681  *
1682  * <em>produces:</em>
1683  *
1684  * Freddie
1685  * Herbie
1686  * Ron
1687  * Max
1688  *
1689  * Also you can call callcc in other methods:
1690  *
1691  * require "continuation"
1692  *
1693  * def g
1694  * arr = [ "Freddie", "Herbie", "Ron", "Max", "Ringo" ]
1695  * cc = callcc { |cc| cc }
1696  * puts arr.shift
1697  * return cc, arr.size
1698  * end
1699  *
1700  * def f
1701  * c, size = g
1702  * c.call(c) if size > 1
1703  * end
1704  *
1705  * f
1706  *
1707  * This (somewhat contrived) example allows the inner loop to abandon
1708  * processing early:
1709  *
1710  * require "continuation"
1711  * callcc {|cont|
1712  * for i in 0..4
1713  * print "#{i}: "
1714  * for j in i*5...(i+1)*5
1715  * cont.call() if j == 17
1716  * printf "%3d", j
1717  * end
1718  * end
1719  * }
1720  * puts
1721  *
1722  * <em>produces:</em>
1723  *
1724  * 0: 0 1 2 3 4
1725  * 1: 5 6 7 8 9
1726  * 2: 10 11 12 13 14
1727  * 3: 15 16
1728  */
1729 
1730 /*
1731  * call-seq:
1732  * callcc {|cont| block } -> obj
1733  *
1734  * Generates a Continuation object, which it passes to
1735  * the associated block. You need to <code>require
1736  * 'continuation'</code> before using this method. Performing a
1737  * <em>cont</em><code>.call</code> will cause the #callcc
1738  * to return (as will falling through the end of the block). The
1739  * value returned by the #callcc is the value of the
1740  * block, or the value passed to <em>cont</em><code>.call</code>. See
1741  * class Continuation for more details. Also see
1742  * Kernel#throw for an alternative mechanism for
1743  * unwinding a call stack.
1744  */
1745 
1746 static VALUE
1747 rb_callcc(VALUE self)
1748 {
1749  volatile int called;
1750  volatile VALUE val = cont_capture(&called);
1751 
1752  if (called) {
1753  return val;
1754  }
1755  else {
1756  return rb_yield(val);
1757  }
1758 }
1759 #ifdef RUBY_ASAN_ENABLED
1760 /* callcc can't possibly work with ASAN; see bug #20273. Also this function
1761  * definition below avoids a "defined and not used" warning. */
1762 MAYBE_UNUSED(static void notusing_callcc(void)) { rb_callcc(Qnil); }
1763 # define rb_callcc rb_f_notimplement
1764 #endif
1765 
1766 
1767 static VALUE
1768 make_passing_arg(int argc, const VALUE *argv)
1769 {
1770  switch (argc) {
1771  case -1:
1772  return argv[0];
1773  case 0:
1774  return Qnil;
1775  case 1:
1776  return argv[0];
1777  default:
1778  return rb_ary_new4(argc, argv);
1779  }
1780 }
1781 
1782 typedef VALUE e_proc(VALUE);
1783 
1784 NORETURN(static VALUE rb_cont_call(int argc, VALUE *argv, VALUE contval));
1785 
1786 /*
1787  * call-seq:
1788  * cont.call(args, ...)
1789  * cont[args, ...]
1790  *
1791  * Invokes the continuation. The program continues from the end of
1792  * the #callcc block. If no arguments are given, the original #callcc
1793  * returns +nil+. If one argument is given, #callcc returns
1794  * it. Otherwise, an array containing <i>args</i> is returned.
1795  *
1796  * callcc {|cont| cont.call } #=> nil
1797  * callcc {|cont| cont.call 1 } #=> 1
1798  * callcc {|cont| cont.call 1, 2, 3 } #=> [1, 2, 3]
1799  */
1800 
1801 static VALUE
1802 rb_cont_call(int argc, VALUE *argv, VALUE contval)
1803 {
1804  rb_context_t *cont = cont_ptr(contval);
1805  rb_thread_t *th = GET_THREAD();
1806 
1807  if (cont_thread_value(cont) != th->self) {
1808  rb_raise(rb_eRuntimeError, "continuation called across threads");
1809  }
1810  if (cont->saved_ec.fiber_ptr) {
1811  if (th->ec->fiber_ptr != cont->saved_ec.fiber_ptr) {
1812  rb_raise(rb_eRuntimeError, "continuation called across fiber");
1813  }
1814  }
1815 
1816  cont->argc = argc;
1817  cont->value = make_passing_arg(argc, argv);
1818 
1819  cont_restore_0(cont, &contval);
1821 }
1822 
1823 /*********/
1824 /* fiber */
1825 /*********/
1826 
1827 /*
1828  * Document-class: Fiber
1829  *
1830  * Fibers are primitives for implementing light weight cooperative
1831  * concurrency in Ruby. Basically they are a means of creating code blocks
1832  * that can be paused and resumed, much like threads. The main difference
1833  * is that they are never preempted and that the scheduling must be done by
1834  * the programmer and not the VM.
1835  *
1836  * As opposed to other stackless light weight concurrency models, each fiber
1837  * comes with a stack. This enables the fiber to be paused from deeply
1838  * nested function calls within the fiber block. See the ruby(1)
1839  * manpage to configure the size of the fiber stack(s).
1840  *
1841  * When a fiber is created it will not run automatically. Rather it must
1842  * be explicitly asked to run using the Fiber#resume method.
1843  * The code running inside the fiber can give up control by calling
1844  * Fiber.yield in which case it yields control back to caller (the
1845  * caller of the Fiber#resume).
1846  *
1847  * Upon yielding or termination the Fiber returns the value of the last
1848  * executed expression
1849  *
1850  * For instance:
1851  *
1852  * fiber = Fiber.new do
1853  * Fiber.yield 1
1854  * 2
1855  * end
1856  *
1857  * puts fiber.resume
1858  * puts fiber.resume
1859  * puts fiber.resume
1860  *
1861  * <em>produces</em>
1862  *
1863  * 1
1864  * 2
1865  * FiberError: dead fiber called
1866  *
1867  * The Fiber#resume method accepts an arbitrary number of parameters,
1868  * if it is the first call to #resume then they will be passed as
1869  * block arguments. Otherwise they will be the return value of the
1870  * call to Fiber.yield
1871  *
1872  * Example:
1873  *
1874  * fiber = Fiber.new do |first|
1875  * second = Fiber.yield first + 2
1876  * end
1877  *
1878  * puts fiber.resume 10
1879  * puts fiber.resume 1_000_000
1880  * puts fiber.resume "The fiber will be dead before I can cause trouble"
1881  *
1882  * <em>produces</em>
1883  *
1884  * 12
1885  * 1000000
1886  * FiberError: dead fiber called
1887  *
1888  * == Non-blocking Fibers
1889  *
1890  * The concept of <em>non-blocking fiber</em> was introduced in Ruby 3.0.
1891  * A non-blocking fiber, when reaching a operation that would normally block
1892  * the fiber (like <code>sleep</code>, or wait for another process or I/O)
1893  * will yield control to other fibers and allow the <em>scheduler</em> to
1894  * handle blocking and waking up (resuming) this fiber when it can proceed.
1895  *
1896  * For a Fiber to behave as non-blocking, it need to be created in Fiber.new with
1897  * <tt>blocking: false</tt> (which is the default), and Fiber.scheduler
1898  * should be set with Fiber.set_scheduler. If Fiber.scheduler is not set in
1899  * the current thread, blocking and non-blocking fibers' behavior is identical.
1900  *
1901  * Ruby doesn't provide a scheduler class: it is expected to be implemented by
1902  * the user and correspond to Fiber::Scheduler.
1903  *
1904  * There is also Fiber.schedule method, which is expected to immediately perform
1905  * the given block in a non-blocking manner. Its actual implementation is up to
1906  * the scheduler.
1907  *
1908  */
1909 
1910 static const rb_data_type_t fiber_data_type = {
1911  "fiber",
1912  {fiber_mark, fiber_free, fiber_memsize, fiber_compact,},
1913  0, 0, RUBY_TYPED_FREE_IMMEDIATELY
1914 };
1915 
1916 static VALUE
1917 fiber_alloc(VALUE klass)
1918 {
1919  return TypedData_Wrap_Struct(klass, &fiber_data_type, 0);
1920 }
1921 
1922 static rb_fiber_t*
1923 fiber_t_alloc(VALUE fiber_value, unsigned int blocking)
1924 {
1925  rb_fiber_t *fiber;
1926  rb_thread_t *th = GET_THREAD();
1927 
1928  if (DATA_PTR(fiber_value) != 0) {
1929  rb_raise(rb_eRuntimeError, "cannot initialize twice");
1930  }
1931 
1932  THREAD_MUST_BE_RUNNING(th);
1933  fiber = ZALLOC(rb_fiber_t);
1934  fiber->cont.self = fiber_value;
1935  fiber->cont.type = FIBER_CONTEXT;
1936  fiber->blocking = blocking;
1937  fiber->killed = 0;
1938  cont_init(&fiber->cont, th);
1939 
1940  fiber->cont.saved_ec.fiber_ptr = fiber;
1941  rb_ec_clear_vm_stack(&fiber->cont.saved_ec);
1942 
1943  fiber->prev = NULL;
1944 
1945  /* fiber->status == 0 == CREATED
1946  * So that we don't need to set status: fiber_status_set(fiber, FIBER_CREATED); */
1947  VM_ASSERT(FIBER_CREATED_P(fiber));
1948 
1949  DATA_PTR(fiber_value) = fiber;
1950 
1951  return fiber;
1952 }
1953 
1954 static rb_fiber_t *
1955 root_fiber_alloc(rb_thread_t *th)
1956 {
1957  VALUE fiber_value = fiber_alloc(rb_cFiber);
1958  rb_fiber_t *fiber = th->ec->fiber_ptr;
1959 
1960  VM_ASSERT(DATA_PTR(fiber_value) == NULL);
1961  VM_ASSERT(fiber->cont.type == FIBER_CONTEXT);
1962  VM_ASSERT(FIBER_RESUMED_P(fiber));
1963 
1964  th->root_fiber = fiber;
1965  DATA_PTR(fiber_value) = fiber;
1966  fiber->cont.self = fiber_value;
1967 
1968  coroutine_initialize_main(&fiber->context);
1969 
1970  return fiber;
1971 }
1972 
1973 static inline rb_fiber_t*
1974 fiber_current(void)
1975 {
1976  rb_execution_context_t *ec = GET_EC();
1977  if (ec->fiber_ptr->cont.self == 0) {
1978  root_fiber_alloc(rb_ec_thread_ptr(ec));
1979  }
1980  return ec->fiber_ptr;
1981 }
1982 
1983 static inline VALUE
1984 current_fiber_storage(void)
1985 {
1986  rb_execution_context_t *ec = GET_EC();
1987  return ec->storage;
1988 }
1989 
1990 static inline VALUE
1991 inherit_fiber_storage(void)
1992 {
1993  return rb_obj_dup(current_fiber_storage());
1994 }
1995 
1996 static inline void
1997 fiber_storage_set(struct rb_fiber_struct *fiber, VALUE storage)
1998 {
1999  fiber->cont.saved_ec.storage = storage;
2000 }
2001 
2002 static inline VALUE
2003 fiber_storage_get(rb_fiber_t *fiber, int allocate)
2004 {
2005  VALUE storage = fiber->cont.saved_ec.storage;
2006  if (storage == Qnil && allocate) {
2007  storage = rb_hash_new();
2008  fiber_storage_set(fiber, storage);
2009  }
2010  return storage;
2011 }
2012 
2013 static void
2014 storage_access_must_be_from_same_fiber(VALUE self)
2015 {
2016  rb_fiber_t *fiber = fiber_ptr(self);
2017  rb_fiber_t *current = fiber_current();
2018  if (fiber != current) {
2019  rb_raise(rb_eArgError, "Fiber storage can only be accessed from the Fiber it belongs to");
2020  }
2021 }
2022 
2029 static VALUE
2030 rb_fiber_storage_get(VALUE self)
2031 {
2032  storage_access_must_be_from_same_fiber(self);
2033 
2034  VALUE storage = fiber_storage_get(fiber_ptr(self), FALSE);
2035 
2036  if (storage == Qnil) {
2037  return Qnil;
2038  }
2039  else {
2040  return rb_obj_dup(storage);
2041  }
2042 }
2043 
2044 static int
2045 fiber_storage_validate_each(VALUE key, VALUE value, VALUE _argument)
2046 {
2047  Check_Type(key, T_SYMBOL);
2048 
2049  return ST_CONTINUE;
2050 }
2051 
2052 static void
2053 fiber_storage_validate(VALUE value)
2054 {
2055  // nil is an allowed value and will be lazily initialized.
2056  if (value == Qnil) return;
2057 
2058  if (!RB_TYPE_P(value, T_HASH)) {
2059  rb_raise(rb_eTypeError, "storage must be a hash");
2060  }
2061 
2062  if (RB_OBJ_FROZEN(value)) {
2063  rb_raise(rb_eFrozenError, "storage must not be frozen");
2064  }
2065 
2066  rb_hash_foreach(value, fiber_storage_validate_each, Qundef);
2067 }
2068 
2091 static VALUE
2092 rb_fiber_storage_set(VALUE self, VALUE value)
2093 {
2094  if (rb_warning_category_enabled_p(RB_WARN_CATEGORY_EXPERIMENTAL)) {
2096  "Fiber#storage= is experimental and may be removed in the future!");
2097  }
2098 
2099  storage_access_must_be_from_same_fiber(self);
2100  fiber_storage_validate(value);
2101 
2102  fiber_ptr(self)->cont.saved_ec.storage = rb_obj_dup(value);
2103  return value;
2104 }
2105 
2116 static VALUE
2117 rb_fiber_storage_aref(VALUE class, VALUE key)
2118 {
2119  Check_Type(key, T_SYMBOL);
2120 
2121  VALUE storage = fiber_storage_get(fiber_current(), FALSE);
2122  if (storage == Qnil) return Qnil;
2123 
2124  return rb_hash_aref(storage, key);
2125 }
2126 
2137 static VALUE
2138 rb_fiber_storage_aset(VALUE class, VALUE key, VALUE value)
2139 {
2140  Check_Type(key, T_SYMBOL);
2141 
2142  VALUE storage = fiber_storage_get(fiber_current(), value != Qnil);
2143  if (storage == Qnil) return Qnil;
2144 
2145  if (value == Qnil) {
2146  return rb_hash_delete(storage, key);
2147  }
2148  else {
2149  return rb_hash_aset(storage, key, value);
2150  }
2151 }
2152 
2153 static VALUE
2154 fiber_initialize(VALUE self, VALUE proc, struct fiber_pool * fiber_pool, unsigned int blocking, VALUE storage)
2155 {
2156  if (storage == Qundef || storage == Qtrue) {
2157  // The default, inherit storage (dup) from the current fiber:
2158  storage = inherit_fiber_storage();
2159  }
2160  else /* nil, hash, etc. */ {
2161  fiber_storage_validate(storage);
2162  storage = rb_obj_dup(storage);
2163  }
2164 
2165  rb_fiber_t *fiber = fiber_t_alloc(self, blocking);
2166 
2167  fiber->cont.saved_ec.storage = storage;
2168  fiber->first_proc = proc;
2169  fiber->stack.base = NULL;
2170  fiber->stack.pool = fiber_pool;
2171 
2172  return self;
2173 }
2174 
2175 static void
2176 fiber_prepare_stack(rb_fiber_t *fiber)
2177 {
2178  rb_context_t *cont = &fiber->cont;
2179  rb_execution_context_t *sec = &cont->saved_ec;
2180 
2181  size_t vm_stack_size = 0;
2182  VALUE *vm_stack = fiber_initialize_coroutine(fiber, &vm_stack_size);
2183 
2184  /* initialize cont */
2185  cont->saved_vm_stack.ptr = NULL;
2186  rb_ec_initialize_vm_stack(sec, vm_stack, vm_stack_size / sizeof(VALUE));
2187 
2188  sec->tag = NULL;
2189  sec->local_storage = NULL;
2190  sec->local_storage_recursive_hash = Qnil;
2191  sec->local_storage_recursive_hash_for_trace = Qnil;
2192 }
2193 
2194 static struct fiber_pool *
2195 rb_fiber_pool_default(VALUE pool)
2196 {
2197  return &shared_fiber_pool;
2198 }
2199 
2200 VALUE rb_fiber_inherit_storage(struct rb_execution_context_struct *ec, struct rb_fiber_struct *fiber)
2201 {
2202  VALUE storage = rb_obj_dup(ec->storage);
2203  fiber->cont.saved_ec.storage = storage;
2204  return storage;
2205 }
2206 
2207 /* :nodoc: */
2208 static VALUE
2209 rb_fiber_initialize_kw(int argc, VALUE* argv, VALUE self, int kw_splat)
2210 {
2211  VALUE pool = Qnil;
2212  VALUE blocking = Qfalse;
2213  VALUE storage = Qundef;
2214 
2215  if (kw_splat != RB_NO_KEYWORDS) {
2216  VALUE options = Qnil;
2217  VALUE arguments[3] = {Qundef};
2218 
2219  argc = rb_scan_args_kw(kw_splat, argc, argv, ":", &options);
2220  rb_get_kwargs(options, fiber_initialize_keywords, 0, 3, arguments);
2221 
2222  if (!UNDEF_P(arguments[0])) {
2223  blocking = arguments[0];
2224  }
2225 
2226  if (!UNDEF_P(arguments[1])) {
2227  pool = arguments[1];
2228  }
2229 
2230  storage = arguments[2];
2231  }
2232 
2233  return fiber_initialize(self, rb_block_proc(), rb_fiber_pool_default(pool), RTEST(blocking), storage);
2234 }
2235 
2236 /*
2237  * call-seq:
2238  * Fiber.new(blocking: false, storage: true) { |*args| ... } -> fiber
2239  *
2240  * Creates new Fiber. Initially, the fiber is not running and can be resumed
2241  * with #resume. Arguments to the first #resume call will be passed to the
2242  * block:
2243  *
2244  * f = Fiber.new do |initial|
2245  * current = initial
2246  * loop do
2247  * puts "current: #{current.inspect}"
2248  * current = Fiber.yield
2249  * end
2250  * end
2251  * f.resume(100) # prints: current: 100
2252  * f.resume(1, 2, 3) # prints: current: [1, 2, 3]
2253  * f.resume # prints: current: nil
2254  * # ... and so on ...
2255  *
2256  * If <tt>blocking: false</tt> is passed to <tt>Fiber.new</tt>, _and_ current
2257  * thread has a Fiber.scheduler defined, the Fiber becomes non-blocking (see
2258  * "Non-blocking Fibers" section in class docs).
2259  *
2260  * If the <tt>storage</tt> is unspecified, the default is to inherit a copy of
2261  * the storage from the current fiber. This is the same as specifying
2262  * <tt>storage: true</tt>.
2263  *
2264  * Fiber[:x] = 1
2265  * Fiber.new do
2266  * Fiber[:x] # => 1
2267  * Fiber[:x] = 2
2268  * end.resume
2269  * Fiber[:x] # => 1
2270  *
2271  * If the given <tt>storage</tt> is <tt>nil</tt>, this function will lazy
2272  * initialize the internal storage, which starts as an empty hash.
2273  *
2274  * Fiber[:x] = "Hello World"
2275  * Fiber.new(storage: nil) do
2276  * Fiber[:x] # nil
2277  * end
2278  *
2279  * Otherwise, the given <tt>storage</tt> is used as the new fiber's storage,
2280  * and it must be an instance of Hash.
2281  *
2282  * Explicitly using <tt>storage: true</tt> is currently experimental and may
2283  * change in the future.
2284  */
2285 static VALUE
2286 rb_fiber_initialize(int argc, VALUE* argv, VALUE self)
2287 {
2288  return rb_fiber_initialize_kw(argc, argv, self, rb_keyword_given_p());
2289 }
2290 
2291 VALUE
2293 {
2294  return fiber_initialize(fiber_alloc(rb_cFiber), rb_proc_new(func, obj), rb_fiber_pool_default(Qnil), 0, storage);
2295 }
2296 
2297 VALUE
2299 {
2300  return rb_fiber_new_storage(func, obj, Qtrue);
2301 }
2302 
2303 static VALUE
2304 rb_fiber_s_schedule_kw(int argc, VALUE* argv, int kw_splat)
2305 {
2306  rb_thread_t * th = GET_THREAD();
2307  VALUE scheduler = th->scheduler;
2308  VALUE fiber = Qnil;
2309 
2310  if (scheduler != Qnil) {
2311  fiber = rb_fiber_scheduler_fiber(scheduler, argc, argv, kw_splat);
2312  }
2313  else {
2314  rb_raise(rb_eRuntimeError, "No scheduler is available!");
2315  }
2316 
2317  return fiber;
2318 }
2319 
2320 /*
2321  * call-seq:
2322  * Fiber.schedule { |*args| ... } -> fiber
2323  *
2324  * The method is <em>expected</em> to immediately run the provided block of code in a
2325  * separate non-blocking fiber.
2326  *
2327  * puts "Go to sleep!"
2328  *
2329  * Fiber.set_scheduler(MyScheduler.new)
2330  *
2331  * Fiber.schedule do
2332  * puts "Going to sleep"
2333  * sleep(1)
2334  * puts "I slept well"
2335  * end
2336  *
2337  * puts "Wakey-wakey, sleepyhead"
2338  *
2339  * Assuming MyScheduler is properly implemented, this program will produce:
2340  *
2341  * Go to sleep!
2342  * Going to sleep
2343  * Wakey-wakey, sleepyhead
2344  * ...1 sec pause here...
2345  * I slept well
2346  *
2347  * ...e.g. on the first blocking operation inside the Fiber (<tt>sleep(1)</tt>),
2348  * the control is yielded to the outside code (main fiber), and <em>at the end
2349  * of that execution</em>, the scheduler takes care of properly resuming all the
2350  * blocked fibers.
2351  *
2352  * Note that the behavior described above is how the method is <em>expected</em>
2353  * to behave, actual behavior is up to the current scheduler's implementation of
2354  * Fiber::Scheduler#fiber method. Ruby doesn't enforce this method to
2355  * behave in any particular way.
2356  *
2357  * If the scheduler is not set, the method raises
2358  * <tt>RuntimeError (No scheduler is available!)</tt>.
2359  *
2360  */
2361 static VALUE
2362 rb_fiber_s_schedule(int argc, VALUE *argv, VALUE obj)
2363 {
2364  return rb_fiber_s_schedule_kw(argc, argv, rb_keyword_given_p());
2365 }
2366 
2367 /*
2368  * call-seq:
2369  * Fiber.scheduler -> obj or nil
2370  *
2371  * Returns the Fiber scheduler, that was last set for the current thread with Fiber.set_scheduler.
2372  * Returns +nil+ if no scheduler is set (which is the default), and non-blocking fibers'
2373  * behavior is the same as blocking.
2374  * (see "Non-blocking fibers" section in class docs for details about the scheduler concept).
2375  *
2376  */
2377 static VALUE
2378 rb_fiber_s_scheduler(VALUE klass)
2379 {
2380  return rb_fiber_scheduler_get();
2381 }
2382 
2383 /*
2384  * call-seq:
2385  * Fiber.current_scheduler -> obj or nil
2386  *
2387  * Returns the Fiber scheduler, that was last set for the current thread with Fiber.set_scheduler
2388  * if and only if the current fiber is non-blocking.
2389  *
2390  */
2391 static VALUE
2392 rb_fiber_current_scheduler(VALUE klass)
2393 {
2394  return rb_fiber_scheduler_current();
2395 }
2396 
2397 /*
2398  * call-seq:
2399  * Fiber.set_scheduler(scheduler) -> scheduler
2400  *
2401  * Sets the Fiber scheduler for the current thread. If the scheduler is set, non-blocking
2402  * fibers (created by Fiber.new with <tt>blocking: false</tt>, or by Fiber.schedule)
2403  * call that scheduler's hook methods on potentially blocking operations, and the current
2404  * thread will call scheduler's +close+ method on finalization (allowing the scheduler to
2405  * properly manage all non-finished fibers).
2406  *
2407  * +scheduler+ can be an object of any class corresponding to Fiber::Scheduler. Its
2408  * implementation is up to the user.
2409  *
2410  * See also the "Non-blocking fibers" section in class docs.
2411  *
2412  */
2413 static VALUE
2414 rb_fiber_set_scheduler(VALUE klass, VALUE scheduler)
2415 {
2416  return rb_fiber_scheduler_set(scheduler);
2417 }
2418 
2419 NORETURN(static void rb_fiber_terminate(rb_fiber_t *fiber, int need_interrupt, VALUE err));
2420 
2421 void
2422 rb_fiber_start(rb_fiber_t *fiber)
2423 {
2424  rb_thread_t * volatile th = fiber->cont.saved_ec.thread_ptr;
2425 
2426  rb_proc_t *proc;
2427  enum ruby_tag_type state;
2428 
2429  VM_ASSERT(th->ec == GET_EC());
2430  VM_ASSERT(FIBER_RESUMED_P(fiber));
2431 
2432  if (fiber->blocking) {
2433  th->blocking += 1;
2434  }
2435 
2436  EC_PUSH_TAG(th->ec);
2437  if ((state = EC_EXEC_TAG()) == TAG_NONE) {
2438  rb_context_t *cont = &VAR_FROM_MEMORY(fiber)->cont;
2439  int argc;
2440  const VALUE *argv, args = cont->value;
2441  GetProcPtr(fiber->first_proc, proc);
2442  argv = (argc = cont->argc) > 1 ? RARRAY_CONST_PTR(args) : &args;
2443  cont->value = Qnil;
2444  th->ec->errinfo = Qnil;
2445  th->ec->root_lep = rb_vm_proc_local_ep(fiber->first_proc);
2446  th->ec->root_svar = Qfalse;
2447 
2448  EXEC_EVENT_HOOK(th->ec, RUBY_EVENT_FIBER_SWITCH, th->self, 0, 0, 0, Qnil);
2449  cont->value = rb_vm_invoke_proc(th->ec, proc, argc, argv, cont->kw_splat, VM_BLOCK_HANDLER_NONE);
2450  }
2451  EC_POP_TAG();
2452 
2453  int need_interrupt = TRUE;
2454  VALUE err = Qfalse;
2455  if (state) {
2456  err = th->ec->errinfo;
2457  VM_ASSERT(FIBER_RESUMED_P(fiber));
2458 
2459  if (state == TAG_RAISE) {
2460  // noop...
2461  }
2462  else if (state == TAG_FATAL && err == RUBY_FATAL_FIBER_KILLED) {
2463  need_interrupt = FALSE;
2464  err = Qfalse;
2465  }
2466  else if (state == TAG_FATAL) {
2467  rb_threadptr_pending_interrupt_enque(th, err);
2468  }
2469  else {
2470  err = rb_vm_make_jump_tag_but_local_jump(state, err);
2471  }
2472  }
2473 
2474  rb_fiber_terminate(fiber, need_interrupt, err);
2475 }
2476 
2477 // Set up a "root fiber", which is the fiber that every Ractor has.
2478 void
2479 rb_threadptr_root_fiber_setup(rb_thread_t *th)
2480 {
2481  rb_fiber_t *fiber = ruby_mimcalloc(1, sizeof(rb_fiber_t));
2482  if (!fiber) {
2483  rb_bug("%s", strerror(errno)); /* ... is it possible to call rb_bug here? */
2484  }
2485  fiber->cont.type = FIBER_CONTEXT;
2486  fiber->cont.saved_ec.fiber_ptr = fiber;
2487  fiber->cont.saved_ec.thread_ptr = th;
2488  fiber->blocking = 1;
2489  fiber->killed = 0;
2490  fiber_status_set(fiber, FIBER_RESUMED); /* skip CREATED */
2491  th->ec = &fiber->cont.saved_ec;
2492  cont_init_jit_cont(&fiber->cont);
2493 }
2494 
2495 void
2496 rb_threadptr_root_fiber_release(rb_thread_t *th)
2497 {
2498  if (th->root_fiber) {
2499  /* ignore. A root fiber object will free th->ec */
2500  }
2501  else {
2502  rb_execution_context_t *ec = rb_current_execution_context(false);
2503 
2504  VM_ASSERT(th->ec->fiber_ptr->cont.type == FIBER_CONTEXT);
2505  VM_ASSERT(th->ec->fiber_ptr->cont.self == 0);
2506 
2507  if (ec && th->ec == ec) {
2508  rb_ractor_set_current_ec(th->ractor, NULL);
2509  }
2510  fiber_free(th->ec->fiber_ptr);
2511  th->ec = NULL;
2512  }
2513 }
2514 
2515 void
2516 rb_threadptr_root_fiber_terminate(rb_thread_t *th)
2517 {
2518  rb_fiber_t *fiber = th->ec->fiber_ptr;
2519 
2520  fiber->status = FIBER_TERMINATED;
2521 
2522  // The vm_stack is `alloca`ed on the thread stack, so it's gone too:
2523  rb_ec_clear_vm_stack(th->ec);
2524 }
2525 
2526 static inline rb_fiber_t*
2527 return_fiber(bool terminate)
2528 {
2529  rb_fiber_t *fiber = fiber_current();
2530  rb_fiber_t *prev = fiber->prev;
2531 
2532  if (prev) {
2533  fiber->prev = NULL;
2534  prev->resuming_fiber = NULL;
2535  return prev;
2536  }
2537  else {
2538  if (!terminate) {
2539  rb_raise(rb_eFiberError, "attempt to yield on a not resumed fiber");
2540  }
2541 
2542  rb_thread_t *th = GET_THREAD();
2543  rb_fiber_t *root_fiber = th->root_fiber;
2544 
2545  VM_ASSERT(root_fiber != NULL);
2546 
2547  // search resuming fiber
2548  for (fiber = root_fiber; fiber->resuming_fiber; fiber = fiber->resuming_fiber) {
2549  }
2550 
2551  return fiber;
2552  }
2553 }
2554 
2555 VALUE
2557 {
2558  return fiber_current()->cont.self;
2559 }
2560 
2561 // Prepare to execute next_fiber on the given thread.
2562 static inline void
2563 fiber_store(rb_fiber_t *next_fiber, rb_thread_t *th)
2564 {
2565  rb_fiber_t *fiber;
2566 
2567  if (th->ec->fiber_ptr != NULL) {
2568  fiber = th->ec->fiber_ptr;
2569  }
2570  else {
2571  /* create root fiber */
2572  fiber = root_fiber_alloc(th);
2573  }
2574 
2575  if (FIBER_CREATED_P(next_fiber)) {
2576  fiber_prepare_stack(next_fiber);
2577  }
2578 
2579  VM_ASSERT(FIBER_RESUMED_P(fiber) || FIBER_TERMINATED_P(fiber));
2580  VM_ASSERT(FIBER_RUNNABLE_P(next_fiber));
2581 
2582  if (FIBER_RESUMED_P(fiber)) fiber_status_set(fiber, FIBER_SUSPENDED);
2583 
2584  fiber_status_set(next_fiber, FIBER_RESUMED);
2585  fiber_setcontext(next_fiber, fiber);
2586 }
2587 
2588 static void
2589 fiber_check_killed(rb_fiber_t *fiber)
2590 {
2591  VM_ASSERT(fiber == fiber_current());
2592 
2593  if (fiber->killed) {
2594  rb_thread_t *thread = fiber->cont.saved_ec.thread_ptr;
2595 
2596  thread->ec->errinfo = RUBY_FATAL_FIBER_KILLED;
2597  EC_JUMP_TAG(thread->ec, RUBY_TAG_FATAL);
2598  }
2599 }
2600 
2601 static inline VALUE
2602 fiber_switch(rb_fiber_t *fiber, int argc, const VALUE *argv, int kw_splat, rb_fiber_t *resuming_fiber, bool yielding)
2603 {
2604  VALUE value;
2605  rb_context_t *cont = &fiber->cont;
2606  rb_thread_t *th = GET_THREAD();
2607 
2608  /* make sure the root_fiber object is available */
2609  if (th->root_fiber == NULL) root_fiber_alloc(th);
2610 
2611  if (th->ec->fiber_ptr == fiber) {
2612  /* ignore fiber context switch
2613  * because destination fiber is the same as current fiber
2614  */
2615  return make_passing_arg(argc, argv);
2616  }
2617 
2618  if (cont_thread_value(cont) != th->self) {
2619  rb_raise(rb_eFiberError, "fiber called across threads");
2620  }
2621 
2622  if (FIBER_TERMINATED_P(fiber)) {
2623  value = rb_exc_new2(rb_eFiberError, "dead fiber called");
2624 
2625  if (!FIBER_TERMINATED_P(th->ec->fiber_ptr)) {
2626  rb_exc_raise(value);
2627  VM_UNREACHABLE(fiber_switch);
2628  }
2629  else {
2630  /* th->ec->fiber_ptr is also dead => switch to root fiber */
2631  /* (this means we're being called from rb_fiber_terminate, */
2632  /* and the terminated fiber's return_fiber() is already dead) */
2633  VM_ASSERT(FIBER_SUSPENDED_P(th->root_fiber));
2634 
2635  cont = &th->root_fiber->cont;
2636  cont->argc = -1;
2637  cont->value = value;
2638 
2639  fiber_setcontext(th->root_fiber, th->ec->fiber_ptr);
2640 
2641  VM_UNREACHABLE(fiber_switch);
2642  }
2643  }
2644 
2645  VM_ASSERT(FIBER_RUNNABLE_P(fiber));
2646 
2647  rb_fiber_t *current_fiber = fiber_current();
2648 
2649  VM_ASSERT(!current_fiber->resuming_fiber);
2650 
2651  if (resuming_fiber) {
2652  current_fiber->resuming_fiber = resuming_fiber;
2653  fiber->prev = fiber_current();
2654  fiber->yielding = 0;
2655  }
2656 
2657  VM_ASSERT(!current_fiber->yielding);
2658  if (yielding) {
2659  current_fiber->yielding = 1;
2660  }
2661 
2662  if (current_fiber->blocking) {
2663  th->blocking -= 1;
2664  }
2665 
2666  cont->argc = argc;
2667  cont->kw_splat = kw_splat;
2668  cont->value = make_passing_arg(argc, argv);
2669 
2670  fiber_store(fiber, th);
2671 
2672  // We cannot free the stack until the pthread is joined:
2673 #ifndef COROUTINE_PTHREAD_CONTEXT
2674  if (resuming_fiber && FIBER_TERMINATED_P(fiber)) {
2675  fiber_stack_release(fiber);
2676  }
2677 #endif
2678 
2679  if (fiber_current()->blocking) {
2680  th->blocking += 1;
2681  }
2682 
2683  RUBY_VM_CHECK_INTS(th->ec);
2684 
2685  EXEC_EVENT_HOOK(th->ec, RUBY_EVENT_FIBER_SWITCH, th->self, 0, 0, 0, Qnil);
2686 
2687  current_fiber = th->ec->fiber_ptr;
2688  value = current_fiber->cont.value;
2689 
2690  fiber_check_killed(current_fiber);
2691 
2692  if (current_fiber->cont.argc == -1) {
2693  // Fiber#raise will trigger this path.
2694  rb_exc_raise(value);
2695  }
2696 
2697  return value;
2698 }
2699 
2700 VALUE
2701 rb_fiber_transfer(VALUE fiber_value, int argc, const VALUE *argv)
2702 {
2703  return fiber_switch(fiber_ptr(fiber_value), argc, argv, RB_NO_KEYWORDS, NULL, false);
2704 }
2705 
2706 /*
2707  * call-seq:
2708  * fiber.blocking? -> true or false
2709  *
2710  * Returns +true+ if +fiber+ is blocking and +false+ otherwise.
2711  * Fiber is non-blocking if it was created via passing <tt>blocking: false</tt>
2712  * to Fiber.new, or via Fiber.schedule.
2713  *
2714  * Note that, even if the method returns +false+, the fiber behaves differently
2715  * only if Fiber.scheduler is set in the current thread.
2716  *
2717  * See the "Non-blocking fibers" section in class docs for details.
2718  *
2719  */
2720 VALUE
2721 rb_fiber_blocking_p(VALUE fiber)
2722 {
2723  return RBOOL(fiber_ptr(fiber)->blocking);
2724 }
2725 
2726 static VALUE
2727 fiber_blocking_yield(VALUE fiber_value)
2728 {
2729  rb_fiber_t *fiber = fiber_ptr(fiber_value);
2730  rb_thread_t * volatile th = fiber->cont.saved_ec.thread_ptr;
2731 
2732  VM_ASSERT(fiber->blocking == 0);
2733 
2734  // fiber->blocking is `unsigned int : 1`, so we use it as a boolean:
2735  fiber->blocking = 1;
2736 
2737  // Once the fiber is blocking, and current, we increment the thread blocking state:
2738  th->blocking += 1;
2739 
2740  return rb_yield(fiber_value);
2741 }
2742 
2743 static VALUE
2744 fiber_blocking_ensure(VALUE fiber_value)
2745 {
2746  rb_fiber_t *fiber = fiber_ptr(fiber_value);
2747  rb_thread_t * volatile th = fiber->cont.saved_ec.thread_ptr;
2748 
2749  // We are no longer blocking:
2750  fiber->blocking = 0;
2751  th->blocking -= 1;
2752 
2753  return Qnil;
2754 }
2755 
2756 /*
2757  * call-seq:
2758  * Fiber.blocking{|fiber| ...} -> result
2759  *
2760  * Forces the fiber to be blocking for the duration of the block. Returns the
2761  * result of the block.
2762  *
2763  * See the "Non-blocking fibers" section in class docs for details.
2764  *
2765  */
2766 VALUE
2767 rb_fiber_blocking(VALUE class)
2768 {
2769  VALUE fiber_value = rb_fiber_current();
2770  rb_fiber_t *fiber = fiber_ptr(fiber_value);
2771 
2772  // If we are already blocking, this is essentially a no-op:
2773  if (fiber->blocking) {
2774  return rb_yield(fiber_value);
2775  }
2776  else {
2777  return rb_ensure(fiber_blocking_yield, fiber_value, fiber_blocking_ensure, fiber_value);
2778  }
2779 }
2780 
2781 /*
2782  * call-seq:
2783  * Fiber.blocking? -> false or 1
2784  *
2785  * Returns +false+ if the current fiber is non-blocking.
2786  * Fiber is non-blocking if it was created via passing <tt>blocking: false</tt>
2787  * to Fiber.new, or via Fiber.schedule.
2788  *
2789  * If the current Fiber is blocking, the method returns 1.
2790  * Future developments may allow for situations where larger integers
2791  * could be returned.
2792  *
2793  * Note that, even if the method returns +false+, Fiber behaves differently
2794  * only if Fiber.scheduler is set in the current thread.
2795  *
2796  * See the "Non-blocking fibers" section in class docs for details.
2797  *
2798  */
2799 static VALUE
2800 rb_fiber_s_blocking_p(VALUE klass)
2801 {
2802  rb_thread_t *thread = GET_THREAD();
2803  unsigned blocking = thread->blocking;
2804 
2805  if (blocking == 0)
2806  return Qfalse;
2807 
2808  return INT2NUM(blocking);
2809 }
2810 
2811 void
2812 rb_fiber_close(rb_fiber_t *fiber)
2813 {
2814  fiber_status_set(fiber, FIBER_TERMINATED);
2815 }
2816 
2817 static void
2818 rb_fiber_terminate(rb_fiber_t *fiber, int need_interrupt, VALUE error)
2819 {
2820  VALUE value = fiber->cont.value;
2821 
2822  VM_ASSERT(FIBER_RESUMED_P(fiber));
2823  rb_fiber_close(fiber);
2824 
2825  fiber->cont.machine.stack = NULL;
2826  fiber->cont.machine.stack_size = 0;
2827 
2828  rb_fiber_t *next_fiber = return_fiber(true);
2829 
2830  if (need_interrupt) RUBY_VM_SET_INTERRUPT(&next_fiber->cont.saved_ec);
2831 
2832  if (RTEST(error))
2833  fiber_switch(next_fiber, -1, &error, RB_NO_KEYWORDS, NULL, false);
2834  else
2835  fiber_switch(next_fiber, 1, &value, RB_NO_KEYWORDS, NULL, false);
2836  ruby_stop(0);
2837 }
2838 
2839 static VALUE
2840 fiber_resume_kw(rb_fiber_t *fiber, int argc, const VALUE *argv, int kw_splat)
2841 {
2842  rb_fiber_t *current_fiber = fiber_current();
2843 
2844  if (argc == -1 && FIBER_CREATED_P(fiber)) {
2845  rb_raise(rb_eFiberError, "cannot raise exception on unborn fiber");
2846  }
2847  else if (FIBER_TERMINATED_P(fiber)) {
2848  rb_raise(rb_eFiberError, "attempt to resume a terminated fiber");
2849  }
2850  else if (fiber == current_fiber) {
2851  rb_raise(rb_eFiberError, "attempt to resume the current fiber");
2852  }
2853  else if (fiber->prev != NULL) {
2854  rb_raise(rb_eFiberError, "attempt to resume a resumed fiber (double resume)");
2855  }
2856  else if (fiber->resuming_fiber) {
2857  rb_raise(rb_eFiberError, "attempt to resume a resuming fiber");
2858  }
2859  else if (fiber->prev == NULL &&
2860  (!fiber->yielding && fiber->status != FIBER_CREATED)) {
2861  rb_raise(rb_eFiberError, "attempt to resume a transferring fiber");
2862  }
2863 
2864  return fiber_switch(fiber, argc, argv, kw_splat, fiber, false);
2865 }
2866 
2867 VALUE
2868 rb_fiber_resume_kw(VALUE self, int argc, const VALUE *argv, int kw_splat)
2869 {
2870  return fiber_resume_kw(fiber_ptr(self), argc, argv, kw_splat);
2871 }
2872 
2873 VALUE
2874 rb_fiber_resume(VALUE self, int argc, const VALUE *argv)
2875 {
2876  return fiber_resume_kw(fiber_ptr(self), argc, argv, RB_NO_KEYWORDS);
2877 }
2878 
2879 VALUE
2880 rb_fiber_yield_kw(int argc, const VALUE *argv, int kw_splat)
2881 {
2882  return fiber_switch(return_fiber(false), argc, argv, kw_splat, NULL, true);
2883 }
2884 
2885 VALUE
2886 rb_fiber_yield(int argc, const VALUE *argv)
2887 {
2888  return fiber_switch(return_fiber(false), argc, argv, RB_NO_KEYWORDS, NULL, true);
2889 }
2890 
2891 void
2892 rb_fiber_reset_root_local_storage(rb_thread_t *th)
2893 {
2894  if (th->root_fiber && th->root_fiber != th->ec->fiber_ptr) {
2895  th->ec->local_storage = th->root_fiber->cont.saved_ec.local_storage;
2896  }
2897 }
2898 
2899 /*
2900  * call-seq:
2901  * fiber.alive? -> true or false
2902  *
2903  * Returns true if the fiber can still be resumed (or transferred
2904  * to). After finishing execution of the fiber block this method will
2905  * always return +false+.
2906  */
2907 VALUE
2909 {
2910  return RBOOL(!FIBER_TERMINATED_P(fiber_ptr(fiber_value)));
2911 }
2912 
2913 /*
2914  * call-seq:
2915  * fiber.resume(args, ...) -> obj
2916  *
2917  * Resumes the fiber from the point at which the last Fiber.yield was
2918  * called, or starts running it if it is the first call to
2919  * #resume. Arguments passed to resume will be the value of the
2920  * Fiber.yield expression or will be passed as block parameters to
2921  * the fiber's block if this is the first #resume.
2922  *
2923  * Alternatively, when resume is called it evaluates to the arguments passed
2924  * to the next Fiber.yield statement inside the fiber's block
2925  * or to the block value if it runs to completion without any
2926  * Fiber.yield
2927  */
2928 static VALUE
2929 rb_fiber_m_resume(int argc, VALUE *argv, VALUE fiber)
2930 {
2931  return rb_fiber_resume_kw(fiber, argc, argv, rb_keyword_given_p());
2932 }
2933 
2934 /*
2935  * call-seq:
2936  * fiber.backtrace -> array
2937  * fiber.backtrace(start) -> array
2938  * fiber.backtrace(start, count) -> array
2939  * fiber.backtrace(start..end) -> array
2940  *
2941  * Returns the current execution stack of the fiber. +start+, +count+ and +end+ allow
2942  * to select only parts of the backtrace.
2943  *
2944  * def level3
2945  * Fiber.yield
2946  * end
2947  *
2948  * def level2
2949  * level3
2950  * end
2951  *
2952  * def level1
2953  * level2
2954  * end
2955  *
2956  * f = Fiber.new { level1 }
2957  *
2958  * # It is empty before the fiber started
2959  * f.backtrace
2960  * #=> []
2961  *
2962  * f.resume
2963  *
2964  * f.backtrace
2965  * #=> ["test.rb:2:in `yield'", "test.rb:2:in `level3'", "test.rb:6:in `level2'", "test.rb:10:in `level1'", "test.rb:13:in `block in <main>'"]
2966  * p f.backtrace(1) # start from the item 1
2967  * #=> ["test.rb:2:in `level3'", "test.rb:6:in `level2'", "test.rb:10:in `level1'", "test.rb:13:in `block in <main>'"]
2968  * p f.backtrace(2, 2) # start from item 2, take 2
2969  * #=> ["test.rb:6:in `level2'", "test.rb:10:in `level1'"]
2970  * p f.backtrace(1..3) # take items from 1 to 3
2971  * #=> ["test.rb:2:in `level3'", "test.rb:6:in `level2'", "test.rb:10:in `level1'"]
2972  *
2973  * f.resume
2974  *
2975  * # It is nil after the fiber is finished
2976  * f.backtrace
2977  * #=> nil
2978  *
2979  */
2980 static VALUE
2981 rb_fiber_backtrace(int argc, VALUE *argv, VALUE fiber)
2982 {
2983  return rb_vm_backtrace(argc, argv, &fiber_ptr(fiber)->cont.saved_ec);
2984 }
2985 
2986 /*
2987  * call-seq:
2988  * fiber.backtrace_locations -> array
2989  * fiber.backtrace_locations(start) -> array
2990  * fiber.backtrace_locations(start, count) -> array
2991  * fiber.backtrace_locations(start..end) -> array
2992  *
2993  * Like #backtrace, but returns each line of the execution stack as a
2994  * Thread::Backtrace::Location. Accepts the same arguments as #backtrace.
2995  *
2996  * f = Fiber.new { Fiber.yield }
2997  * f.resume
2998  * loc = f.backtrace_locations.first
2999  * loc.label #=> "yield"
3000  * loc.path #=> "test.rb"
3001  * loc.lineno #=> 1
3002  *
3003  *
3004  */
3005 static VALUE
3006 rb_fiber_backtrace_locations(int argc, VALUE *argv, VALUE fiber)
3007 {
3008  return rb_vm_backtrace_locations(argc, argv, &fiber_ptr(fiber)->cont.saved_ec);
3009 }
3010 
3011 /*
3012  * call-seq:
3013  * fiber.transfer(args, ...) -> obj
3014  *
3015  * Transfer control to another fiber, resuming it from where it last
3016  * stopped or starting it if it was not resumed before. The calling
3017  * fiber will be suspended much like in a call to
3018  * Fiber.yield.
3019  *
3020  * The fiber which receives the transfer call treats it much like
3021  * a resume call. Arguments passed to transfer are treated like those
3022  * passed to resume.
3023  *
3024  * The two style of control passing to and from fiber (one is #resume and
3025  * Fiber::yield, another is #transfer to and from fiber) can't be freely
3026  * mixed.
3027  *
3028  * * If the Fiber's lifecycle had started with transfer, it will never
3029  * be able to yield or be resumed control passing, only
3030  * finish or transfer back. (It still can resume other fibers that
3031  * are allowed to be resumed.)
3032  * * If the Fiber's lifecycle had started with resume, it can yield
3033  * or transfer to another Fiber, but can receive control back only
3034  * the way compatible with the way it was given away: if it had
3035  * transferred, it only can be transferred back, and if it had
3036  * yielded, it only can be resumed back. After that, it again can
3037  * transfer or yield.
3038  *
3039  * If those rules are broken FiberError is raised.
3040  *
3041  * For an individual Fiber design, yield/resume is easier to use
3042  * (the Fiber just gives away control, it doesn't need to think
3043  * about who the control is given to), while transfer is more flexible
3044  * for complex cases, allowing to build arbitrary graphs of Fibers
3045  * dependent on each other.
3046  *
3047  *
3048  * Example:
3049  *
3050  * manager = nil # For local var to be visible inside worker block
3051  *
3052  * # This fiber would be started with transfer
3053  * # It can't yield, and can't be resumed
3054  * worker = Fiber.new { |work|
3055  * puts "Worker: starts"
3056  * puts "Worker: Performed #{work.inspect}, transferring back"
3057  * # Fiber.yield # this would raise FiberError: attempt to yield on a not resumed fiber
3058  * # manager.resume # this would raise FiberError: attempt to resume a resumed fiber (double resume)
3059  * manager.transfer(work.capitalize)
3060  * }
3061  *
3062  * # This fiber would be started with resume
3063  * # It can yield or transfer, and can be transferred
3064  * # back or resumed
3065  * manager = Fiber.new {
3066  * puts "Manager: starts"
3067  * puts "Manager: transferring 'something' to worker"
3068  * result = worker.transfer('something')
3069  * puts "Manager: worker returned #{result.inspect}"
3070  * # worker.resume # this would raise FiberError: attempt to resume a transferring fiber
3071  * Fiber.yield # this is OK, the fiber transferred from and to, now it can yield
3072  * puts "Manager: finished"
3073  * }
3074  *
3075  * puts "Starting the manager"
3076  * manager.resume
3077  * puts "Resuming the manager"
3078  * # manager.transfer # this would raise FiberError: attempt to transfer to a yielding fiber
3079  * manager.resume
3080  *
3081  * <em>produces</em>
3082  *
3083  * Starting the manager
3084  * Manager: starts
3085  * Manager: transferring 'something' to worker
3086  * Worker: starts
3087  * Worker: Performed "something", transferring back
3088  * Manager: worker returned "Something"
3089  * Resuming the manager
3090  * Manager: finished
3091  *
3092  */
3093 static VALUE
3094 rb_fiber_m_transfer(int argc, VALUE *argv, VALUE self)
3095 {
3096  return rb_fiber_transfer_kw(self, argc, argv, rb_keyword_given_p());
3097 }
3098 
3099 static VALUE
3100 fiber_transfer_kw(rb_fiber_t *fiber, int argc, const VALUE *argv, int kw_splat)
3101 {
3102  if (fiber->resuming_fiber) {
3103  rb_raise(rb_eFiberError, "attempt to transfer to a resuming fiber");
3104  }
3105 
3106  if (fiber->yielding) {
3107  rb_raise(rb_eFiberError, "attempt to transfer to a yielding fiber");
3108  }
3109 
3110  return fiber_switch(fiber, argc, argv, kw_splat, NULL, false);
3111 }
3112 
3113 VALUE
3114 rb_fiber_transfer_kw(VALUE self, int argc, const VALUE *argv, int kw_splat)
3115 {
3116  return fiber_transfer_kw(fiber_ptr(self), argc, argv, kw_splat);
3117 }
3118 
3119 /*
3120  * call-seq:
3121  * Fiber.yield(args, ...) -> obj
3122  *
3123  * Yields control back to the context that resumed the fiber, passing
3124  * along any arguments that were passed to it. The fiber will resume
3125  * processing at this point when #resume is called next.
3126  * Any arguments passed to the next #resume will be the value that
3127  * this Fiber.yield expression evaluates to.
3128  */
3129 static VALUE
3130 rb_fiber_s_yield(int argc, VALUE *argv, VALUE klass)
3131 {
3132  return rb_fiber_yield_kw(argc, argv, rb_keyword_given_p());
3133 }
3134 
3135 static VALUE
3136 fiber_raise(rb_fiber_t *fiber, VALUE exception)
3137 {
3138  if (fiber == fiber_current()) {
3139  rb_exc_raise(exception);
3140  }
3141  else if (fiber->resuming_fiber) {
3142  return fiber_raise(fiber->resuming_fiber, exception);
3143  }
3144  else if (FIBER_SUSPENDED_P(fiber) && !fiber->yielding) {
3145  return fiber_transfer_kw(fiber, -1, &exception, RB_NO_KEYWORDS);
3146  }
3147  else {
3148  return fiber_resume_kw(fiber, -1, &exception, RB_NO_KEYWORDS);
3149  }
3150 }
3151 
3152 VALUE
3153 rb_fiber_raise(VALUE fiber, int argc, const VALUE *argv)
3154 {
3155  VALUE exception = rb_make_exception(argc, argv);
3156 
3157  return fiber_raise(fiber_ptr(fiber), exception);
3158 }
3159 
3160 /*
3161  * call-seq:
3162  * fiber.raise -> obj
3163  * fiber.raise(string) -> obj
3164  * fiber.raise(exception [, string [, array]]) -> obj
3165  *
3166  * Raises an exception in the fiber at the point at which the last
3167  * +Fiber.yield+ was called. If the fiber has not been started or has
3168  * already run to completion, raises +FiberError+. If the fiber is
3169  * yielding, it is resumed. If it is transferring, it is transferred into.
3170  * But if it is resuming, raises +FiberError+.
3171  *
3172  * With no arguments, raises a +RuntimeError+. With a single +String+
3173  * argument, raises a +RuntimeError+ with the string as a message. Otherwise,
3174  * the first parameter should be the name of an +Exception+ class (or an
3175  * object that returns an +Exception+ object when sent an +exception+
3176  * message). The optional second parameter sets the message associated with
3177  * the exception, and the third parameter is an array of callback information.
3178  * Exceptions are caught by the +rescue+ clause of <code>begin...end</code>
3179  * blocks.
3180  *
3181  * Raises +FiberError+ if called on a Fiber belonging to another +Thread+.
3182  *
3183  * See Kernel#raise for more information.
3184  */
3185 static VALUE
3186 rb_fiber_m_raise(int argc, VALUE *argv, VALUE self)
3187 {
3188  return rb_fiber_raise(self, argc, argv);
3189 }
3190 
3191 /*
3192  * call-seq:
3193  * fiber.kill -> nil
3194  *
3195  * Terminates the fiber by raising an uncatchable exception.
3196  * It only terminates the given fiber and no other fiber, returning +nil+ to
3197  * another fiber if that fiber was calling #resume or #transfer.
3198  *
3199  * <tt>Fiber#kill</tt> only interrupts another fiber when it is in Fiber.yield.
3200  * If called on the current fiber then it raises that exception at the <tt>Fiber#kill</tt> call site.
3201  *
3202  * If the fiber has not been started, transition directly to the terminated state.
3203  *
3204  * If the fiber is already terminated, does nothing.
3205  *
3206  * Raises FiberError if called on a fiber belonging to another thread.
3207  */
3208 static VALUE
3209 rb_fiber_m_kill(VALUE self)
3210 {
3211  rb_fiber_t *fiber = fiber_ptr(self);
3212 
3213  if (fiber->killed) return Qfalse;
3214  fiber->killed = 1;
3215 
3216  if (fiber->status == FIBER_CREATED) {
3217  fiber->status = FIBER_TERMINATED;
3218  }
3219  else if (fiber->status != FIBER_TERMINATED) {
3220  if (fiber_current() == fiber) {
3221  fiber_check_killed(fiber);
3222  }
3223  else {
3224  fiber_raise(fiber_ptr(self), Qnil);
3225  }
3226  }
3227 
3228  return self;
3229 }
3230 
3231 /*
3232  * call-seq:
3233  * Fiber.current -> fiber
3234  *
3235  * Returns the current fiber. If you are not running in the context of
3236  * a fiber this method will return the root fiber.
3237  */
3238 static VALUE
3239 rb_fiber_s_current(VALUE klass)
3240 {
3241  return rb_fiber_current();
3242 }
3243 
3244 static VALUE
3245 fiber_to_s(VALUE fiber_value)
3246 {
3247  const rb_fiber_t *fiber = fiber_ptr(fiber_value);
3248  const rb_proc_t *proc;
3249  char status_info[0x20];
3250 
3251  if (fiber->resuming_fiber) {
3252  snprintf(status_info, 0x20, " (%s by resuming)", fiber_status_name(fiber->status));
3253  }
3254  else {
3255  snprintf(status_info, 0x20, " (%s)", fiber_status_name(fiber->status));
3256  }
3257 
3258  if (!rb_obj_is_proc(fiber->first_proc)) {
3259  VALUE str = rb_any_to_s(fiber_value);
3260  strlcat(status_info, ">", sizeof(status_info));
3261  rb_str_set_len(str, RSTRING_LEN(str)-1);
3262  rb_str_cat_cstr(str, status_info);
3263  return str;
3264  }
3265  GetProcPtr(fiber->first_proc, proc);
3266  return rb_block_to_s(fiber_value, &proc->block, status_info);
3267 }
3268 
3269 #ifdef HAVE_WORKING_FORK
3270 void
3271 rb_fiber_atfork(rb_thread_t *th)
3272 {
3273  if (th->root_fiber) {
3274  if (&th->root_fiber->cont.saved_ec != th->ec) {
3275  th->root_fiber = th->ec->fiber_ptr;
3276  }
3277  th->root_fiber->prev = 0;
3278  }
3279 }
3280 #endif
3281 
3282 #ifdef RB_EXPERIMENTAL_FIBER_POOL
3283 static void
3284 fiber_pool_free(void *ptr)
3285 {
3286  struct fiber_pool * fiber_pool = ptr;
3287  RUBY_FREE_ENTER("fiber_pool");
3288 
3289  fiber_pool_allocation_free(fiber_pool->allocations);
3291 
3292  RUBY_FREE_LEAVE("fiber_pool");
3293 }
3294 
3295 static size_t
3296 fiber_pool_memsize(const void *ptr)
3297 {
3298  const struct fiber_pool * fiber_pool = ptr;
3299  size_t size = sizeof(*fiber_pool);
3300 
3301  size += fiber_pool->count * fiber_pool->size;
3302 
3303  return size;
3304 }
3305 
3306 static const rb_data_type_t FiberPoolDataType = {
3307  "fiber_pool",
3308  {NULL, fiber_pool_free, fiber_pool_memsize,},
3309  0, 0, RUBY_TYPED_FREE_IMMEDIATELY
3310 };
3311 
3312 static VALUE
3313 fiber_pool_alloc(VALUE klass)
3314 {
3315  struct fiber_pool *fiber_pool;
3316 
3317  return TypedData_Make_Struct(klass, struct fiber_pool, &FiberPoolDataType, fiber_pool);
3318 }
3319 
3320 static VALUE
3321 rb_fiber_pool_initialize(int argc, VALUE* argv, VALUE self)
3322 {
3323  rb_thread_t *th = GET_THREAD();
3324  VALUE size = Qnil, count = Qnil, vm_stack_size = Qnil;
3325  struct fiber_pool * fiber_pool = NULL;
3326 
3327  // Maybe these should be keyword arguments.
3328  rb_scan_args(argc, argv, "03", &size, &count, &vm_stack_size);
3329 
3330  if (NIL_P(size)) {
3331  size = SIZET2NUM(th->vm->default_params.fiber_machine_stack_size);
3332  }
3333 
3334  if (NIL_P(count)) {
3335  count = INT2NUM(128);
3336  }
3337 
3338  if (NIL_P(vm_stack_size)) {
3339  vm_stack_size = SIZET2NUM(th->vm->default_params.fiber_vm_stack_size);
3340  }
3341 
3342  TypedData_Get_Struct(self, struct fiber_pool, &FiberPoolDataType, fiber_pool);
3343 
3344  fiber_pool_initialize(fiber_pool, NUM2SIZET(size), NUM2SIZET(count), NUM2SIZET(vm_stack_size));
3345 
3346  return self;
3347 }
3348 #endif
3349 
3350 /*
3351  * Document-class: FiberError
3352  *
3353  * Raised when an invalid operation is attempted on a Fiber, in
3354  * particular when attempting to call/resume a dead fiber,
3355  * attempting to yield from the root fiber, or calling a fiber across
3356  * threads.
3357  *
3358  * fiber = Fiber.new{}
3359  * fiber.resume #=> nil
3360  * fiber.resume #=> FiberError: dead fiber called
3361  */
3362 
3363 void
3364 Init_Cont(void)
3365 {
3366  rb_thread_t *th = GET_THREAD();
3367  size_t vm_stack_size = th->vm->default_params.fiber_vm_stack_size;
3368  size_t machine_stack_size = th->vm->default_params.fiber_machine_stack_size;
3369  size_t stack_size = machine_stack_size + vm_stack_size;
3370 
3371 #ifdef _WIN32
3372  SYSTEM_INFO info;
3373  GetSystemInfo(&info);
3374  pagesize = info.dwPageSize;
3375 #else /* not WIN32 */
3376  pagesize = sysconf(_SC_PAGESIZE);
3377 #endif
3378  SET_MACHINE_STACK_END(&th->ec->machine.stack_end);
3379 
3380  fiber_pool_initialize(&shared_fiber_pool, stack_size, FIBER_POOL_INITIAL_SIZE, vm_stack_size);
3381 
3382  fiber_initialize_keywords[0] = rb_intern_const("blocking");
3383  fiber_initialize_keywords[1] = rb_intern_const("pool");
3384  fiber_initialize_keywords[2] = rb_intern_const("storage");
3385 
3386  const char *fiber_shared_fiber_pool_free_stacks = getenv("RUBY_SHARED_FIBER_POOL_FREE_STACKS");
3387  if (fiber_shared_fiber_pool_free_stacks) {
3388  shared_fiber_pool.free_stacks = atoi(fiber_shared_fiber_pool_free_stacks);
3389 
3390  if (shared_fiber_pool.free_stacks < 0) {
3391  rb_warn("Setting RUBY_SHARED_FIBER_POOL_FREE_STACKS to a negative value is not allowed.");
3392  shared_fiber_pool.free_stacks = 0;
3393  }
3394 
3395  if (shared_fiber_pool.free_stacks > 1) {
3396  rb_warn("Setting RUBY_SHARED_FIBER_POOL_FREE_STACKS to a value greater than 1 is operating system specific, and may cause crashes.");
3397  }
3398  }
3399 
3400  rb_cFiber = rb_define_class("Fiber", rb_cObject);
3401  rb_define_alloc_func(rb_cFiber, fiber_alloc);
3402  rb_eFiberError = rb_define_class("FiberError", rb_eStandardError);
3403  rb_define_singleton_method(rb_cFiber, "yield", rb_fiber_s_yield, -1);
3404  rb_define_singleton_method(rb_cFiber, "current", rb_fiber_s_current, 0);
3405  rb_define_singleton_method(rb_cFiber, "blocking", rb_fiber_blocking, 0);
3406  rb_define_singleton_method(rb_cFiber, "[]", rb_fiber_storage_aref, 1);
3407  rb_define_singleton_method(rb_cFiber, "[]=", rb_fiber_storage_aset, 2);
3408 
3409  rb_define_method(rb_cFiber, "initialize", rb_fiber_initialize, -1);
3410  rb_define_method(rb_cFiber, "blocking?", rb_fiber_blocking_p, 0);
3411  rb_define_method(rb_cFiber, "storage", rb_fiber_storage_get, 0);
3412  rb_define_method(rb_cFiber, "storage=", rb_fiber_storage_set, 1);
3413  rb_define_method(rb_cFiber, "resume", rb_fiber_m_resume, -1);
3414  rb_define_method(rb_cFiber, "raise", rb_fiber_m_raise, -1);
3415  rb_define_method(rb_cFiber, "kill", rb_fiber_m_kill, 0);
3416  rb_define_method(rb_cFiber, "backtrace", rb_fiber_backtrace, -1);
3417  rb_define_method(rb_cFiber, "backtrace_locations", rb_fiber_backtrace_locations, -1);
3418  rb_define_method(rb_cFiber, "to_s", fiber_to_s, 0);
3419  rb_define_alias(rb_cFiber, "inspect", "to_s");
3420  rb_define_method(rb_cFiber, "transfer", rb_fiber_m_transfer, -1);
3421  rb_define_method(rb_cFiber, "alive?", rb_fiber_alive_p, 0);
3422 
3423  rb_define_singleton_method(rb_cFiber, "blocking?", rb_fiber_s_blocking_p, 0);
3424  rb_define_singleton_method(rb_cFiber, "scheduler", rb_fiber_s_scheduler, 0);
3425  rb_define_singleton_method(rb_cFiber, "set_scheduler", rb_fiber_set_scheduler, 1);
3426  rb_define_singleton_method(rb_cFiber, "current_scheduler", rb_fiber_current_scheduler, 0);
3427 
3428  rb_define_singleton_method(rb_cFiber, "schedule", rb_fiber_s_schedule, -1);
3429 
3430 #ifdef RB_EXPERIMENTAL_FIBER_POOL
3431  rb_cFiberPool = rb_define_class_under(rb_cFiber, "Pool", rb_cObject);
3432  rb_define_alloc_func(rb_cFiberPool, fiber_pool_alloc);
3433  rb_define_method(rb_cFiberPool, "initialize", rb_fiber_pool_initialize, -1);
3434 #endif
3435 
3436  rb_provide("fiber.so");
3437 }
3438 
3439 RUBY_SYMBOL_EXPORT_BEGIN
3440 
3441 void
3442 ruby_Init_Continuation_body(void)
3443 {
3444  rb_cContinuation = rb_define_class("Continuation", rb_cObject);
3445  rb_undef_alloc_func(rb_cContinuation);
3446  rb_undef_method(CLASS_OF(rb_cContinuation), "new");
3447  rb_define_method(rb_cContinuation, "call", rb_cont_call, -1);
3448  rb_define_method(rb_cContinuation, "[]", rb_cont_call, -1);
3449  rb_define_global_function("callcc", rb_callcc, 0);
3450 }
3451 
3452 RUBY_SYMBOL_EXPORT_END
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
Definition: cxxanyargs.hpp:685
#define RUBY_EVENT_FIBER_SWITCH
Encountered a Fiber#yield.
Definition: event.h:59
static bool RB_OBJ_FROZEN(VALUE obj)
Checks if an object is frozen.
Definition: fl_type.h:898
VALUE rb_define_class(const char *name, VALUE super)
Defines a top-level class.
Definition: class.c:980
VALUE rb_define_class_under(VALUE outer, const char *name, VALUE super)
Defines a class under the namespace of outer.
Definition: class.c:1012
void rb_define_alias(VALUE klass, const char *name1, const char *name2)
Defines an alias of a method.
Definition: class.c:2345
void rb_undef_method(VALUE klass, const char *name)
Defines an undef of a method.
Definition: class.c:2166
int rb_scan_args_kw(int kw_flag, int argc, const VALUE *argv, const char *fmt,...)
Identical to rb_scan_args(), except it also accepts kw_splat.
Definition: class.c:2648
int rb_scan_args(int argc, const VALUE *argv, const char *fmt,...)
Retrieves argument from argc and argv to given VALUE references according to the format string.
Definition: class.c:2635
void rb_define_method(VALUE klass, const char *name, VALUE(*func)(ANYARGS), int argc)
Defines a method.
Definition: class.c:2142
int rb_keyword_given_p(void)
Determines if the current method is given a keyword argument.
Definition: eval.c:929
int rb_get_kwargs(VALUE keyword_hash, const ID *table, int required, int optional, VALUE *values)
Keyword argument deconstructor.
Definition: class.c:2424
void rb_define_global_function(const char *name, VALUE(*func)(ANYARGS), int argc)
Defines a global function.
Definition: class.c:2339
#define REALLOC_N
Old name of RB_REALLOC_N.
Definition: memory.h:398
#define xfree
Old name of ruby_xfree.
Definition: xmalloc.h:58
#define Qundef
Old name of RUBY_Qundef.
#define UNREACHABLE_RETURN
Old name of RBIMPL_UNREACHABLE_RETURN.
Definition: assume.h:29
#define ZALLOC
Old name of RB_ZALLOC.
Definition: memory.h:397
#define CLASS_OF
Old name of rb_class_of.
Definition: globals.h:203
#define rb_ary_new4
Old name of rb_ary_new_from_values.
Definition: array.h:659
#define SIZET2NUM
Old name of RB_SIZE2NUM.
Definition: size_t.h:62
#define rb_exc_new2
Old name of rb_exc_new_cstr.
Definition: error.h:37
#define T_HASH
Old name of RUBY_T_HASH.
Definition: value_type.h:65
#define ALLOC_N
Old name of RB_ALLOC_N.
Definition: memory.h:394
#define Qtrue
Old name of RUBY_Qtrue.
#define INT2NUM
Old name of RB_INT2NUM.
Definition: int.h:43
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define NIL_P
Old name of RB_NIL_P.
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
Definition: value_type.h:80
#define NUM2SIZET
Old name of RB_NUM2SIZE.
Definition: size_t.h:61
void ruby_stop(int ex)
Calls ruby_cleanup() and exits the process.
Definition: eval.c:288
void rb_category_warn(rb_warning_category_t category, const char *fmt,...)
Identical to rb_category_warning(), except it reports unless $VERBOSE is nil.
Definition: error.c:476
void rb_raise(VALUE exc_class, const char *fmt,...)
Exception entry point.
Definition: error.c:3635
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
Definition: eval.c:676
int rb_typeddata_is_kind_of(VALUE obj, const rb_data_type_t *data_type)
Checks if the given object is of given kind.
Definition: error.c:1358
void rb_syserr_fail(int e, const char *mesg)
Raises appropriate exception that represents a C errno.
Definition: error.c:3748
void rb_bug(const char *fmt,...)
Interpreter panic switch.
Definition: error.c:1089
VALUE rb_eStandardError
StandardError exception.
Definition: error.c:1405
VALUE rb_eFrozenError
FrozenError exception.
Definition: error.c:1407
VALUE rb_eTypeError
TypeError exception.
Definition: error.c:1408
VALUE rb_eRuntimeError
RuntimeError exception.
Definition: error.c:1406
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition: error.c:466
VALUE rb_eArgError
ArgumentError exception.
Definition: error.c:1409
VALUE rb_ensure(VALUE(*b_proc)(VALUE), VALUE data1, VALUE(*e_proc)(VALUE), VALUE data2)
An equivalent to ensure clause.
Definition: eval.c:1045
@ RB_WARN_CATEGORY_EXPERIMENTAL
Warning is for experimental features.
Definition: error.h:51
VALUE rb_any_to_s(VALUE obj)
Generates a textual representation of the given object.
Definition: object.c:669
VALUE rb_obj_dup(VALUE obj)
Duplicates the given object.
Definition: object.c:576
void rb_gc_mark(VALUE obj)
Marks an object.
Definition: gc.c:2211
void rb_memerror(void)
Triggers out-of-memory error.
Definition: gc.c:4426
void rb_gc_mark_movable(VALUE obj)
Maybe this is the only function provided for C extensions to control the pinning of objects,...
Definition: gc.c:2193
VALUE rb_gc_location(VALUE obj)
Finds a new "location" of an object.
Definition: gc.c:3136
void rb_gc_mark_locations(const VALUE *start, const VALUE *end)
Marks objects between the two pointers.
Definition: gc.c:2277
VALUE rb_fiber_transfer_kw(VALUE fiber, int argc, const VALUE *argv, int kw_splat)
Identical to rb_fiber_transfer(), except you can specify how to handle the last element of the given ...
Definition: cont.c:3114
VALUE rb_fiber_new_storage(rb_block_call_func_t func, VALUE callback_obj, VALUE storage)
Creates a Fiber instance from a C-backended block with the specified storage.
Definition: cont.c:2292
VALUE rb_fiber_raise(VALUE fiber, int argc, const VALUE *argv)
Identical to rb_fiber_resume() but instead of resuming normal execution of the passed fiber,...
Definition: cont.c:3153
VALUE rb_fiber_current(void)
Queries the fiber which is calling this function.
Definition: cont.c:2556
VALUE rb_fiber_yield_kw(int argc, const VALUE *argv, int kw_splat)
Identical to rb_fiber_yield(), except you can specify how to handle the last element of the given arr...
Definition: cont.c:2880
VALUE rb_fiber_transfer(VALUE fiber, int argc, const VALUE *argv)
Transfers control to another fiber, resuming it from where it last stopped or starting it if it was n...
Definition: cont.c:2701
VALUE rb_fiber_resume_kw(VALUE fiber, int argc, const VALUE *argv, int kw_splat)
Identical to rb_fiber_resume(), except you can specify how to handle the last element of the given ar...
Definition: cont.c:2868
VALUE rb_fiber_alive_p(VALUE fiber)
Queries the liveness of the passed fiber.
Definition: cont.c:2908
VALUE rb_fiber_new(rb_block_call_func_t func, VALUE callback_obj)
Creates a Fiber instance from a C-backended block.
Definition: cont.c:2298
VALUE rb_obj_is_fiber(VALUE obj)
Queries if an object is a fiber.
Definition: cont.c:1181
VALUE rb_fiber_yield(int argc, const VALUE *argv)
Yields the control back to the point where the current fiber was resumed.
Definition: cont.c:2886
VALUE rb_fiber_resume(VALUE fiber, int argc, const VALUE *argv)
Resumes the execution of the passed fiber, either from the point at which the last rb_fiber_yield() w...
Definition: cont.c:2874
VALUE rb_make_exception(int argc, const VALUE *argv)
Constructs an exception object from the list of arguments, in a manner similar to Ruby's raise.
Definition: eval.c:883
void rb_hash_foreach(VALUE hash, int(*func)(VALUE key, VALUE val, VALUE arg), VALUE arg)
Iterates over a hash.
VALUE rb_hash_delete(VALUE hash, VALUE key)
Deletes the passed key from the passed hash table, if any.
Definition: hash.c:2353
VALUE rb_hash_aref(VALUE hash, VALUE key)
Queries the given key in the given hash table.
Definition: hash.c:2073
VALUE rb_hash_aset(VALUE hash, VALUE key, VALUE val)
Inserts or replaces ("upsert"s) the objects into the given hash table.
Definition: hash.c:2893
VALUE rb_hash_new(void)
Creates a new, empty hash object.
Definition: hash.c:1475
void rb_provide(const char *feature)
Declares that the given feature is already provided by someone else.
Definition: load.c:715
VALUE rb_block_proc(void)
Constructs a Proc object from implicitly passed components.
Definition: proc.c:813
VALUE rb_proc_new(rb_block_call_func_t func, VALUE callback_arg)
This is an rb_iterate() + rb_block_proc() combo.
Definition: proc.c:3332
VALUE rb_obj_is_proc(VALUE recv)
Queries if the given object is a proc.
Definition: proc.c:119
void rb_str_set_len(VALUE str, long len)
Overwrites the length of the string.
Definition: string.c:3267
VALUE rb_str_cat_cstr(VALUE dst, const char *src)
Identical to rb_str_cat(), except it assumes the passed pointer is a pointer to a C string.
Definition: string.c:3453
void rb_undef_alloc_func(VALUE klass)
Deletes the allocator function of a class.
Definition: vm_method.c:1291
void rb_define_alloc_func(VALUE klass, rb_alloc_func_t func)
Sets the allocator function of a class.
static ID rb_intern_const(const char *str)
This is a "tiny optimisation" over rb_intern().
Definition: symbol.h:277
char * ptr
Pointer to the underlying memory region, of at least capa bytes.
Definition: io.h:2
VALUE rb_yield(VALUE val)
Yields the block.
Definition: vm_eval.c:1354
rb_block_call_func * rb_block_call_func_t
Shorthand type that represents an iterator-written-in-C function pointer.
Definition: iterator.h:88
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition: memory.h:367
#define ALLOCA_N(type, n)
Definition: memory.h:287
#define RB_ALLOC(type)
Shorthand of RB_ALLOC_N with n=1.
Definition: memory.h:208
#define RARRAY_CONST_PTR
Just another name of rb_array_const_ptr.
Definition: rarray.h:52
#define DATA_PTR(obj)
Convenient getter macro.
Definition: rdata.h:67
static long RSTRING_LEN(VALUE str)
Queries the length of the string.
Definition: rstring.h:367
#define TypedData_Get_Struct(obj, type, data_type, sval)
Obtains a C struct from inside of a wrapper Ruby object.
Definition: rtypeddata.h:515
#define TypedData_Wrap_Struct(klass, data_type, sval)
Converts sval, a pointer to your struct, into a Ruby object.
Definition: rtypeddata.h:449
#define TypedData_Make_Struct(klass, type, data_type, sval)
Identical to TypedData_Wrap_Struct, except it allocates a new data region internally instead of takin...
Definition: rtypeddata.h:497
#define errno
Ractor-aware version of errno.
Definition: ruby.h:388
#define RB_NO_KEYWORDS
Do not pass keywords.
Definition: scan_args.h:69
Scheduler APIs.
VALUE rb_fiber_scheduler_current(void)
Identical to rb_fiber_scheduler_get(), except it also returns RUBY_Qnil in case of a blocking fiber.
Definition: scheduler.c:226
VALUE rb_fiber_scheduler_set(VALUE scheduler)
Destructively assigns the passed scheduler to that of the current thread that is calling this functio...
Definition: scheduler.c:187
VALUE rb_fiber_scheduler_get(void)
Queries the current scheduler of the current thread that is calling this function.
Definition: scheduler.c:141
VALUE rb_fiber_scheduler_fiber(VALUE scheduler, int argc, VALUE *argv, int kw_splat)
Create and schedule a non-blocking fiber.
Definition: scheduler.c:775
#define RTEST
This is an old name of RB_TEST.
This is the struct that holds necessary info for a struct.
Definition: rtypeddata.h:200
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition: value.h:52
uintptr_t VALUE
Type that represents a Ruby object.
Definition: value.h:40
static void Check_Type(VALUE v, enum ruby_value_type t)
Identical to RB_TYPE_P(), except it raises exceptions on predication failure.
Definition: value_type.h:433
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition: value_type.h:376
void ruby_xfree(void *ptr)
Deallocates a storage instance.
Definition: gc.c:4594