Ruby 3.5.0dev (2025-08-02 revision 30a20bc166bc37acd7dcb3788686df149c7f428a)
cont.c (30a20bc166bc37acd7dcb3788686df149c7f428a)
1/**********************************************************************
2
3 cont.c -
4
5 $Author$
6 created at: Thu May 23 09:03:43 2007
7
8 Copyright (C) 2007 Koichi Sasada
9
10**********************************************************************/
11
12#include "ruby/internal/config.h"
13
14#ifndef _WIN32
15#include <unistd.h>
16#include <sys/mman.h>
17#endif
18
19// On Solaris, madvise() is NOT declared for SUS (XPG4v2) or later,
20// but MADV_* macros are defined when __EXTENSIONS__ is defined.
21#ifdef NEED_MADVICE_PROTOTYPE_USING_CADDR_T
22#include <sys/types.h>
23extern int madvise(caddr_t, size_t, int);
24#endif
25
26#include COROUTINE_H
27
28#include "eval_intern.h"
29#include "internal.h"
30#include "internal/cont.h"
31#include "internal/thread.h"
32#include "internal/error.h"
33#include "internal/eval.h"
34#include "internal/gc.h"
35#include "internal/proc.h"
36#include "internal/sanitizers.h"
37#include "internal/warnings.h"
39#include "yjit.h"
40#include "vm_core.h"
41#include "vm_sync.h"
42#include "id_table.h"
43#include "ractor_core.h"
44
45static const int DEBUG = 0;
46
47#define RB_PAGE_SIZE (pagesize)
48#define RB_PAGE_MASK (~(RB_PAGE_SIZE - 1))
49static long pagesize;
50
51static const rb_data_type_t cont_data_type, fiber_data_type;
52static VALUE rb_cContinuation;
53static VALUE rb_cFiber;
54static VALUE rb_eFiberError;
55#ifdef RB_EXPERIMENTAL_FIBER_POOL
56static VALUE rb_cFiberPool;
57#endif
58
59#define CAPTURE_JUST_VALID_VM_STACK 1
60
61// Defined in `coroutine/$arch/Context.h`:
62#ifdef COROUTINE_LIMITED_ADDRESS_SPACE
63#define FIBER_POOL_ALLOCATION_FREE
64#define FIBER_POOL_INITIAL_SIZE 8
65#define FIBER_POOL_ALLOCATION_MAXIMUM_SIZE 32
66#else
67#define FIBER_POOL_INITIAL_SIZE 32
68#define FIBER_POOL_ALLOCATION_MAXIMUM_SIZE 1024
69#endif
70#ifdef RB_EXPERIMENTAL_FIBER_POOL
71#define FIBER_POOL_ALLOCATION_FREE
72#endif
73
74enum context_type {
75 CONTINUATION_CONTEXT = 0,
76 FIBER_CONTEXT = 1
77};
78
80 VALUE *ptr;
81#ifdef CAPTURE_JUST_VALID_VM_STACK
82 size_t slen; /* length of stack (head of ec->vm_stack) */
83 size_t clen; /* length of control frames (tail of ec->vm_stack) */
84#endif
85};
86
87struct fiber_pool;
88
89// Represents a single stack.
91 // A pointer to the memory allocation (lowest address) for the stack.
92 void * base;
93
94 // The current stack pointer, taking into account the direction of the stack.
95 void * current;
96
97 // The size of the stack excluding any guard pages.
98 size_t size;
99
100 // The available stack capacity w.r.t. the current stack offset.
101 size_t available;
102
103 // The pool this stack should be allocated from.
104 struct fiber_pool * pool;
105
106 // If the stack is allocated, the allocation it came from.
107 struct fiber_pool_allocation * allocation;
108};
109
110// A linked list of vacant (unused) stacks.
111// This structure is stored in the first page of a stack if it is not in use.
112// @sa fiber_pool_vacancy_pointer
114 // Details about the vacant stack:
115 struct fiber_pool_stack stack;
116
117 // The vacancy linked list.
118#ifdef FIBER_POOL_ALLOCATION_FREE
119 struct fiber_pool_vacancy * previous;
120#endif
121 struct fiber_pool_vacancy * next;
122};
123
124// Manages singly linked list of mapped regions of memory which contains 1 more more stack:
125//
126// base = +-------------------------------+-----------------------+ +
127// |VM Stack |VM Stack | | |
128// | | | | |
129// | | | | |
130// +-------------------------------+ | |
131// |Machine Stack |Machine Stack | | |
132// | | | | |
133// | | | | |
134// | | | . . . . | | size
135// | | | | |
136// | | | | |
137// | | | | |
138// | | | | |
139// | | | | |
140// +-------------------------------+ | |
141// |Guard Page |Guard Page | | |
142// +-------------------------------+-----------------------+ v
143//
144// +------------------------------------------------------->
145//
146// count
147//
149 // A pointer to the memory mapped region.
150 void * base;
151
152 // The size of the individual stacks.
153 size_t size;
154
155 // The stride of individual stacks (including any guard pages or other accounting details).
156 size_t stride;
157
158 // The number of stacks that were allocated.
159 size_t count;
160
161#ifdef FIBER_POOL_ALLOCATION_FREE
162 // The number of stacks used in this allocation.
163 size_t used;
164#endif
165
166 struct fiber_pool * pool;
167
168 // The allocation linked list.
169#ifdef FIBER_POOL_ALLOCATION_FREE
170 struct fiber_pool_allocation * previous;
171#endif
172 struct fiber_pool_allocation * next;
173};
174
175// A fiber pool manages vacant stacks to reduce the overhead of creating fibers.
177 // A singly-linked list of allocations which contain 1 or more stacks each.
178 struct fiber_pool_allocation * allocations;
179
180 // Free list that provides O(1) stack "allocation".
181 struct fiber_pool_vacancy * vacancies;
182
183 // The size of the stack allocations (excluding any guard page).
184 size_t size;
185
186 // The total number of stacks that have been allocated in this pool.
187 size_t count;
188
189 // The initial number of stacks to allocate.
190 size_t initial_count;
191
192 // Whether to madvise(free) the stack or not.
193 // If this value is set to 1, the stack will be madvise(free)ed
194 // (or equivalent), where possible, when it is returned to the pool.
195 int free_stacks;
196
197 // The number of stacks that have been used in this pool.
198 size_t used;
199
200 // The amount to allocate for the vm_stack.
201 size_t vm_stack_size;
202};
203
204// Continuation contexts used by JITs
206 rb_execution_context_t *ec; // continuation ec
207 struct rb_jit_cont *prev, *next; // used to form lists
208};
209
210// Doubly linked list for enumerating all on-stack ISEQs.
211static struct rb_jit_cont *first_jit_cont;
212
213typedef struct rb_context_struct {
214 enum context_type type;
215 int argc;
216 int kw_splat;
217 VALUE self;
218 VALUE value;
219
220 struct cont_saved_vm_stack saved_vm_stack;
221
222 struct {
223 VALUE *stack;
224 VALUE *stack_src;
225 size_t stack_size;
226 } machine;
227 rb_execution_context_t saved_ec;
228 rb_jmpbuf_t jmpbuf;
229 struct rb_jit_cont *jit_cont; // Continuation contexts for JITs
231
232/*
233 * Fiber status:
234 * [Fiber.new] ------> FIBER_CREATED ----> [Fiber#kill] --> |
235 * | [Fiber#resume] |
236 * v |
237 * +--> FIBER_RESUMED ----> [return] ------> |
238 * [Fiber#resume] | | [Fiber.yield/transfer] |
239 * [Fiber#transfer] | v |
240 * +--- FIBER_SUSPENDED --> [Fiber#kill] --> |
241 * |
242 * |
243 * FIBER_TERMINATED <-------------------+
244 */
245enum fiber_status {
246 FIBER_CREATED,
247 FIBER_RESUMED,
248 FIBER_SUSPENDED,
249 FIBER_TERMINATED
250};
251
252#define FIBER_CREATED_P(fiber) ((fiber)->status == FIBER_CREATED)
253#define FIBER_RESUMED_P(fiber) ((fiber)->status == FIBER_RESUMED)
254#define FIBER_SUSPENDED_P(fiber) ((fiber)->status == FIBER_SUSPENDED)
255#define FIBER_TERMINATED_P(fiber) ((fiber)->status == FIBER_TERMINATED)
256#define FIBER_RUNNABLE_P(fiber) (FIBER_CREATED_P(fiber) || FIBER_SUSPENDED_P(fiber))
257
259 rb_context_t cont;
260 VALUE first_proc;
261 struct rb_fiber_struct *prev;
262 struct rb_fiber_struct *resuming_fiber;
263
264 BITFIELD(enum fiber_status, status, 2);
265 /* Whether the fiber is allowed to implicitly yield. */
266 unsigned int yielding : 1;
267 unsigned int blocking : 1;
268
269 unsigned int killed : 1;
270
271 struct coroutine_context context;
272 struct fiber_pool_stack stack;
273};
274
275static struct fiber_pool shared_fiber_pool = {NULL, NULL, 0, 0, 0, 0};
276
277void
278rb_free_shared_fiber_pool(void)
279{
280 struct fiber_pool_allocation *allocations = shared_fiber_pool.allocations;
281 while (allocations) {
282 struct fiber_pool_allocation *next = allocations->next;
283 xfree(allocations);
284 allocations = next;
285 }
286}
287
288static ID fiber_initialize_keywords[3] = {0};
289
290/*
291 * FreeBSD require a first (i.e. addr) argument of mmap(2) is not NULL
292 * if MAP_STACK is passed.
293 * https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=158755
294 */
295#if defined(MAP_STACK) && !defined(__FreeBSD__) && !defined(__FreeBSD_kernel__)
296#define FIBER_STACK_FLAGS (MAP_PRIVATE | MAP_ANON | MAP_STACK)
297#else
298#define FIBER_STACK_FLAGS (MAP_PRIVATE | MAP_ANON)
299#endif
300
301#define ERRNOMSG strerror(errno)
302
303// Locates the stack vacancy details for the given stack.
304inline static struct fiber_pool_vacancy *
305fiber_pool_vacancy_pointer(void * base, size_t size)
306{
307 STACK_GROW_DIR_DETECTION;
308
309 return (struct fiber_pool_vacancy *)(
310 (char*)base + STACK_DIR_UPPER(0, size - RB_PAGE_SIZE)
311 );
312}
313
314#if defined(COROUTINE_SANITIZE_ADDRESS)
315// Compute the base pointer for a vacant stack, for the area which can be poisoned.
316inline static void *
317fiber_pool_stack_poison_base(struct fiber_pool_stack * stack)
318{
319 STACK_GROW_DIR_DETECTION;
320
321 return (char*)stack->base + STACK_DIR_UPPER(RB_PAGE_SIZE, 0);
322}
323
324// Compute the size of the vacant stack, for the area that can be poisoned.
325inline static size_t
326fiber_pool_stack_poison_size(struct fiber_pool_stack * stack)
327{
328 return stack->size - RB_PAGE_SIZE;
329}
330#endif
331
332// Reset the current stack pointer and available size of the given stack.
333inline static void
334fiber_pool_stack_reset(struct fiber_pool_stack * stack)
335{
336 STACK_GROW_DIR_DETECTION;
337
338 stack->current = (char*)stack->base + STACK_DIR_UPPER(0, stack->size);
339 stack->available = stack->size;
340}
341
342// A pointer to the base of the current unused portion of the stack.
343inline static void *
344fiber_pool_stack_base(struct fiber_pool_stack * stack)
345{
346 STACK_GROW_DIR_DETECTION;
347
348 VM_ASSERT(stack->current);
349
350 return STACK_DIR_UPPER(stack->current, (char*)stack->current - stack->available);
351}
352
353// Allocate some memory from the stack. Used to allocate vm_stack inline with machine stack.
354// @sa fiber_initialize_coroutine
355inline static void *
356fiber_pool_stack_alloca(struct fiber_pool_stack * stack, size_t offset)
357{
358 STACK_GROW_DIR_DETECTION;
359
360 if (DEBUG) fprintf(stderr, "fiber_pool_stack_alloca(%p): %"PRIuSIZE"/%"PRIuSIZE"\n", (void*)stack, offset, stack->available);
361 VM_ASSERT(stack->available >= offset);
362
363 // The pointer to the memory being allocated:
364 void * pointer = STACK_DIR_UPPER(stack->current, (char*)stack->current - offset);
365
366 // Move the stack pointer:
367 stack->current = STACK_DIR_UPPER((char*)stack->current + offset, (char*)stack->current - offset);
368 stack->available -= offset;
369
370 return pointer;
371}
372
373// Reset the current stack pointer and available size of the given stack.
374inline static void
375fiber_pool_vacancy_reset(struct fiber_pool_vacancy * vacancy)
376{
377 fiber_pool_stack_reset(&vacancy->stack);
378
379 // Consume one page of the stack because it's used for the vacancy list:
380 fiber_pool_stack_alloca(&vacancy->stack, RB_PAGE_SIZE);
381}
382
383inline static struct fiber_pool_vacancy *
384fiber_pool_vacancy_push(struct fiber_pool_vacancy * vacancy, struct fiber_pool_vacancy * head)
385{
386 vacancy->next = head;
387
388#ifdef FIBER_POOL_ALLOCATION_FREE
389 if (head) {
390 head->previous = vacancy;
391 vacancy->previous = NULL;
392 }
393#endif
394
395 return vacancy;
396}
397
398#ifdef FIBER_POOL_ALLOCATION_FREE
399static void
400fiber_pool_vacancy_remove(struct fiber_pool_vacancy * vacancy)
401{
402 if (vacancy->next) {
403 vacancy->next->previous = vacancy->previous;
404 }
405
406 if (vacancy->previous) {
407 vacancy->previous->next = vacancy->next;
408 }
409 else {
410 // It's the head of the list:
411 vacancy->stack.pool->vacancies = vacancy->next;
412 }
413}
414
415inline static struct fiber_pool_vacancy *
416fiber_pool_vacancy_pop(struct fiber_pool * pool)
417{
418 struct fiber_pool_vacancy * vacancy = pool->vacancies;
419
420 if (vacancy) {
421 fiber_pool_vacancy_remove(vacancy);
422 }
423
424 return vacancy;
425}
426#else
427inline static struct fiber_pool_vacancy *
428fiber_pool_vacancy_pop(struct fiber_pool * pool)
429{
430 struct fiber_pool_vacancy * vacancy = pool->vacancies;
431
432 if (vacancy) {
433 pool->vacancies = vacancy->next;
434 }
435
436 return vacancy;
437}
438#endif
439
440// Initialize the vacant stack. The [base, size] allocation should not include the guard page.
441// @param base The pointer to the lowest address of the allocated memory.
442// @param size The size of the allocated memory.
443inline static struct fiber_pool_vacancy *
444fiber_pool_vacancy_initialize(struct fiber_pool * fiber_pool, struct fiber_pool_vacancy * vacancies, void * base, size_t size)
445{
446 struct fiber_pool_vacancy * vacancy = fiber_pool_vacancy_pointer(base, size);
447
448 vacancy->stack.base = base;
449 vacancy->stack.size = size;
450
451 fiber_pool_vacancy_reset(vacancy);
452
453 vacancy->stack.pool = fiber_pool;
454
455 return fiber_pool_vacancy_push(vacancy, vacancies);
456}
457
458// Allocate a maximum of count stacks, size given by stride.
459// @param count the number of stacks to allocate / were allocated.
460// @param stride the size of the individual stacks.
461// @return [void *] the allocated memory or NULL if allocation failed.
462inline static void *
463fiber_pool_allocate_memory(size_t * count, size_t stride)
464{
465 // We use a divide-by-2 strategy to try and allocate memory. We are trying
466 // to allocate `count` stacks. In normal situation, this won't fail. But
467 // if we ran out of address space, or we are allocating more memory than
468 // the system would allow (e.g. overcommit * physical memory + swap), we
469 // divide count by two and try again. This condition should only be
470 // encountered in edge cases, but we handle it here gracefully.
471 while (*count > 1) {
472#if defined(_WIN32)
473 void * base = VirtualAlloc(0, (*count)*stride, MEM_COMMIT, PAGE_READWRITE);
474
475 if (!base) {
476 *count = (*count) >> 1;
477 }
478 else {
479 return base;
480 }
481#else
482 errno = 0;
483 size_t mmap_size = (*count)*stride;
484 void * base = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, FIBER_STACK_FLAGS, -1, 0);
485
486 if (base == MAP_FAILED) {
487 // If the allocation fails, count = count / 2, and try again.
488 *count = (*count) >> 1;
489 }
490 else {
491 ruby_annotate_mmap(base, mmap_size, "Ruby:fiber_pool_allocate_memory");
492#if defined(MADV_FREE_REUSE)
493 // On Mac MADV_FREE_REUSE is necessary for the task_info api
494 // to keep the accounting accurate as possible when a page is marked as reusable
495 // it can possibly not occurring at first call thus re-iterating if necessary.
496 while (madvise(base, mmap_size, MADV_FREE_REUSE) == -1 && errno == EAGAIN);
497#endif
498 return base;
499 }
500#endif
501 }
502
503 return NULL;
504}
505
506// Given an existing fiber pool, expand it by the specified number of stacks.
507// @param count the maximum number of stacks to allocate.
508// @return the allocated fiber pool.
509// @sa fiber_pool_allocation_free
510static struct fiber_pool_allocation *
511fiber_pool_expand(struct fiber_pool * fiber_pool, size_t count)
512{
513 struct fiber_pool_allocation * allocation;
514 RB_VM_LOCK_ENTER();
515 {
516 STACK_GROW_DIR_DETECTION;
517
518 size_t size = fiber_pool->size;
519 size_t stride = size + RB_PAGE_SIZE;
520
521 // Allocate the memory required for the stacks:
522 void * base = fiber_pool_allocate_memory(&count, stride);
523
524 if (base == NULL) {
525 rb_raise(rb_eFiberError, "can't alloc machine stack to fiber (%"PRIuSIZE" x %"PRIuSIZE" bytes): %s", count, size, ERRNOMSG);
526 }
527
528 struct fiber_pool_vacancy * vacancies = fiber_pool->vacancies;
529 allocation = RB_ALLOC(struct fiber_pool_allocation);
530
531 // Initialize fiber pool allocation:
532 allocation->base = base;
533 allocation->size = size;
534 allocation->stride = stride;
535 allocation->count = count;
536#ifdef FIBER_POOL_ALLOCATION_FREE
537 allocation->used = 0;
538#endif
539 allocation->pool = fiber_pool;
540
541 if (DEBUG) {
542 fprintf(stderr, "fiber_pool_expand(%"PRIuSIZE"): %p, %"PRIuSIZE"/%"PRIuSIZE" x [%"PRIuSIZE":%"PRIuSIZE"]\n",
543 count, (void*)fiber_pool, fiber_pool->used, fiber_pool->count, size, fiber_pool->vm_stack_size);
544 }
545
546 // Iterate over all stacks, initializing the vacancy list:
547 for (size_t i = 0; i < count; i += 1) {
548 void * base = (char*)allocation->base + (stride * i);
549 void * page = (char*)base + STACK_DIR_UPPER(size, 0);
550#if defined(_WIN32)
551 DWORD old_protect;
552
553 if (!VirtualProtect(page, RB_PAGE_SIZE, PAGE_READWRITE | PAGE_GUARD, &old_protect)) {
554 VirtualFree(allocation->base, 0, MEM_RELEASE);
555 rb_raise(rb_eFiberError, "can't set a guard page: %s", ERRNOMSG);
556 }
557#elif defined(__wasi__)
558 // wasi-libc's mprotect emulation doesn't support PROT_NONE.
559 (void)page;
560#else
561 if (mprotect(page, RB_PAGE_SIZE, PROT_NONE) < 0) {
562 munmap(allocation->base, count*stride);
563 rb_raise(rb_eFiberError, "can't set a guard page: %s", ERRNOMSG);
564 }
565#endif
566
567 vacancies = fiber_pool_vacancy_initialize(
568 fiber_pool, vacancies,
569 (char*)base + STACK_DIR_UPPER(0, RB_PAGE_SIZE),
570 size
571 );
572
573#ifdef FIBER_POOL_ALLOCATION_FREE
574 vacancies->stack.allocation = allocation;
575#endif
576 }
577
578 // Insert the allocation into the head of the pool:
579 allocation->next = fiber_pool->allocations;
580
581#ifdef FIBER_POOL_ALLOCATION_FREE
582 if (allocation->next) {
583 allocation->next->previous = allocation;
584 }
585
586 allocation->previous = NULL;
587#endif
588
589 fiber_pool->allocations = allocation;
590 fiber_pool->vacancies = vacancies;
591 fiber_pool->count += count;
592 }
593 RB_VM_LOCK_LEAVE();
594
595 return allocation;
596}
597
598// Initialize the specified fiber pool with the given number of stacks.
599// @param vm_stack_size The size of the vm stack to allocate.
600static void
601fiber_pool_initialize(struct fiber_pool * fiber_pool, size_t size, size_t count, size_t vm_stack_size)
602{
603 VM_ASSERT(vm_stack_size < size);
604
605 fiber_pool->allocations = NULL;
606 fiber_pool->vacancies = NULL;
607 fiber_pool->size = ((size / RB_PAGE_SIZE) + 1) * RB_PAGE_SIZE;
608 fiber_pool->count = 0;
609 fiber_pool->initial_count = count;
610 fiber_pool->free_stacks = 1;
611 fiber_pool->used = 0;
612
613 fiber_pool->vm_stack_size = vm_stack_size;
614
615 fiber_pool_expand(fiber_pool, count);
616}
617
618#ifdef FIBER_POOL_ALLOCATION_FREE
619// Free the list of fiber pool allocations.
620static void
621fiber_pool_allocation_free(struct fiber_pool_allocation * allocation)
622{
623 STACK_GROW_DIR_DETECTION;
624
625 VM_ASSERT(allocation->used == 0);
626
627 if (DEBUG) fprintf(stderr, "fiber_pool_allocation_free: %p base=%p count=%"PRIuSIZE"\n", (void*)allocation, allocation->base, allocation->count);
628
629 size_t i;
630 for (i = 0; i < allocation->count; i += 1) {
631 void * base = (char*)allocation->base + (allocation->stride * i) + STACK_DIR_UPPER(0, RB_PAGE_SIZE);
632
633 struct fiber_pool_vacancy * vacancy = fiber_pool_vacancy_pointer(base, allocation->size);
634
635 // Pop the vacant stack off the free list:
636 fiber_pool_vacancy_remove(vacancy);
637 }
638
639#ifdef _WIN32
640 VirtualFree(allocation->base, 0, MEM_RELEASE);
641#else
642 munmap(allocation->base, allocation->stride * allocation->count);
643#endif
644
645 if (allocation->previous) {
646 allocation->previous->next = allocation->next;
647 }
648 else {
649 // We are the head of the list, so update the pool:
650 allocation->pool->allocations = allocation->next;
651 }
652
653 if (allocation->next) {
654 allocation->next->previous = allocation->previous;
655 }
656
657 allocation->pool->count -= allocation->count;
658
659 ruby_xfree(allocation);
660}
661#endif
662
663// Acquire a stack from the given fiber pool. If none are available, allocate more.
664static struct fiber_pool_stack
665fiber_pool_stack_acquire(struct fiber_pool * fiber_pool)
666{
667 struct fiber_pool_vacancy * vacancy ;
668 RB_VM_LOCK_ENTER();
669 {
670 vacancy = fiber_pool_vacancy_pop(fiber_pool);
671
672 if (DEBUG) fprintf(stderr, "fiber_pool_stack_acquire: %p used=%"PRIuSIZE"\n", (void*)fiber_pool->vacancies, fiber_pool->used);
673
674 if (!vacancy) {
675 const size_t maximum = FIBER_POOL_ALLOCATION_MAXIMUM_SIZE;
676 const size_t minimum = fiber_pool->initial_count;
677
678 size_t count = fiber_pool->count;
679 if (count > maximum) count = maximum;
680 if (count < minimum) count = minimum;
681
682 fiber_pool_expand(fiber_pool, count);
683
684 // The free list should now contain some stacks:
685 VM_ASSERT(fiber_pool->vacancies);
686
687 vacancy = fiber_pool_vacancy_pop(fiber_pool);
688 }
689
690 VM_ASSERT(vacancy);
691 VM_ASSERT(vacancy->stack.base);
692
693#if defined(COROUTINE_SANITIZE_ADDRESS)
694 __asan_unpoison_memory_region(fiber_pool_stack_poison_base(&vacancy->stack), fiber_pool_stack_poison_size(&vacancy->stack));
695#endif
696
697 // Take the top item from the free list:
698 fiber_pool->used += 1;
699
700#ifdef FIBER_POOL_ALLOCATION_FREE
701 vacancy->stack.allocation->used += 1;
702#endif
703
704 fiber_pool_stack_reset(&vacancy->stack);
705 }
706 RB_VM_LOCK_LEAVE();
707
708 return vacancy->stack;
709}
710
711// We advise the operating system that the stack memory pages are no longer being used.
712// This introduce some performance overhead but allows system to relaim memory when there is pressure.
713static inline void
714fiber_pool_stack_free(struct fiber_pool_stack * stack)
715{
716 void * base = fiber_pool_stack_base(stack);
717 size_t size = stack->available;
718
719 // If this is not true, the vacancy information will almost certainly be destroyed:
720 VM_ASSERT(size <= (stack->size - RB_PAGE_SIZE));
721
722 int advice = stack->pool->free_stacks >> 1;
723
724 if (DEBUG) fprintf(stderr, "fiber_pool_stack_free: %p+%"PRIuSIZE" [base=%p, size=%"PRIuSIZE"] advice=%d\n", base, size, stack->base, stack->size, advice);
725
726 // The pages being used by the stack can be returned back to the system.
727 // That doesn't change the page mapping, but it does allow the system to
728 // reclaim the physical memory.
729 // Since we no longer care about the data itself, we don't need to page
730 // out to disk, since that is costly. Not all systems support that, so
731 // we try our best to select the most efficient implementation.
732 // In addition, it's actually slightly desirable to not do anything here,
733 // but that results in higher memory usage.
734
735#ifdef __wasi__
736 // WebAssembly doesn't support madvise, so we just don't do anything.
737#elif VM_CHECK_MODE > 0 && defined(MADV_DONTNEED)
738 if (!advice) advice = MADV_DONTNEED;
739 // This immediately discards the pages and the memory is reset to zero.
740 madvise(base, size, advice);
741#elif defined(MADV_FREE_REUSABLE)
742 if (!advice) advice = MADV_FREE_REUSABLE;
743 // Darwin / macOS / iOS.
744 // Acknowledge the kernel down to the task info api we make this
745 // page reusable for future use.
746 // As for MADV_FREE_REUSABLE below we ensure in the rare occasions the task was not
747 // completed at the time of the call to re-iterate.
748 while (madvise(base, size, advice) == -1 && errno == EAGAIN);
749#elif defined(MADV_FREE)
750 if (!advice) advice = MADV_FREE;
751 // Recent Linux.
752 madvise(base, size, advice);
753#elif defined(MADV_DONTNEED)
754 if (!advice) advice = MADV_DONTNEED;
755 // Old Linux.
756 madvise(base, size, advice);
757#elif defined(POSIX_MADV_DONTNEED)
758 if (!advice) advice = POSIX_MADV_DONTNEED;
759 // Solaris?
760 posix_madvise(base, size, advice);
761#elif defined(_WIN32)
762 VirtualAlloc(base, size, MEM_RESET, PAGE_READWRITE);
763 // Not available in all versions of Windows.
764 //DiscardVirtualMemory(base, size);
765#endif
766
767#if defined(COROUTINE_SANITIZE_ADDRESS)
768 __asan_poison_memory_region(fiber_pool_stack_poison_base(stack), fiber_pool_stack_poison_size(stack));
769#endif
770}
771
772// Release and return a stack to the vacancy list.
773static void
774fiber_pool_stack_release(struct fiber_pool_stack * stack)
775{
776 struct fiber_pool * pool = stack->pool;
777 RB_VM_LOCK_ENTER();
778 {
779 struct fiber_pool_vacancy * vacancy = fiber_pool_vacancy_pointer(stack->base, stack->size);
780
781 if (DEBUG) fprintf(stderr, "fiber_pool_stack_release: %p used=%"PRIuSIZE"\n", stack->base, stack->pool->used);
782
783 // Copy the stack details into the vacancy area:
784 vacancy->stack = *stack;
785 // After this point, be careful about updating/using state in stack, since it's copied to the vacancy area.
786
787 // Reset the stack pointers and reserve space for the vacancy data:
788 fiber_pool_vacancy_reset(vacancy);
789
790 // Push the vacancy into the vancancies list:
791 pool->vacancies = fiber_pool_vacancy_push(vacancy, pool->vacancies);
792 pool->used -= 1;
793
794#ifdef FIBER_POOL_ALLOCATION_FREE
795 struct fiber_pool_allocation * allocation = stack->allocation;
796
797 allocation->used -= 1;
798
799 // Release address space and/or dirty memory:
800 if (allocation->used == 0) {
801 fiber_pool_allocation_free(allocation);
802 }
803 else if (stack->pool->free_stacks) {
804 fiber_pool_stack_free(&vacancy->stack);
805 }
806#else
807 // This is entirely optional, but clears the dirty flag from the stack
808 // memory, so it won't get swapped to disk when there is memory pressure:
809 if (stack->pool->free_stacks) {
810 fiber_pool_stack_free(&vacancy->stack);
811 }
812#endif
813 }
814 RB_VM_LOCK_LEAVE();
815}
816
817static inline void
818ec_switch(rb_thread_t *th, rb_fiber_t *fiber)
819{
820 rb_execution_context_t *ec = &fiber->cont.saved_ec;
821#ifdef RUBY_ASAN_ENABLED
822 ec->machine.asan_fake_stack_handle = asan_get_thread_fake_stack_handle();
823#endif
824 rb_ractor_set_current_ec(th->ractor, th->ec = ec);
825 // ruby_current_execution_context_ptr = th->ec = ec;
826
827 /*
828 * timer-thread may set trap interrupt on previous th->ec at any time;
829 * ensure we do not delay (or lose) the trap interrupt handling.
830 */
831 if (th->vm->ractor.main_thread == th &&
832 rb_signal_buff_size() > 0) {
833 RUBY_VM_SET_TRAP_INTERRUPT(ec);
834 }
835
836 VM_ASSERT(ec->fiber_ptr->cont.self == 0 || ec->vm_stack != NULL);
837}
838
839static inline void
840fiber_restore_thread(rb_thread_t *th, rb_fiber_t *fiber)
841{
842 ec_switch(th, fiber);
843 VM_ASSERT(th->ec->fiber_ptr == fiber);
844}
845
846#ifndef COROUTINE_DECL
847# define COROUTINE_DECL COROUTINE
848#endif
849NORETURN(static COROUTINE_DECL fiber_entry(struct coroutine_context * from, struct coroutine_context * to));
850static COROUTINE
851fiber_entry(struct coroutine_context * from, struct coroutine_context * to)
852{
853 rb_fiber_t *fiber = to->argument;
854
855#if defined(COROUTINE_SANITIZE_ADDRESS)
856 // Address sanitizer will copy the previous stack base and stack size into
857 // the "from" fiber. `coroutine_initialize_main` doesn't generally know the
858 // stack bounds (base + size). Therefore, the main fiber `stack_base` and
859 // `stack_size` will be NULL/0. It's specifically important in that case to
860 // get the (base+size) of the previous fiber and save it, so that later when
861 // we return to the main coroutine, we don't supply (NULL, 0) to
862 // __sanitizer_start_switch_fiber which royally messes up the internal state
863 // of ASAN and causes (sometimes) the following message:
864 // "WARNING: ASan is ignoring requested __asan_handle_no_return"
865 __sanitizer_finish_switch_fiber(to->fake_stack, (const void**)&from->stack_base, &from->stack_size);
866#endif
867
868 rb_thread_t *thread = fiber->cont.saved_ec.thread_ptr;
869
870#ifdef COROUTINE_PTHREAD_CONTEXT
871 ruby_thread_set_native(thread);
872#endif
873
874 fiber_restore_thread(thread, fiber);
875
876 rb_fiber_start(fiber);
877
878#ifndef COROUTINE_PTHREAD_CONTEXT
879 VM_UNREACHABLE(fiber_entry);
880#endif
881}
882
883// Initialize a fiber's coroutine's machine stack and vm stack.
884static VALUE *
885fiber_initialize_coroutine(rb_fiber_t *fiber, size_t * vm_stack_size)
886{
887 struct fiber_pool * fiber_pool = fiber->stack.pool;
888 rb_execution_context_t *sec = &fiber->cont.saved_ec;
889 void * vm_stack = NULL;
890
891 VM_ASSERT(fiber_pool != NULL);
892
893 fiber->stack = fiber_pool_stack_acquire(fiber_pool);
894 vm_stack = fiber_pool_stack_alloca(&fiber->stack, fiber_pool->vm_stack_size);
895 *vm_stack_size = fiber_pool->vm_stack_size;
896
897 coroutine_initialize(&fiber->context, fiber_entry, fiber_pool_stack_base(&fiber->stack), fiber->stack.available);
898
899 // The stack for this execution context is the one we allocated:
900 sec->machine.stack_start = fiber->stack.current;
901 sec->machine.stack_maxsize = fiber->stack.available;
902
903 fiber->context.argument = (void*)fiber;
904
905 return vm_stack;
906}
907
908// Release the stack from the fiber, it's execution context, and return it to
909// the fiber pool.
910static void
911fiber_stack_release(rb_fiber_t * fiber)
912{
913 rb_execution_context_t *ec = &fiber->cont.saved_ec;
914
915 if (DEBUG) fprintf(stderr, "fiber_stack_release: %p, stack.base=%p\n", (void*)fiber, fiber->stack.base);
916
917 // Return the stack back to the fiber pool if it wasn't already:
918 if (fiber->stack.base) {
919 fiber_pool_stack_release(&fiber->stack);
920 fiber->stack.base = NULL;
921 }
922
923 // The stack is no longer associated with this execution context:
924 rb_ec_clear_vm_stack(ec);
925}
926
927static const char *
928fiber_status_name(enum fiber_status s)
929{
930 switch (s) {
931 case FIBER_CREATED: return "created";
932 case FIBER_RESUMED: return "resumed";
933 case FIBER_SUSPENDED: return "suspended";
934 case FIBER_TERMINATED: return "terminated";
935 }
936 VM_UNREACHABLE(fiber_status_name);
937 return NULL;
938}
939
940static void
941fiber_verify(const rb_fiber_t *fiber)
942{
943#if VM_CHECK_MODE > 0
944 VM_ASSERT(fiber->cont.saved_ec.fiber_ptr == fiber);
945
946 switch (fiber->status) {
947 case FIBER_RESUMED:
948 VM_ASSERT(fiber->cont.saved_ec.vm_stack != NULL);
949 break;
950 case FIBER_SUSPENDED:
951 VM_ASSERT(fiber->cont.saved_ec.vm_stack != NULL);
952 break;
953 case FIBER_CREATED:
954 case FIBER_TERMINATED:
955 /* TODO */
956 break;
957 default:
958 VM_UNREACHABLE(fiber_verify);
959 }
960#endif
961}
962
963inline static void
964fiber_status_set(rb_fiber_t *fiber, enum fiber_status s)
965{
966 // if (DEBUG) fprintf(stderr, "fiber: %p, status: %s -> %s\n", (void *)fiber, fiber_status_name(fiber->status), fiber_status_name(s));
967 VM_ASSERT(!FIBER_TERMINATED_P(fiber));
968 VM_ASSERT(fiber->status != s);
969 fiber_verify(fiber);
970 fiber->status = s;
971}
972
973static rb_context_t *
974cont_ptr(VALUE obj)
975{
976 rb_context_t *cont;
977
978 TypedData_Get_Struct(obj, rb_context_t, &cont_data_type, cont);
979
980 return cont;
981}
982
983static rb_fiber_t *
984fiber_ptr(VALUE obj)
985{
986 rb_fiber_t *fiber;
987
988 TypedData_Get_Struct(obj, rb_fiber_t, &fiber_data_type, fiber);
989 if (!fiber) rb_raise(rb_eFiberError, "uninitialized fiber");
990
991 return fiber;
992}
993
994NOINLINE(static VALUE cont_capture(volatile int *volatile stat));
995
996#define THREAD_MUST_BE_RUNNING(th) do { \
997 if (!(th)->ec->tag) rb_raise(rb_eThreadError, "not running thread"); \
998 } while (0)
999
1001rb_fiber_threadptr(const rb_fiber_t *fiber)
1002{
1003 return fiber->cont.saved_ec.thread_ptr;
1004}
1005
1006static VALUE
1007cont_thread_value(const rb_context_t *cont)
1008{
1009 return cont->saved_ec.thread_ptr->self;
1010}
1011
1012static void
1013cont_compact(void *ptr)
1014{
1015 rb_context_t *cont = ptr;
1016
1017 if (cont->self) {
1018 cont->self = rb_gc_location(cont->self);
1019 }
1020 cont->value = rb_gc_location(cont->value);
1021 rb_execution_context_update(&cont->saved_ec);
1022}
1023
1024static void
1025cont_mark(void *ptr)
1026{
1027 rb_context_t *cont = ptr;
1028
1029 RUBY_MARK_ENTER("cont");
1030 if (cont->self) {
1031 rb_gc_mark_movable(cont->self);
1032 }
1033 rb_gc_mark_movable(cont->value);
1034
1035 rb_execution_context_mark(&cont->saved_ec);
1036 rb_gc_mark(cont_thread_value(cont));
1037
1038 if (cont->saved_vm_stack.ptr) {
1039#ifdef CAPTURE_JUST_VALID_VM_STACK
1040 rb_gc_mark_locations(cont->saved_vm_stack.ptr,
1041 cont->saved_vm_stack.ptr + cont->saved_vm_stack.slen + cont->saved_vm_stack.clen);
1042#else
1043 rb_gc_mark_locations(cont->saved_vm_stack.ptr,
1044 cont->saved_vm_stack.ptr, cont->saved_ec.stack_size);
1045#endif
1046 }
1047
1048 if (cont->machine.stack) {
1049 if (cont->type == CONTINUATION_CONTEXT) {
1050 /* cont */
1051 rb_gc_mark_locations(cont->machine.stack,
1052 cont->machine.stack + cont->machine.stack_size);
1053 }
1054 else {
1055 /* fiber machine context is marked as part of rb_execution_context_mark, no need to
1056 * do anything here. */
1057 }
1058 }
1059
1060 RUBY_MARK_LEAVE("cont");
1061}
1062
1063#if 0
1064static int
1065fiber_is_root_p(const rb_fiber_t *fiber)
1066{
1067 return fiber == fiber->cont.saved_ec.thread_ptr->root_fiber;
1068}
1069#endif
1070
1071static void jit_cont_free(struct rb_jit_cont *cont);
1072
1073static void
1074cont_free(void *ptr)
1075{
1076 rb_context_t *cont = ptr;
1077
1078 RUBY_FREE_ENTER("cont");
1079
1080 if (cont->type == CONTINUATION_CONTEXT) {
1081 ruby_xfree(cont->saved_ec.vm_stack);
1082 RUBY_FREE_UNLESS_NULL(cont->machine.stack);
1083 }
1084 else {
1085 rb_fiber_t *fiber = (rb_fiber_t*)cont;
1086 coroutine_destroy(&fiber->context);
1087 fiber_stack_release(fiber);
1088 }
1089
1090 RUBY_FREE_UNLESS_NULL(cont->saved_vm_stack.ptr);
1091
1092 VM_ASSERT(cont->jit_cont != NULL);
1093 jit_cont_free(cont->jit_cont);
1094 /* free rb_cont_t or rb_fiber_t */
1095 ruby_xfree(ptr);
1096 RUBY_FREE_LEAVE("cont");
1097}
1098
1099static size_t
1100cont_memsize(const void *ptr)
1101{
1102 const rb_context_t *cont = ptr;
1103 size_t size = 0;
1104
1105 size = sizeof(*cont);
1106 if (cont->saved_vm_stack.ptr) {
1107#ifdef CAPTURE_JUST_VALID_VM_STACK
1108 size_t n = (cont->saved_vm_stack.slen + cont->saved_vm_stack.clen);
1109#else
1110 size_t n = cont->saved_ec.vm_stack_size;
1111#endif
1112 size += n * sizeof(*cont->saved_vm_stack.ptr);
1113 }
1114
1115 if (cont->machine.stack) {
1116 size += cont->machine.stack_size * sizeof(*cont->machine.stack);
1117 }
1118
1119 return size;
1120}
1121
1122void
1123rb_fiber_update_self(rb_fiber_t *fiber)
1124{
1125 if (fiber->cont.self) {
1126 fiber->cont.self = rb_gc_location(fiber->cont.self);
1127 }
1128 else {
1129 rb_execution_context_update(&fiber->cont.saved_ec);
1130 }
1131}
1132
1133void
1134rb_fiber_mark_self(const rb_fiber_t *fiber)
1135{
1136 if (fiber->cont.self) {
1137 rb_gc_mark_movable(fiber->cont.self);
1138 }
1139 else {
1140 rb_execution_context_mark(&fiber->cont.saved_ec);
1141 }
1142}
1143
1144static void
1145fiber_compact(void *ptr)
1146{
1147 rb_fiber_t *fiber = ptr;
1148 fiber->first_proc = rb_gc_location(fiber->first_proc);
1149
1150 if (fiber->prev) rb_fiber_update_self(fiber->prev);
1151
1152 cont_compact(&fiber->cont);
1153 fiber_verify(fiber);
1154}
1155
1156static void
1157fiber_mark(void *ptr)
1158{
1159 rb_fiber_t *fiber = ptr;
1160 RUBY_MARK_ENTER("cont");
1161 fiber_verify(fiber);
1162 rb_gc_mark_movable(fiber->first_proc);
1163 if (fiber->prev) rb_fiber_mark_self(fiber->prev);
1164 cont_mark(&fiber->cont);
1165 RUBY_MARK_LEAVE("cont");
1166}
1167
1168static void
1169fiber_free(void *ptr)
1170{
1171 rb_fiber_t *fiber = ptr;
1172 RUBY_FREE_ENTER("fiber");
1173
1174 if (DEBUG) fprintf(stderr, "fiber_free: %p[%p]\n", (void *)fiber, fiber->stack.base);
1175
1176 if (fiber->cont.saved_ec.local_storage) {
1177 rb_id_table_free(fiber->cont.saved_ec.local_storage);
1178 }
1179
1180 cont_free(&fiber->cont);
1181 RUBY_FREE_LEAVE("fiber");
1182}
1183
1184static size_t
1185fiber_memsize(const void *ptr)
1186{
1187 const rb_fiber_t *fiber = ptr;
1188 size_t size = sizeof(*fiber);
1189 const rb_execution_context_t *saved_ec = &fiber->cont.saved_ec;
1190 const rb_thread_t *th = rb_ec_thread_ptr(saved_ec);
1191
1192 /*
1193 * vm.c::thread_memsize already counts th->ec->local_storage
1194 */
1195 if (saved_ec->local_storage && fiber != th->root_fiber) {
1196 size += rb_id_table_memsize(saved_ec->local_storage);
1197 size += rb_obj_memsize_of(saved_ec->storage);
1198 }
1199
1200 size += cont_memsize(&fiber->cont);
1201 return size;
1202}
1203
1204VALUE
1205rb_obj_is_fiber(VALUE obj)
1206{
1207 return RBOOL(rb_typeddata_is_kind_of(obj, &fiber_data_type));
1208}
1209
1210static void
1211cont_save_machine_stack(rb_thread_t *th, rb_context_t *cont)
1212{
1213 size_t size;
1214
1215 SET_MACHINE_STACK_END(&th->ec->machine.stack_end);
1216
1217 if (th->ec->machine.stack_start > th->ec->machine.stack_end) {
1218 size = cont->machine.stack_size = th->ec->machine.stack_start - th->ec->machine.stack_end;
1219 cont->machine.stack_src = th->ec->machine.stack_end;
1220 }
1221 else {
1222 size = cont->machine.stack_size = th->ec->machine.stack_end - th->ec->machine.stack_start;
1223 cont->machine.stack_src = th->ec->machine.stack_start;
1224 }
1225
1226 if (cont->machine.stack) {
1227 REALLOC_N(cont->machine.stack, VALUE, size);
1228 }
1229 else {
1230 cont->machine.stack = ALLOC_N(VALUE, size);
1231 }
1232
1233 FLUSH_REGISTER_WINDOWS;
1234 asan_unpoison_memory_region(cont->machine.stack_src, size, false);
1235 MEMCPY(cont->machine.stack, cont->machine.stack_src, VALUE, size);
1236}
1237
1238static const rb_data_type_t cont_data_type = {
1239 "continuation",
1240 {cont_mark, cont_free, cont_memsize, cont_compact},
1241 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
1242};
1243
1244static inline void
1245cont_save_thread(rb_context_t *cont, rb_thread_t *th)
1246{
1247 rb_execution_context_t *sec = &cont->saved_ec;
1248
1249 VM_ASSERT(th->status == THREAD_RUNNABLE);
1250
1251 /* save thread context */
1252 *sec = *th->ec;
1253
1254 /* saved_ec->machine.stack_end should be NULL */
1255 /* because it may happen GC afterward */
1256 sec->machine.stack_end = NULL;
1257}
1258
1259static rb_nativethread_lock_t jit_cont_lock;
1260
1261// Register a new continuation with execution context `ec`. Return JIT info about
1262// the continuation.
1263static struct rb_jit_cont *
1264jit_cont_new(rb_execution_context_t *ec)
1265{
1266 struct rb_jit_cont *cont;
1267
1268 // We need to use calloc instead of something like ZALLOC to avoid triggering GC here.
1269 // When this function is called from rb_thread_alloc through rb_threadptr_root_fiber_setup,
1270 // the thread is still being prepared and marking it causes SEGV.
1271 cont = calloc(1, sizeof(struct rb_jit_cont));
1272 if (cont == NULL)
1273 rb_memerror();
1274 cont->ec = ec;
1275
1276 rb_native_mutex_lock(&jit_cont_lock);
1277 if (first_jit_cont == NULL) {
1278 cont->next = cont->prev = NULL;
1279 }
1280 else {
1281 cont->prev = NULL;
1282 cont->next = first_jit_cont;
1283 first_jit_cont->prev = cont;
1284 }
1285 first_jit_cont = cont;
1286 rb_native_mutex_unlock(&jit_cont_lock);
1287
1288 return cont;
1289}
1290
1291// Unregister continuation `cont`.
1292static void
1293jit_cont_free(struct rb_jit_cont *cont)
1294{
1295 if (!cont) return;
1296
1297 rb_native_mutex_lock(&jit_cont_lock);
1298 if (cont == first_jit_cont) {
1299 first_jit_cont = cont->next;
1300 if (first_jit_cont != NULL)
1301 first_jit_cont->prev = NULL;
1302 }
1303 else {
1304 cont->prev->next = cont->next;
1305 if (cont->next != NULL)
1306 cont->next->prev = cont->prev;
1307 }
1308 rb_native_mutex_unlock(&jit_cont_lock);
1309
1310 free(cont);
1311}
1312
1313// Call a given callback against all on-stack ISEQs.
1314void
1315rb_jit_cont_each_iseq(rb_iseq_callback callback, void *data)
1316{
1317 struct rb_jit_cont *cont;
1318 for (cont = first_jit_cont; cont != NULL; cont = cont->next) {
1319 if (cont->ec->vm_stack == NULL)
1320 continue;
1321
1322 const rb_control_frame_t *cfp = cont->ec->cfp;
1323 while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) {
1324 if (cfp->pc && cfp->iseq && imemo_type((VALUE)cfp->iseq) == imemo_iseq) {
1325 callback(cfp->iseq, data);
1326 }
1327 cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1328 }
1329 }
1330}
1331
1332#if USE_YJIT
1333// Update the jit_return of all CFPs to leave_exit unless it's leave_exception or not set.
1334// This prevents jit_exec_exception from jumping to the caller after invalidation.
1335void
1336rb_yjit_cancel_jit_return(void *leave_exit, void *leave_exception)
1337{
1338 struct rb_jit_cont *cont;
1339 for (cont = first_jit_cont; cont != NULL; cont = cont->next) {
1340 if (cont->ec->vm_stack == NULL)
1341 continue;
1342
1343 const rb_control_frame_t *cfp = cont->ec->cfp;
1344 while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) {
1345 if (cfp->jit_return && cfp->jit_return != leave_exception) {
1346 ((rb_control_frame_t *)cfp)->jit_return = leave_exit;
1347 }
1348 cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1349 }
1350 }
1351}
1352#endif
1353
1354// Finish working with jit_cont.
1355void
1356rb_jit_cont_finish(void)
1357{
1358 struct rb_jit_cont *cont, *next;
1359 for (cont = first_jit_cont; cont != NULL; cont = next) {
1360 next = cont->next;
1361 free(cont); // Don't use xfree because it's allocated by calloc.
1362 }
1363 rb_native_mutex_destroy(&jit_cont_lock);
1364}
1365
1366static void
1367cont_init_jit_cont(rb_context_t *cont)
1368{
1369 VM_ASSERT(cont->jit_cont == NULL);
1370 // We always allocate this since YJIT may be enabled later
1371 cont->jit_cont = jit_cont_new(&(cont->saved_ec));
1372}
1373
1375rb_fiberptr_get_ec(struct rb_fiber_struct *fiber)
1376{
1377 return &fiber->cont.saved_ec;
1378}
1379
1380static void
1381cont_init(rb_context_t *cont, rb_thread_t *th)
1382{
1383 /* save thread context */
1384 cont_save_thread(cont, th);
1385 cont->saved_ec.thread_ptr = th;
1386 cont->saved_ec.local_storage = NULL;
1387 cont->saved_ec.local_storage_recursive_hash = Qnil;
1388 cont->saved_ec.local_storage_recursive_hash_for_trace = Qnil;
1389 cont_init_jit_cont(cont);
1390}
1391
1392static rb_context_t *
1393cont_new(VALUE klass)
1394{
1395 rb_context_t *cont;
1396 volatile VALUE contval;
1397 rb_thread_t *th = GET_THREAD();
1398
1399 THREAD_MUST_BE_RUNNING(th);
1400 contval = TypedData_Make_Struct(klass, rb_context_t, &cont_data_type, cont);
1401 cont->self = contval;
1402 cont_init(cont, th);
1403 return cont;
1404}
1405
1406VALUE
1407rb_fiberptr_self(struct rb_fiber_struct *fiber)
1408{
1409 return fiber->cont.self;
1410}
1411
1412unsigned int
1413rb_fiberptr_blocking(struct rb_fiber_struct *fiber)
1414{
1415 return fiber->blocking;
1416}
1417
1418// Initialize the jit_cont_lock
1419void
1420rb_jit_cont_init(void)
1421{
1422 rb_native_mutex_initialize(&jit_cont_lock);
1423}
1424
1425#if 0
1426void
1427show_vm_stack(const rb_execution_context_t *ec)
1428{
1429 VALUE *p = ec->vm_stack;
1430 while (p < ec->cfp->sp) {
1431 fprintf(stderr, "%3d ", (int)(p - ec->vm_stack));
1432 rb_obj_info_dump(*p);
1433 p++;
1434 }
1435}
1436
1437void
1438show_vm_pcs(const rb_control_frame_t *cfp,
1439 const rb_control_frame_t *end_of_cfp)
1440{
1441 int i=0;
1442 while (cfp != end_of_cfp) {
1443 int pc = 0;
1444 if (cfp->iseq) {
1445 pc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded;
1446 }
1447 fprintf(stderr, "%2d pc: %d\n", i++, pc);
1448 cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1449 }
1450}
1451#endif
1452
1453static VALUE
1454cont_capture(volatile int *volatile stat)
1455{
1456 rb_context_t *volatile cont;
1457 rb_thread_t *th = GET_THREAD();
1458 volatile VALUE contval;
1459 const rb_execution_context_t *ec = th->ec;
1460
1461 THREAD_MUST_BE_RUNNING(th);
1462 rb_vm_stack_to_heap(th->ec);
1463 cont = cont_new(rb_cContinuation);
1464 contval = cont->self;
1465
1466#ifdef CAPTURE_JUST_VALID_VM_STACK
1467 cont->saved_vm_stack.slen = ec->cfp->sp - ec->vm_stack;
1468 cont->saved_vm_stack.clen = ec->vm_stack + ec->vm_stack_size - (VALUE*)ec->cfp;
1469 cont->saved_vm_stack.ptr = ALLOC_N(VALUE, cont->saved_vm_stack.slen + cont->saved_vm_stack.clen);
1470 MEMCPY(cont->saved_vm_stack.ptr,
1471 ec->vm_stack,
1472 VALUE, cont->saved_vm_stack.slen);
1473 MEMCPY(cont->saved_vm_stack.ptr + cont->saved_vm_stack.slen,
1474 (VALUE*)ec->cfp,
1475 VALUE,
1476 cont->saved_vm_stack.clen);
1477#else
1478 cont->saved_vm_stack.ptr = ALLOC_N(VALUE, ec->vm_stack_size);
1479 MEMCPY(cont->saved_vm_stack.ptr, ec->vm_stack, VALUE, ec->vm_stack_size);
1480#endif
1481 // At this point, `cfp` is valid but `vm_stack` should be cleared:
1482 rb_ec_set_vm_stack(&cont->saved_ec, NULL, 0);
1483 VM_ASSERT(cont->saved_ec.cfp != NULL);
1484 cont_save_machine_stack(th, cont);
1485
1486 if (ruby_setjmp(cont->jmpbuf)) {
1487 VALUE value;
1488
1489 VAR_INITIALIZED(cont);
1490 value = cont->value;
1491 if (cont->argc == -1) rb_exc_raise(value);
1492 cont->value = Qnil;
1493 *stat = 1;
1494 return value;
1495 }
1496 else {
1497 *stat = 0;
1498 return contval;
1499 }
1500}
1501
1502static inline void
1503cont_restore_thread(rb_context_t *cont)
1504{
1505 rb_thread_t *th = GET_THREAD();
1506
1507 /* restore thread context */
1508 if (cont->type == CONTINUATION_CONTEXT) {
1509 /* continuation */
1510 rb_execution_context_t *sec = &cont->saved_ec;
1511 rb_fiber_t *fiber = NULL;
1512
1513 if (sec->fiber_ptr != NULL) {
1514 fiber = sec->fiber_ptr;
1515 }
1516 else if (th->root_fiber) {
1517 fiber = th->root_fiber;
1518 }
1519
1520 if (fiber && th->ec != &fiber->cont.saved_ec) {
1521 ec_switch(th, fiber);
1522 }
1523
1524 if (th->ec->trace_arg != sec->trace_arg) {
1525 rb_raise(rb_eRuntimeError, "can't call across trace_func");
1526 }
1527
1528#if defined(__wasm__) && !defined(__EMSCRIPTEN__)
1529 if (th->ec->tag != sec->tag) {
1530 /* find the lowest common ancestor tag of the current EC and the saved EC */
1531
1532 struct rb_vm_tag *lowest_common_ancestor = NULL;
1533 size_t num_tags = 0;
1534 size_t num_saved_tags = 0;
1535 for (struct rb_vm_tag *tag = th->ec->tag; tag != NULL; tag = tag->prev) {
1536 ++num_tags;
1537 }
1538 for (struct rb_vm_tag *tag = sec->tag; tag != NULL; tag = tag->prev) {
1539 ++num_saved_tags;
1540 }
1541
1542 size_t min_tags = num_tags <= num_saved_tags ? num_tags : num_saved_tags;
1543
1544 struct rb_vm_tag *tag = th->ec->tag;
1545 while (num_tags > min_tags) {
1546 tag = tag->prev;
1547 --num_tags;
1548 }
1549
1550 struct rb_vm_tag *saved_tag = sec->tag;
1551 while (num_saved_tags > min_tags) {
1552 saved_tag = saved_tag->prev;
1553 --num_saved_tags;
1554 }
1555
1556 while (min_tags > 0) {
1557 if (tag == saved_tag) {
1558 lowest_common_ancestor = tag;
1559 break;
1560 }
1561 tag = tag->prev;
1562 saved_tag = saved_tag->prev;
1563 --min_tags;
1564 }
1565
1566 /* free all the jump buffers between the current EC's tag and the lowest common ancestor tag */
1567 for (struct rb_vm_tag *tag = th->ec->tag; tag != lowest_common_ancestor; tag = tag->prev) {
1568 rb_vm_tag_jmpbuf_deinit(&tag->buf);
1569 }
1570 }
1571#endif
1572
1573 /* copy vm stack */
1574#ifdef CAPTURE_JUST_VALID_VM_STACK
1575 MEMCPY(th->ec->vm_stack,
1576 cont->saved_vm_stack.ptr,
1577 VALUE, cont->saved_vm_stack.slen);
1578 MEMCPY(th->ec->vm_stack + th->ec->vm_stack_size - cont->saved_vm_stack.clen,
1579 cont->saved_vm_stack.ptr + cont->saved_vm_stack.slen,
1580 VALUE, cont->saved_vm_stack.clen);
1581#else
1582 MEMCPY(th->ec->vm_stack, cont->saved_vm_stack.ptr, VALUE, sec->vm_stack_size);
1583#endif
1584 /* other members of ec */
1585
1586 th->ec->cfp = sec->cfp;
1587 th->ec->raised_flag = sec->raised_flag;
1588 th->ec->tag = sec->tag;
1589 th->ec->root_lep = sec->root_lep;
1590 th->ec->root_svar = sec->root_svar;
1591 th->ec->errinfo = sec->errinfo;
1592
1593 VM_ASSERT(th->ec->vm_stack != NULL);
1594 }
1595 else {
1596 /* fiber */
1597 fiber_restore_thread(th, (rb_fiber_t*)cont);
1598 }
1599}
1600
1601NOINLINE(static void fiber_setcontext(rb_fiber_t *new_fiber, rb_fiber_t *old_fiber));
1602
1603static void
1604fiber_setcontext(rb_fiber_t *new_fiber, rb_fiber_t *old_fiber)
1605{
1606 rb_thread_t *th = GET_THREAD();
1607
1608 /* save old_fiber's machine stack - to ensure efficient garbage collection */
1609 if (!FIBER_TERMINATED_P(old_fiber)) {
1610 STACK_GROW_DIR_DETECTION;
1611 SET_MACHINE_STACK_END(&th->ec->machine.stack_end);
1612 if (STACK_DIR_UPPER(0, 1)) {
1613 old_fiber->cont.machine.stack_size = th->ec->machine.stack_start - th->ec->machine.stack_end;
1614 old_fiber->cont.machine.stack = th->ec->machine.stack_end;
1615 }
1616 else {
1617 old_fiber->cont.machine.stack_size = th->ec->machine.stack_end - th->ec->machine.stack_start;
1618 old_fiber->cont.machine.stack = th->ec->machine.stack_start;
1619 }
1620 }
1621
1622 /* these values are used in rb_gc_mark_machine_context to mark the fiber's stack. */
1623 old_fiber->cont.saved_ec.machine.stack_start = th->ec->machine.stack_start;
1624 old_fiber->cont.saved_ec.machine.stack_end = FIBER_TERMINATED_P(old_fiber) ? NULL : th->ec->machine.stack_end;
1625
1626
1627 // if (DEBUG) fprintf(stderr, "fiber_setcontext: %p[%p] -> %p[%p]\n", (void*)old_fiber, old_fiber->stack.base, (void*)new_fiber, new_fiber->stack.base);
1628
1629#if defined(COROUTINE_SANITIZE_ADDRESS)
1630 __sanitizer_start_switch_fiber(FIBER_TERMINATED_P(old_fiber) ? NULL : &old_fiber->context.fake_stack, new_fiber->context.stack_base, new_fiber->context.stack_size);
1631#endif
1632
1633 /* swap machine context */
1634 struct coroutine_context * from = coroutine_transfer(&old_fiber->context, &new_fiber->context);
1635
1636#if defined(COROUTINE_SANITIZE_ADDRESS)
1637 __sanitizer_finish_switch_fiber(old_fiber->context.fake_stack, NULL, NULL);
1638#endif
1639
1640 if (from == NULL) {
1641 rb_syserr_fail(errno, "coroutine_transfer");
1642 }
1643
1644 /* restore thread context */
1645 fiber_restore_thread(th, old_fiber);
1646
1647 // It's possible to get here, and new_fiber is already freed.
1648 // if (DEBUG) fprintf(stderr, "fiber_setcontext: %p[%p] <- %p[%p]\n", (void*)old_fiber, old_fiber->stack.base, (void*)new_fiber, new_fiber->stack.base);
1649}
1650
1651NOINLINE(NORETURN(static void cont_restore_1(rb_context_t *)));
1652
1653static void
1654cont_restore_1(rb_context_t *cont)
1655{
1656 cont_restore_thread(cont);
1657
1658 /* restore machine stack */
1659#if (defined(_M_AMD64) && !defined(__MINGW64__)) || defined(_M_ARM64)
1660 {
1661 /* workaround for x64 and arm64 SEH on Windows */
1662 jmp_buf buf;
1663 setjmp(buf);
1664 _JUMP_BUFFER *bp = (void*)&cont->jmpbuf;
1665 bp->Frame = ((_JUMP_BUFFER*)((void*)&buf))->Frame;
1666 }
1667#endif
1668 if (cont->machine.stack_src) {
1669 FLUSH_REGISTER_WINDOWS;
1670 MEMCPY(cont->machine.stack_src, cont->machine.stack,
1671 VALUE, cont->machine.stack_size);
1672 }
1673
1674 ruby_longjmp(cont->jmpbuf, 1);
1675}
1676
1677NORETURN(NOINLINE(static void cont_restore_0(rb_context_t *, VALUE *)));
1678
1679static void
1680cont_restore_0(rb_context_t *cont, VALUE *addr_in_prev_frame)
1681{
1682 if (cont->machine.stack_src) {
1683#ifdef HAVE_ALLOCA
1684#define STACK_PAD_SIZE 1
1685#else
1686#define STACK_PAD_SIZE 1024
1687#endif
1688 VALUE space[STACK_PAD_SIZE];
1689
1690#if !STACK_GROW_DIRECTION
1691 if (addr_in_prev_frame > &space[0]) {
1692 /* Stack grows downward */
1693#endif
1694#if STACK_GROW_DIRECTION <= 0
1695 volatile VALUE *const end = cont->machine.stack_src;
1696 if (&space[0] > end) {
1697# ifdef HAVE_ALLOCA
1698 volatile VALUE *sp = ALLOCA_N(VALUE, &space[0] - end);
1699 // We need to make sure that the stack pointer is moved,
1700 // but some compilers may remove the allocation by optimization.
1701 // We hope that the following read/write will prevent such an optimization.
1702 *sp = Qfalse;
1703 space[0] = *sp;
1704# else
1705 cont_restore_0(cont, &space[0]);
1706# endif
1707 }
1708#endif
1709#if !STACK_GROW_DIRECTION
1710 }
1711 else {
1712 /* Stack grows upward */
1713#endif
1714#if STACK_GROW_DIRECTION >= 0
1715 volatile VALUE *const end = cont->machine.stack_src + cont->machine.stack_size;
1716 if (&space[STACK_PAD_SIZE] < end) {
1717# ifdef HAVE_ALLOCA
1718 volatile VALUE *sp = ALLOCA_N(VALUE, end - &space[STACK_PAD_SIZE]);
1719 space[0] = *sp;
1720# else
1721 cont_restore_0(cont, &space[STACK_PAD_SIZE-1]);
1722# endif
1723 }
1724#endif
1725#if !STACK_GROW_DIRECTION
1726 }
1727#endif
1728 }
1729 cont_restore_1(cont);
1730}
1731
1732/*
1733 * Document-class: Continuation
1734 *
1735 * Continuation objects are generated by Kernel#callcc,
1736 * after having +require+d <i>continuation</i>. They hold
1737 * a return address and execution context, allowing a nonlocal return
1738 * to the end of the #callcc block from anywhere within a
1739 * program. Continuations are somewhat analogous to a structured
1740 * version of C's <code>setjmp/longjmp</code> (although they contain
1741 * more state, so you might consider them closer to threads).
1742 *
1743 * For instance:
1744 *
1745 * require "continuation"
1746 * arr = [ "Freddie", "Herbie", "Ron", "Max", "Ringo" ]
1747 * callcc{|cc| $cc = cc}
1748 * puts(message = arr.shift)
1749 * $cc.call unless message =~ /Max/
1750 *
1751 * <em>produces:</em>
1752 *
1753 * Freddie
1754 * Herbie
1755 * Ron
1756 * Max
1757 *
1758 * Also you can call callcc in other methods:
1759 *
1760 * require "continuation"
1761 *
1762 * def g
1763 * arr = [ "Freddie", "Herbie", "Ron", "Max", "Ringo" ]
1764 * cc = callcc { |cc| cc }
1765 * puts arr.shift
1766 * return cc, arr.size
1767 * end
1768 *
1769 * def f
1770 * c, size = g
1771 * c.call(c) if size > 1
1772 * end
1773 *
1774 * f
1775 *
1776 * This (somewhat contrived) example allows the inner loop to abandon
1777 * processing early:
1778 *
1779 * require "continuation"
1780 * callcc {|cont|
1781 * for i in 0..4
1782 * print "#{i}: "
1783 * for j in i*5...(i+1)*5
1784 * cont.call() if j == 17
1785 * printf "%3d", j
1786 * end
1787 * end
1788 * }
1789 * puts
1790 *
1791 * <em>produces:</em>
1792 *
1793 * 0: 0 1 2 3 4
1794 * 1: 5 6 7 8 9
1795 * 2: 10 11 12 13 14
1796 * 3: 15 16
1797 */
1798
1799/*
1800 * call-seq:
1801 * callcc {|cont| block } -> obj
1802 *
1803 * Generates a Continuation object, which it passes to
1804 * the associated block. You need to <code>require
1805 * 'continuation'</code> before using this method. Performing a
1806 * <em>cont</em><code>.call</code> will cause the #callcc
1807 * to return (as will falling through the end of the block). The
1808 * value returned by the #callcc is the value of the
1809 * block, or the value passed to <em>cont</em><code>.call</code>. See
1810 * class Continuation for more details. Also see
1811 * Kernel#throw for an alternative mechanism for
1812 * unwinding a call stack.
1813 */
1814
1815static VALUE
1816rb_callcc(VALUE self)
1817{
1818 volatile int called;
1819 volatile VALUE val = cont_capture(&called);
1820
1821 if (called) {
1822 return val;
1823 }
1824 else {
1825 return rb_yield(val);
1826 }
1827}
1828#ifdef RUBY_ASAN_ENABLED
1829/* callcc can't possibly work with ASAN; see bug #20273. Also this function
1830 * definition below avoids a "defined and not used" warning. */
1831MAYBE_UNUSED(static void notusing_callcc(void)) { rb_callcc(Qnil); }
1832# define rb_callcc rb_f_notimplement
1833#endif
1834
1835
1836static VALUE
1837make_passing_arg(int argc, const VALUE *argv)
1838{
1839 switch (argc) {
1840 case -1:
1841 return argv[0];
1842 case 0:
1843 return Qnil;
1844 case 1:
1845 return argv[0];
1846 default:
1847 return rb_ary_new4(argc, argv);
1848 }
1849}
1850
1851typedef VALUE e_proc(VALUE);
1852
1853NORETURN(static VALUE rb_cont_call(int argc, VALUE *argv, VALUE contval));
1854
1855/*
1856 * call-seq:
1857 * cont.call(args, ...)
1858 * cont[args, ...]
1859 *
1860 * Invokes the continuation. The program continues from the end of
1861 * the #callcc block. If no arguments are given, the original #callcc
1862 * returns +nil+. If one argument is given, #callcc returns
1863 * it. Otherwise, an array containing <i>args</i> is returned.
1864 *
1865 * callcc {|cont| cont.call } #=> nil
1866 * callcc {|cont| cont.call 1 } #=> 1
1867 * callcc {|cont| cont.call 1, 2, 3 } #=> [1, 2, 3]
1868 */
1869
1870static VALUE
1871rb_cont_call(int argc, VALUE *argv, VALUE contval)
1872{
1873 rb_context_t *cont = cont_ptr(contval);
1874 rb_thread_t *th = GET_THREAD();
1875
1876 if (cont_thread_value(cont) != th->self) {
1877 rb_raise(rb_eRuntimeError, "continuation called across threads");
1878 }
1879 if (cont->saved_ec.fiber_ptr) {
1880 if (th->ec->fiber_ptr != cont->saved_ec.fiber_ptr) {
1881 rb_raise(rb_eRuntimeError, "continuation called across fiber");
1882 }
1883 }
1884
1885 cont->argc = argc;
1886 cont->value = make_passing_arg(argc, argv);
1887
1888 cont_restore_0(cont, &contval);
1890}
1891
1892/*********/
1893/* fiber */
1894/*********/
1895
1896/*
1897 * Document-class: Fiber
1898 *
1899 * Fibers are primitives for implementing light weight cooperative
1900 * concurrency in Ruby. Basically they are a means of creating code blocks
1901 * that can be paused and resumed, much like threads. The main difference
1902 * is that they are never preempted and that the scheduling must be done by
1903 * the programmer and not the VM.
1904 *
1905 * As opposed to other stackless light weight concurrency models, each fiber
1906 * comes with a stack. This enables the fiber to be paused from deeply
1907 * nested function calls within the fiber block. See the ruby(1)
1908 * manpage to configure the size of the fiber stack(s).
1909 *
1910 * When a fiber is created it will not run automatically. Rather it must
1911 * be explicitly asked to run using the Fiber#resume method.
1912 * The code running inside the fiber can give up control by calling
1913 * Fiber.yield in which case it yields control back to caller (the
1914 * caller of the Fiber#resume).
1915 *
1916 * Upon yielding or termination the Fiber returns the value of the last
1917 * executed expression
1918 *
1919 * For instance:
1920 *
1921 * fiber = Fiber.new do
1922 * Fiber.yield 1
1923 * 2
1924 * end
1925 *
1926 * puts fiber.resume
1927 * puts fiber.resume
1928 * puts fiber.resume
1929 *
1930 * <em>produces</em>
1931 *
1932 * 1
1933 * 2
1934 * FiberError: dead fiber called
1935 *
1936 * The Fiber#resume method accepts an arbitrary number of parameters,
1937 * if it is the first call to #resume then they will be passed as
1938 * block arguments. Otherwise they will be the return value of the
1939 * call to Fiber.yield
1940 *
1941 * Example:
1942 *
1943 * fiber = Fiber.new do |first|
1944 * second = Fiber.yield first + 2
1945 * end
1946 *
1947 * puts fiber.resume 10
1948 * puts fiber.resume 1_000_000
1949 * puts fiber.resume "The fiber will be dead before I can cause trouble"
1950 *
1951 * <em>produces</em>
1952 *
1953 * 12
1954 * 1000000
1955 * FiberError: dead fiber called
1956 *
1957 * == Non-blocking Fibers
1958 *
1959 * The concept of <em>non-blocking fiber</em> was introduced in Ruby 3.0.
1960 * A non-blocking fiber, when reaching an operation that would normally block
1961 * the fiber (like <code>sleep</code>, or wait for another process or I/O)
1962 * will yield control to other fibers and allow the <em>scheduler</em> to
1963 * handle blocking and waking up (resuming) this fiber when it can proceed.
1964 *
1965 * For a Fiber to behave as non-blocking, it need to be created in Fiber.new with
1966 * <tt>blocking: false</tt> (which is the default), and Fiber.scheduler
1967 * should be set with Fiber.set_scheduler. If Fiber.scheduler is not set in
1968 * the current thread, blocking and non-blocking fibers' behavior is identical.
1969 *
1970 * Ruby doesn't provide a scheduler class: it is expected to be implemented by
1971 * the user and correspond to Fiber::Scheduler.
1972 *
1973 * There is also Fiber.schedule method, which is expected to immediately perform
1974 * the given block in a non-blocking manner. Its actual implementation is up to
1975 * the scheduler.
1976 *
1977 */
1978
1979static const rb_data_type_t fiber_data_type = {
1980 "fiber",
1981 {fiber_mark, fiber_free, fiber_memsize, fiber_compact,},
1982 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
1983};
1984
1985static VALUE
1986fiber_alloc(VALUE klass)
1987{
1988 return TypedData_Wrap_Struct(klass, &fiber_data_type, 0);
1989}
1990
1991static rb_fiber_t*
1992fiber_t_alloc(VALUE fiber_value, unsigned int blocking)
1993{
1994 rb_fiber_t *fiber;
1995 rb_thread_t *th = GET_THREAD();
1996
1997 if (DATA_PTR(fiber_value) != 0) {
1998 rb_raise(rb_eRuntimeError, "cannot initialize twice");
1999 }
2000
2001 THREAD_MUST_BE_RUNNING(th);
2002 fiber = ZALLOC(rb_fiber_t);
2003 fiber->cont.self = fiber_value;
2004 fiber->cont.type = FIBER_CONTEXT;
2005 fiber->blocking = blocking;
2006 fiber->killed = 0;
2007 cont_init(&fiber->cont, th);
2008
2009 fiber->cont.saved_ec.fiber_ptr = fiber;
2010 rb_ec_clear_vm_stack(&fiber->cont.saved_ec);
2011
2012 fiber->prev = NULL;
2013
2014 /* fiber->status == 0 == CREATED
2015 * So that we don't need to set status: fiber_status_set(fiber, FIBER_CREATED); */
2016 VM_ASSERT(FIBER_CREATED_P(fiber));
2017
2018 DATA_PTR(fiber_value) = fiber;
2019
2020 return fiber;
2021}
2022
2023static rb_fiber_t *
2024root_fiber_alloc(rb_thread_t *th)
2025{
2026 VALUE fiber_value = fiber_alloc(rb_cFiber);
2027 rb_fiber_t *fiber = th->ec->fiber_ptr;
2028
2029 VM_ASSERT(DATA_PTR(fiber_value) == NULL);
2030 VM_ASSERT(fiber->cont.type == FIBER_CONTEXT);
2031 VM_ASSERT(FIBER_RESUMED_P(fiber));
2032
2033 th->root_fiber = fiber;
2034 DATA_PTR(fiber_value) = fiber;
2035 fiber->cont.self = fiber_value;
2036
2037 coroutine_initialize_main(&fiber->context);
2038
2039 return fiber;
2040}
2041
2042static inline rb_fiber_t*
2043fiber_current(void)
2044{
2045 rb_execution_context_t *ec = GET_EC();
2046 if (ec->fiber_ptr->cont.self == 0) {
2047 root_fiber_alloc(rb_ec_thread_ptr(ec));
2048 }
2049 return ec->fiber_ptr;
2050}
2051
2052static inline VALUE
2053current_fiber_storage(void)
2054{
2055 rb_execution_context_t *ec = GET_EC();
2056 return ec->storage;
2057}
2058
2059static inline VALUE
2060inherit_fiber_storage(void)
2061{
2062 return rb_obj_dup(current_fiber_storage());
2063}
2064
2065static inline void
2066fiber_storage_set(struct rb_fiber_struct *fiber, VALUE storage)
2067{
2068 fiber->cont.saved_ec.storage = storage;
2069}
2070
2071static inline VALUE
2072fiber_storage_get(rb_fiber_t *fiber, int allocate)
2073{
2074 VALUE storage = fiber->cont.saved_ec.storage;
2075 if (storage == Qnil && allocate) {
2076 storage = rb_hash_new();
2077 fiber_storage_set(fiber, storage);
2078 }
2079 return storage;
2080}
2081
2082static void
2083storage_access_must_be_from_same_fiber(VALUE self)
2084{
2085 rb_fiber_t *fiber = fiber_ptr(self);
2086 rb_fiber_t *current = fiber_current();
2087 if (fiber != current) {
2088 rb_raise(rb_eArgError, "Fiber storage can only be accessed from the Fiber it belongs to");
2089 }
2090}
2091
2098static VALUE
2099rb_fiber_storage_get(VALUE self)
2100{
2101 storage_access_must_be_from_same_fiber(self);
2102
2103 VALUE storage = fiber_storage_get(fiber_ptr(self), FALSE);
2104
2105 if (storage == Qnil) {
2106 return Qnil;
2107 }
2108 else {
2109 return rb_obj_dup(storage);
2110 }
2111}
2112
2113static int
2114fiber_storage_validate_each(VALUE key, VALUE value, VALUE _argument)
2115{
2116 Check_Type(key, T_SYMBOL);
2117
2118 return ST_CONTINUE;
2119}
2120
2121static void
2122fiber_storage_validate(VALUE value)
2123{
2124 // nil is an allowed value and will be lazily initialized.
2125 if (value == Qnil) return;
2126
2127 if (!RB_TYPE_P(value, T_HASH)) {
2128 rb_raise(rb_eTypeError, "storage must be a hash");
2129 }
2130
2131 if (RB_OBJ_FROZEN(value)) {
2132 rb_raise(rb_eFrozenError, "storage must not be frozen");
2133 }
2134
2135 rb_hash_foreach(value, fiber_storage_validate_each, Qundef);
2136}
2137
2160static VALUE
2161rb_fiber_storage_set(VALUE self, VALUE value)
2162{
2163 if (rb_warning_category_enabled_p(RB_WARN_CATEGORY_EXPERIMENTAL)) {
2165 "Fiber#storage= is experimental and may be removed in the future!");
2166 }
2167
2168 storage_access_must_be_from_same_fiber(self);
2169 fiber_storage_validate(value);
2170
2171 fiber_ptr(self)->cont.saved_ec.storage = rb_obj_dup(value);
2172 return value;
2173}
2174
2185static VALUE
2186rb_fiber_storage_aref(VALUE class, VALUE key)
2187{
2188 key = rb_to_symbol(key);
2189
2190 VALUE storage = fiber_storage_get(fiber_current(), FALSE);
2191 if (storage == Qnil) return Qnil;
2192
2193 return rb_hash_aref(storage, key);
2194}
2195
2206static VALUE
2207rb_fiber_storage_aset(VALUE class, VALUE key, VALUE value)
2208{
2209 key = rb_to_symbol(key);
2210
2211 VALUE storage = fiber_storage_get(fiber_current(), value != Qnil);
2212 if (storage == Qnil) return Qnil;
2213
2214 if (value == Qnil) {
2215 return rb_hash_delete(storage, key);
2216 }
2217 else {
2218 return rb_hash_aset(storage, key, value);
2219 }
2220}
2221
2222static VALUE
2223fiber_initialize(VALUE self, VALUE proc, struct fiber_pool * fiber_pool, unsigned int blocking, VALUE storage)
2224{
2225 if (storage == Qundef || storage == Qtrue) {
2226 // The default, inherit storage (dup) from the current fiber:
2227 storage = inherit_fiber_storage();
2228 }
2229 else /* nil, hash, etc. */ {
2230 fiber_storage_validate(storage);
2231 storage = rb_obj_dup(storage);
2232 }
2233
2234 rb_fiber_t *fiber = fiber_t_alloc(self, blocking);
2235
2236 fiber->cont.saved_ec.storage = storage;
2237 fiber->first_proc = proc;
2238 fiber->stack.base = NULL;
2239 fiber->stack.pool = fiber_pool;
2240
2241 return self;
2242}
2243
2244static void
2245fiber_prepare_stack(rb_fiber_t *fiber)
2246{
2247 rb_context_t *cont = &fiber->cont;
2248 rb_execution_context_t *sec = &cont->saved_ec;
2249
2250 size_t vm_stack_size = 0;
2251 VALUE *vm_stack = fiber_initialize_coroutine(fiber, &vm_stack_size);
2252
2253 /* initialize cont */
2254 cont->saved_vm_stack.ptr = NULL;
2255 rb_ec_initialize_vm_stack(sec, vm_stack, vm_stack_size / sizeof(VALUE));
2256
2257 sec->tag = NULL;
2258 sec->local_storage = NULL;
2259 sec->local_storage_recursive_hash = Qnil;
2260 sec->local_storage_recursive_hash_for_trace = Qnil;
2261}
2262
2263static struct fiber_pool *
2264rb_fiber_pool_default(VALUE pool)
2265{
2266 return &shared_fiber_pool;
2267}
2268
2269VALUE rb_fiber_inherit_storage(struct rb_execution_context_struct *ec, struct rb_fiber_struct *fiber)
2270{
2271 VALUE storage = rb_obj_dup(ec->storage);
2272 fiber->cont.saved_ec.storage = storage;
2273 return storage;
2274}
2275
2276/* :nodoc: */
2277static VALUE
2278rb_fiber_initialize_kw(int argc, VALUE* argv, VALUE self, int kw_splat)
2279{
2280 VALUE pool = Qnil;
2281 VALUE blocking = Qfalse;
2282 VALUE storage = Qundef;
2283
2284 if (kw_splat != RB_NO_KEYWORDS) {
2285 VALUE options = Qnil;
2286 VALUE arguments[3] = {Qundef};
2287
2288 argc = rb_scan_args_kw(kw_splat, argc, argv, ":", &options);
2289 rb_get_kwargs(options, fiber_initialize_keywords, 0, 3, arguments);
2290
2291 if (!UNDEF_P(arguments[0])) {
2292 blocking = arguments[0];
2293 }
2294
2295 if (!UNDEF_P(arguments[1])) {
2296 pool = arguments[1];
2297 }
2298
2299 storage = arguments[2];
2300 }
2301
2302 return fiber_initialize(self, rb_block_proc(), rb_fiber_pool_default(pool), RTEST(blocking), storage);
2303}
2304
2305/*
2306 * call-seq:
2307 * Fiber.new(blocking: false, storage: true) { |*args| ... } -> fiber
2308 *
2309 * Creates new Fiber. Initially, the fiber is not running and can be resumed
2310 * with #resume. Arguments to the first #resume call will be passed to the
2311 * block:
2312 *
2313 * f = Fiber.new do |initial|
2314 * current = initial
2315 * loop do
2316 * puts "current: #{current.inspect}"
2317 * current = Fiber.yield
2318 * end
2319 * end
2320 * f.resume(100) # prints: current: 100
2321 * f.resume(1, 2, 3) # prints: current: [1, 2, 3]
2322 * f.resume # prints: current: nil
2323 * # ... and so on ...
2324 *
2325 * If <tt>blocking: false</tt> is passed to <tt>Fiber.new</tt>, _and_ current
2326 * thread has a Fiber.scheduler defined, the Fiber becomes non-blocking (see
2327 * "Non-blocking Fibers" section in class docs).
2328 *
2329 * If the <tt>storage</tt> is unspecified, the default is to inherit a copy of
2330 * the storage from the current fiber. This is the same as specifying
2331 * <tt>storage: true</tt>.
2332 *
2333 * Fiber[:x] = 1
2334 * Fiber.new do
2335 * Fiber[:x] # => 1
2336 * Fiber[:x] = 2
2337 * end.resume
2338 * Fiber[:x] # => 1
2339 *
2340 * If the given <tt>storage</tt> is <tt>nil</tt>, this function will lazy
2341 * initialize the internal storage, which starts as an empty hash.
2342 *
2343 * Fiber[:x] = "Hello World"
2344 * Fiber.new(storage: nil) do
2345 * Fiber[:x] # nil
2346 * end
2347 *
2348 * Otherwise, the given <tt>storage</tt> is used as the new fiber's storage,
2349 * and it must be an instance of Hash.
2350 *
2351 * Explicitly using <tt>storage: true</tt> is currently experimental and may
2352 * change in the future.
2353 */
2354static VALUE
2355rb_fiber_initialize(int argc, VALUE* argv, VALUE self)
2356{
2357 return rb_fiber_initialize_kw(argc, argv, self, rb_keyword_given_p());
2358}
2359
2360VALUE
2361rb_fiber_new_storage(rb_block_call_func_t func, VALUE obj, VALUE storage)
2362{
2363 return fiber_initialize(fiber_alloc(rb_cFiber), rb_proc_new(func, obj), rb_fiber_pool_default(Qnil), 0, storage);
2364}
2365
2366VALUE
2367rb_fiber_new(rb_block_call_func_t func, VALUE obj)
2368{
2369 return rb_fiber_new_storage(func, obj, Qtrue);
2370}
2371
2372static VALUE
2373rb_fiber_s_schedule_kw(int argc, VALUE* argv, int kw_splat)
2374{
2375 rb_thread_t * th = GET_THREAD();
2376 VALUE scheduler = th->scheduler;
2377 VALUE fiber = Qnil;
2378
2379 if (scheduler != Qnil) {
2380 fiber = rb_fiber_scheduler_fiber(scheduler, argc, argv, kw_splat);
2381 }
2382 else {
2383 rb_raise(rb_eRuntimeError, "No scheduler is available!");
2384 }
2385
2386 return fiber;
2387}
2388
2389/*
2390 * call-seq:
2391 * Fiber.schedule { |*args| ... } -> fiber
2392 *
2393 * The method is <em>expected</em> to immediately run the provided block of code in a
2394 * separate non-blocking fiber.
2395 *
2396 * puts "Go to sleep!"
2397 *
2398 * Fiber.set_scheduler(MyScheduler.new)
2399 *
2400 * Fiber.schedule do
2401 * puts "Going to sleep"
2402 * sleep(1)
2403 * puts "I slept well"
2404 * end
2405 *
2406 * puts "Wakey-wakey, sleepyhead"
2407 *
2408 * Assuming MyScheduler is properly implemented, this program will produce:
2409 *
2410 * Go to sleep!
2411 * Going to sleep
2412 * Wakey-wakey, sleepyhead
2413 * ...1 sec pause here...
2414 * I slept well
2415 *
2416 * ...e.g. on the first blocking operation inside the Fiber (<tt>sleep(1)</tt>),
2417 * the control is yielded to the outside code (main fiber), and <em>at the end
2418 * of that execution</em>, the scheduler takes care of properly resuming all the
2419 * blocked fibers.
2420 *
2421 * Note that the behavior described above is how the method is <em>expected</em>
2422 * to behave, actual behavior is up to the current scheduler's implementation of
2423 * Fiber::Scheduler#fiber method. Ruby doesn't enforce this method to
2424 * behave in any particular way.
2425 *
2426 * If the scheduler is not set, the method raises
2427 * <tt>RuntimeError (No scheduler is available!)</tt>.
2428 *
2429 */
2430static VALUE
2431rb_fiber_s_schedule(int argc, VALUE *argv, VALUE obj)
2432{
2433 return rb_fiber_s_schedule_kw(argc, argv, rb_keyword_given_p());
2434}
2435
2436/*
2437 * call-seq:
2438 * Fiber.scheduler -> obj or nil
2439 *
2440 * Returns the Fiber scheduler, that was last set for the current thread with Fiber.set_scheduler.
2441 * Returns +nil+ if no scheduler is set (which is the default), and non-blocking fibers'
2442 * behavior is the same as blocking.
2443 * (see "Non-blocking fibers" section in class docs for details about the scheduler concept).
2444 *
2445 */
2446static VALUE
2447rb_fiber_s_scheduler(VALUE klass)
2448{
2449 return rb_fiber_scheduler_get();
2450}
2451
2452/*
2453 * call-seq:
2454 * Fiber.current_scheduler -> obj or nil
2455 *
2456 * Returns the Fiber scheduler, that was last set for the current thread with Fiber.set_scheduler
2457 * if and only if the current fiber is non-blocking.
2458 *
2459 */
2460static VALUE
2461rb_fiber_current_scheduler(VALUE klass)
2462{
2464}
2465
2466/*
2467 * call-seq:
2468 * Fiber.set_scheduler(scheduler) -> scheduler
2469 *
2470 * Sets the Fiber scheduler for the current thread. If the scheduler is set, non-blocking
2471 * fibers (created by Fiber.new with <tt>blocking: false</tt>, or by Fiber.schedule)
2472 * call that scheduler's hook methods on potentially blocking operations, and the current
2473 * thread will call scheduler's +close+ method on finalization (allowing the scheduler to
2474 * properly manage all non-finished fibers).
2475 *
2476 * +scheduler+ can be an object of any class corresponding to Fiber::Scheduler. Its
2477 * implementation is up to the user.
2478 *
2479 * See also the "Non-blocking fibers" section in class docs.
2480 *
2481 */
2482static VALUE
2483rb_fiber_set_scheduler(VALUE klass, VALUE scheduler)
2484{
2485 return rb_fiber_scheduler_set(scheduler);
2486}
2487
2488NORETURN(static void rb_fiber_terminate(rb_fiber_t *fiber, int need_interrupt, VALUE err));
2489
2490void
2491rb_fiber_start(rb_fiber_t *fiber)
2492{
2493 rb_thread_t * volatile th = fiber->cont.saved_ec.thread_ptr;
2494
2495 rb_proc_t *proc;
2496 enum ruby_tag_type state;
2497
2498 VM_ASSERT(th->ec == GET_EC());
2499 VM_ASSERT(FIBER_RESUMED_P(fiber));
2500
2501 if (fiber->blocking) {
2502 th->blocking += 1;
2503 }
2504
2505 EC_PUSH_TAG(th->ec);
2506 if ((state = EC_EXEC_TAG()) == TAG_NONE) {
2507 rb_context_t *cont = &VAR_FROM_MEMORY(fiber)->cont;
2508 int argc;
2509 const VALUE *argv, args = cont->value;
2510 GetProcPtr(fiber->first_proc, proc);
2511 argv = (argc = cont->argc) > 1 ? RARRAY_CONST_PTR(args) : &args;
2512 cont->value = Qnil;
2513 th->ec->errinfo = Qnil;
2514 th->ec->root_lep = rb_vm_proc_local_ep(fiber->first_proc);
2515 th->ec->root_svar = Qfalse;
2516
2517 EXEC_EVENT_HOOK(th->ec, RUBY_EVENT_FIBER_SWITCH, th->self, 0, 0, 0, Qnil);
2518 cont->value = rb_vm_invoke_proc(th->ec, proc, argc, argv, cont->kw_splat, VM_BLOCK_HANDLER_NONE);
2519 }
2520 EC_POP_TAG();
2521
2522 int need_interrupt = TRUE;
2523 VALUE err = Qfalse;
2524 if (state) {
2525 err = th->ec->errinfo;
2526 VM_ASSERT(FIBER_RESUMED_P(fiber));
2527
2528 if (state == TAG_RAISE) {
2529 // noop...
2530 }
2531 else if (state == TAG_FATAL && err == RUBY_FATAL_FIBER_KILLED) {
2532 need_interrupt = FALSE;
2533 err = Qfalse;
2534 }
2535 else if (state == TAG_FATAL) {
2536 rb_threadptr_pending_interrupt_enque(th, err);
2537 }
2538 else {
2539 err = rb_vm_make_jump_tag_but_local_jump(state, err);
2540 }
2541 }
2542
2543 rb_fiber_terminate(fiber, need_interrupt, err);
2544}
2545
2546// Set up a "root fiber", which is the fiber that every Ractor has.
2547void
2548rb_threadptr_root_fiber_setup(rb_thread_t *th)
2549{
2550 rb_fiber_t *fiber = ruby_mimcalloc(1, sizeof(rb_fiber_t));
2551 if (!fiber) {
2552 rb_bug("%s", strerror(errno)); /* ... is it possible to call rb_bug here? */
2553 }
2554 fiber->cont.type = FIBER_CONTEXT;
2555 fiber->cont.saved_ec.fiber_ptr = fiber;
2556 fiber->cont.saved_ec.thread_ptr = th;
2557 fiber->blocking = 1;
2558 fiber->killed = 0;
2559 fiber_status_set(fiber, FIBER_RESUMED); /* skip CREATED */
2560 th->ec = &fiber->cont.saved_ec;
2561 cont_init_jit_cont(&fiber->cont);
2562}
2563
2564void
2565rb_threadptr_root_fiber_release(rb_thread_t *th)
2566{
2567 if (th->root_fiber) {
2568 /* ignore. A root fiber object will free th->ec */
2569 }
2570 else {
2571 rb_execution_context_t *ec = rb_current_execution_context(false);
2572
2573 VM_ASSERT(th->ec->fiber_ptr->cont.type == FIBER_CONTEXT);
2574 VM_ASSERT(th->ec->fiber_ptr->cont.self == 0);
2575
2576 if (ec && th->ec == ec) {
2577 rb_ractor_set_current_ec(th->ractor, NULL);
2578 }
2579 fiber_free(th->ec->fiber_ptr);
2580 th->ec = NULL;
2581 }
2582}
2583
2584void
2585rb_threadptr_root_fiber_terminate(rb_thread_t *th)
2586{
2587 rb_fiber_t *fiber = th->ec->fiber_ptr;
2588
2589 fiber->status = FIBER_TERMINATED;
2590
2591 // The vm_stack is `alloca`ed on the thread stack, so it's gone too:
2592 rb_ec_clear_vm_stack(th->ec);
2593}
2594
2595static inline rb_fiber_t*
2596return_fiber(bool terminate)
2597{
2598 rb_fiber_t *fiber = fiber_current();
2599 rb_fiber_t *prev = fiber->prev;
2600
2601 if (prev) {
2602 fiber->prev = NULL;
2603 prev->resuming_fiber = NULL;
2604 return prev;
2605 }
2606 else {
2607 if (!terminate) {
2608 rb_raise(rb_eFiberError, "attempt to yield on a not resumed fiber");
2609 }
2610
2611 rb_thread_t *th = GET_THREAD();
2612 rb_fiber_t *root_fiber = th->root_fiber;
2613
2614 VM_ASSERT(root_fiber != NULL);
2615
2616 // search resuming fiber
2617 for (fiber = root_fiber; fiber->resuming_fiber; fiber = fiber->resuming_fiber) {
2618 }
2619
2620 return fiber;
2621 }
2622}
2623
2624VALUE
2625rb_fiber_current(void)
2626{
2627 return fiber_current()->cont.self;
2628}
2629
2630// Prepare to execute next_fiber on the given thread.
2631static inline void
2632fiber_store(rb_fiber_t *next_fiber, rb_thread_t *th)
2633{
2634 rb_fiber_t *fiber;
2635
2636 if (th->ec->fiber_ptr != NULL) {
2637 fiber = th->ec->fiber_ptr;
2638 }
2639 else {
2640 /* create root fiber */
2641 fiber = root_fiber_alloc(th);
2642 }
2643
2644 if (FIBER_CREATED_P(next_fiber)) {
2645 fiber_prepare_stack(next_fiber);
2646 }
2647
2648 VM_ASSERT(FIBER_RESUMED_P(fiber) || FIBER_TERMINATED_P(fiber));
2649 VM_ASSERT(FIBER_RUNNABLE_P(next_fiber));
2650
2651 if (FIBER_RESUMED_P(fiber)) fiber_status_set(fiber, FIBER_SUSPENDED);
2652
2653 fiber_status_set(next_fiber, FIBER_RESUMED);
2654 fiber_setcontext(next_fiber, fiber);
2655}
2656
2657static void
2658fiber_check_killed(rb_fiber_t *fiber)
2659{
2660 VM_ASSERT(fiber == fiber_current());
2661
2662 if (fiber->killed) {
2663 rb_thread_t *thread = fiber->cont.saved_ec.thread_ptr;
2664
2665 thread->ec->errinfo = RUBY_FATAL_FIBER_KILLED;
2666 EC_JUMP_TAG(thread->ec, RUBY_TAG_FATAL);
2667 }
2668}
2669
2670static inline VALUE
2671fiber_switch(rb_fiber_t *fiber, int argc, const VALUE *argv, int kw_splat, rb_fiber_t *resuming_fiber, bool yielding)
2672{
2673 VALUE value;
2674 rb_context_t *cont = &fiber->cont;
2675 rb_thread_t *th = GET_THREAD();
2676
2677 /* make sure the root_fiber object is available */
2678 if (th->root_fiber == NULL) root_fiber_alloc(th);
2679
2680 if (th->ec->fiber_ptr == fiber) {
2681 /* ignore fiber context switch
2682 * because destination fiber is the same as current fiber
2683 */
2684 return make_passing_arg(argc, argv);
2685 }
2686
2687 if (cont_thread_value(cont) != th->self) {
2688 rb_raise(rb_eFiberError, "fiber called across threads");
2689 }
2690
2691 if (FIBER_TERMINATED_P(fiber)) {
2692 value = rb_exc_new2(rb_eFiberError, "dead fiber called");
2693
2694 if (!FIBER_TERMINATED_P(th->ec->fiber_ptr)) {
2695 rb_exc_raise(value);
2696 VM_UNREACHABLE(fiber_switch);
2697 }
2698 else {
2699 /* th->ec->fiber_ptr is also dead => switch to root fiber */
2700 /* (this means we're being called from rb_fiber_terminate, */
2701 /* and the terminated fiber's return_fiber() is already dead) */
2702 VM_ASSERT(FIBER_SUSPENDED_P(th->root_fiber));
2703
2704 cont = &th->root_fiber->cont;
2705 cont->argc = -1;
2706 cont->value = value;
2707
2708 fiber_setcontext(th->root_fiber, th->ec->fiber_ptr);
2709
2710 VM_UNREACHABLE(fiber_switch);
2711 }
2712 }
2713
2714 VM_ASSERT(FIBER_RUNNABLE_P(fiber));
2715
2716 rb_fiber_t *current_fiber = fiber_current();
2717
2718 VM_ASSERT(!current_fiber->resuming_fiber);
2719
2720 if (resuming_fiber) {
2721 current_fiber->resuming_fiber = resuming_fiber;
2722 fiber->prev = fiber_current();
2723 fiber->yielding = 0;
2724 }
2725
2726 VM_ASSERT(!current_fiber->yielding);
2727 if (yielding) {
2728 current_fiber->yielding = 1;
2729 }
2730
2731 if (current_fiber->blocking) {
2732 th->blocking -= 1;
2733 }
2734
2735 cont->argc = argc;
2736 cont->kw_splat = kw_splat;
2737 cont->value = make_passing_arg(argc, argv);
2738
2739 fiber_store(fiber, th);
2740
2741 // We cannot free the stack until the pthread is joined:
2742#ifndef COROUTINE_PTHREAD_CONTEXT
2743 if (resuming_fiber && FIBER_TERMINATED_P(fiber)) {
2744 fiber_stack_release(fiber);
2745 }
2746#endif
2747
2748 if (fiber_current()->blocking) {
2749 th->blocking += 1;
2750 }
2751
2752 RUBY_VM_CHECK_INTS(th->ec);
2753
2754 EXEC_EVENT_HOOK(th->ec, RUBY_EVENT_FIBER_SWITCH, th->self, 0, 0, 0, Qnil);
2755
2756 current_fiber = th->ec->fiber_ptr;
2757 value = current_fiber->cont.value;
2758
2759 fiber_check_killed(current_fiber);
2760
2761 if (current_fiber->cont.argc == -1) {
2762 // Fiber#raise will trigger this path.
2763 rb_exc_raise(value);
2764 }
2765
2766 return value;
2767}
2768
2769VALUE
2770rb_fiber_transfer(VALUE fiber_value, int argc, const VALUE *argv)
2771{
2772 return fiber_switch(fiber_ptr(fiber_value), argc, argv, RB_NO_KEYWORDS, NULL, false);
2773}
2774
2775/*
2776 * call-seq:
2777 * fiber.blocking? -> true or false
2778 *
2779 * Returns +true+ if +fiber+ is blocking and +false+ otherwise.
2780 * Fiber is non-blocking if it was created via passing <tt>blocking: false</tt>
2781 * to Fiber.new, or via Fiber.schedule.
2782 *
2783 * Note that, even if the method returns +false+, the fiber behaves differently
2784 * only if Fiber.scheduler is set in the current thread.
2785 *
2786 * See the "Non-blocking fibers" section in class docs for details.
2787 *
2788 */
2789VALUE
2790rb_fiber_blocking_p(VALUE fiber)
2791{
2792 return RBOOL(fiber_ptr(fiber)->blocking);
2793}
2794
2795static VALUE
2796fiber_blocking_yield(VALUE fiber_value)
2797{
2798 rb_fiber_t *fiber = fiber_ptr(fiber_value);
2799 rb_thread_t * volatile th = fiber->cont.saved_ec.thread_ptr;
2800
2801 VM_ASSERT(fiber->blocking == 0);
2802
2803 // fiber->blocking is `unsigned int : 1`, so we use it as a boolean:
2804 fiber->blocking = 1;
2805
2806 // Once the fiber is blocking, and current, we increment the thread blocking state:
2807 th->blocking += 1;
2808
2809 return rb_yield(fiber_value);
2810}
2811
2812static VALUE
2813fiber_blocking_ensure(VALUE fiber_value)
2814{
2815 rb_fiber_t *fiber = fiber_ptr(fiber_value);
2816 rb_thread_t * volatile th = fiber->cont.saved_ec.thread_ptr;
2817
2818 // We are no longer blocking:
2819 fiber->blocking = 0;
2820 th->blocking -= 1;
2821
2822 return Qnil;
2823}
2824
2825/*
2826 * call-seq:
2827 * Fiber.blocking{|fiber| ...} -> result
2828 *
2829 * Forces the fiber to be blocking for the duration of the block. Returns the
2830 * result of the block.
2831 *
2832 * See the "Non-blocking fibers" section in class docs for details.
2833 *
2834 */
2835VALUE
2836rb_fiber_blocking(VALUE class)
2837{
2838 VALUE fiber_value = rb_fiber_current();
2839 rb_fiber_t *fiber = fiber_ptr(fiber_value);
2840
2841 // If we are already blocking, this is essentially a no-op:
2842 if (fiber->blocking) {
2843 return rb_yield(fiber_value);
2844 }
2845 else {
2846 return rb_ensure(fiber_blocking_yield, fiber_value, fiber_blocking_ensure, fiber_value);
2847 }
2848}
2849
2850/*
2851 * call-seq:
2852 * Fiber.blocking? -> false or 1
2853 *
2854 * Returns +false+ if the current fiber is non-blocking.
2855 * Fiber is non-blocking if it was created via passing <tt>blocking: false</tt>
2856 * to Fiber.new, or via Fiber.schedule.
2857 *
2858 * If the current Fiber is blocking, the method returns 1.
2859 * Future developments may allow for situations where larger integers
2860 * could be returned.
2861 *
2862 * Note that, even if the method returns +false+, Fiber behaves differently
2863 * only if Fiber.scheduler is set in the current thread.
2864 *
2865 * See the "Non-blocking fibers" section in class docs for details.
2866 *
2867 */
2868static VALUE
2869rb_fiber_s_blocking_p(VALUE klass)
2870{
2871 rb_thread_t *thread = GET_THREAD();
2872 unsigned blocking = thread->blocking;
2873
2874 if (blocking == 0)
2875 return Qfalse;
2876
2877 return INT2NUM(blocking);
2878}
2879
2880void
2881rb_fiber_close(rb_fiber_t *fiber)
2882{
2883 fiber_status_set(fiber, FIBER_TERMINATED);
2884}
2885
2886static void
2887rb_fiber_terminate(rb_fiber_t *fiber, int need_interrupt, VALUE error)
2888{
2889 VALUE value = fiber->cont.value;
2890
2891 VM_ASSERT(FIBER_RESUMED_P(fiber));
2892 rb_fiber_close(fiber);
2893
2894 fiber->cont.machine.stack = NULL;
2895 fiber->cont.machine.stack_size = 0;
2896
2897 rb_fiber_t *next_fiber = return_fiber(true);
2898
2899 if (need_interrupt) RUBY_VM_SET_INTERRUPT(&next_fiber->cont.saved_ec);
2900
2901 if (RTEST(error))
2902 fiber_switch(next_fiber, -1, &error, RB_NO_KEYWORDS, NULL, false);
2903 else
2904 fiber_switch(next_fiber, 1, &value, RB_NO_KEYWORDS, NULL, false);
2905 ruby_stop(0);
2906}
2907
2908static VALUE
2909fiber_resume_kw(rb_fiber_t *fiber, int argc, const VALUE *argv, int kw_splat)
2910{
2911 rb_fiber_t *current_fiber = fiber_current();
2912
2913 if (argc == -1 && FIBER_CREATED_P(fiber)) {
2914 rb_raise(rb_eFiberError, "cannot raise exception on unborn fiber");
2915 }
2916 else if (FIBER_TERMINATED_P(fiber)) {
2917 rb_raise(rb_eFiberError, "attempt to resume a terminated fiber");
2918 }
2919 else if (fiber == current_fiber) {
2920 rb_raise(rb_eFiberError, "attempt to resume the current fiber");
2921 }
2922 else if (fiber->prev != NULL) {
2923 rb_raise(rb_eFiberError, "attempt to resume a resumed fiber (double resume)");
2924 }
2925 else if (fiber->resuming_fiber) {
2926 rb_raise(rb_eFiberError, "attempt to resume a resuming fiber");
2927 }
2928 else if (fiber->prev == NULL &&
2929 (!fiber->yielding && fiber->status != FIBER_CREATED)) {
2930 rb_raise(rb_eFiberError, "attempt to resume a transferring fiber");
2931 }
2932
2933 return fiber_switch(fiber, argc, argv, kw_splat, fiber, false);
2934}
2935
2936VALUE
2937rb_fiber_resume_kw(VALUE self, int argc, const VALUE *argv, int kw_splat)
2938{
2939 return fiber_resume_kw(fiber_ptr(self), argc, argv, kw_splat);
2940}
2941
2942VALUE
2943rb_fiber_resume(VALUE self, int argc, const VALUE *argv)
2944{
2945 return fiber_resume_kw(fiber_ptr(self), argc, argv, RB_NO_KEYWORDS);
2946}
2947
2948VALUE
2949rb_fiber_yield_kw(int argc, const VALUE *argv, int kw_splat)
2950{
2951 return fiber_switch(return_fiber(false), argc, argv, kw_splat, NULL, true);
2952}
2953
2954VALUE
2955rb_fiber_yield(int argc, const VALUE *argv)
2956{
2957 return fiber_switch(return_fiber(false), argc, argv, RB_NO_KEYWORDS, NULL, true);
2958}
2959
2960void
2961rb_fiber_reset_root_local_storage(rb_thread_t *th)
2962{
2963 if (th->root_fiber && th->root_fiber != th->ec->fiber_ptr) {
2964 th->ec->local_storage = th->root_fiber->cont.saved_ec.local_storage;
2965 }
2966}
2967
2968/*
2969 * call-seq:
2970 * fiber.alive? -> true or false
2971 *
2972 * Returns true if the fiber can still be resumed (or transferred
2973 * to). After finishing execution of the fiber block this method will
2974 * always return +false+.
2975 */
2976VALUE
2977rb_fiber_alive_p(VALUE fiber_value)
2978{
2979 return RBOOL(!FIBER_TERMINATED_P(fiber_ptr(fiber_value)));
2980}
2981
2982/*
2983 * call-seq:
2984 * fiber.resume(args, ...) -> obj
2985 *
2986 * Resumes the fiber from the point at which the last Fiber.yield was
2987 * called, or starts running it if it is the first call to
2988 * #resume. Arguments passed to resume will be the value of the
2989 * Fiber.yield expression or will be passed as block parameters to
2990 * the fiber's block if this is the first #resume.
2991 *
2992 * Alternatively, when resume is called it evaluates to the arguments passed
2993 * to the next Fiber.yield statement inside the fiber's block
2994 * or to the block value if it runs to completion without any
2995 * Fiber.yield
2996 */
2997static VALUE
2998rb_fiber_m_resume(int argc, VALUE *argv, VALUE fiber)
2999{
3000 return rb_fiber_resume_kw(fiber, argc, argv, rb_keyword_given_p());
3001}
3002
3003/*
3004 * call-seq:
3005 * fiber.backtrace -> array
3006 * fiber.backtrace(start) -> array
3007 * fiber.backtrace(start, count) -> array
3008 * fiber.backtrace(start..end) -> array
3009 *
3010 * Returns the current execution stack of the fiber. +start+, +count+ and +end+ allow
3011 * to select only parts of the backtrace.
3012 *
3013 * def level3
3014 * Fiber.yield
3015 * end
3016 *
3017 * def level2
3018 * level3
3019 * end
3020 *
3021 * def level1
3022 * level2
3023 * end
3024 *
3025 * f = Fiber.new { level1 }
3026 *
3027 * # It is empty before the fiber started
3028 * f.backtrace
3029 * #=> []
3030 *
3031 * f.resume
3032 *
3033 * f.backtrace
3034 * #=> ["test.rb:2:in `yield'", "test.rb:2:in `level3'", "test.rb:6:in `level2'", "test.rb:10:in `level1'", "test.rb:13:in `block in <main>'"]
3035 * p f.backtrace(1) # start from the item 1
3036 * #=> ["test.rb:2:in `level3'", "test.rb:6:in `level2'", "test.rb:10:in `level1'", "test.rb:13:in `block in <main>'"]
3037 * p f.backtrace(2, 2) # start from item 2, take 2
3038 * #=> ["test.rb:6:in `level2'", "test.rb:10:in `level1'"]
3039 * p f.backtrace(1..3) # take items from 1 to 3
3040 * #=> ["test.rb:2:in `level3'", "test.rb:6:in `level2'", "test.rb:10:in `level1'"]
3041 *
3042 * f.resume
3043 *
3044 * # It is nil after the fiber is finished
3045 * f.backtrace
3046 * #=> nil
3047 *
3048 */
3049static VALUE
3050rb_fiber_backtrace(int argc, VALUE *argv, VALUE fiber)
3051{
3052 return rb_vm_backtrace(argc, argv, &fiber_ptr(fiber)->cont.saved_ec);
3053}
3054
3055/*
3056 * call-seq:
3057 * fiber.backtrace_locations -> array
3058 * fiber.backtrace_locations(start) -> array
3059 * fiber.backtrace_locations(start, count) -> array
3060 * fiber.backtrace_locations(start..end) -> array
3061 *
3062 * Like #backtrace, but returns each line of the execution stack as a
3063 * Thread::Backtrace::Location. Accepts the same arguments as #backtrace.
3064 *
3065 * f = Fiber.new { Fiber.yield }
3066 * f.resume
3067 * loc = f.backtrace_locations.first
3068 * loc.label #=> "yield"
3069 * loc.path #=> "test.rb"
3070 * loc.lineno #=> 1
3071 *
3072 *
3073 */
3074static VALUE
3075rb_fiber_backtrace_locations(int argc, VALUE *argv, VALUE fiber)
3076{
3077 return rb_vm_backtrace_locations(argc, argv, &fiber_ptr(fiber)->cont.saved_ec);
3078}
3079
3080/*
3081 * call-seq:
3082 * fiber.transfer(args, ...) -> obj
3083 *
3084 * Transfer control to another fiber, resuming it from where it last
3085 * stopped or starting it if it was not resumed before. The calling
3086 * fiber will be suspended much like in a call to
3087 * Fiber.yield.
3088 *
3089 * The fiber which receives the transfer call treats it much like
3090 * a resume call. Arguments passed to transfer are treated like those
3091 * passed to resume.
3092 *
3093 * The two style of control passing to and from fiber (one is #resume and
3094 * Fiber::yield, another is #transfer to and from fiber) can't be freely
3095 * mixed.
3096 *
3097 * * If the Fiber's lifecycle had started with transfer, it will never
3098 * be able to yield or be resumed control passing, only
3099 * finish or transfer back. (It still can resume other fibers that
3100 * are allowed to be resumed.)
3101 * * If the Fiber's lifecycle had started with resume, it can yield
3102 * or transfer to another Fiber, but can receive control back only
3103 * the way compatible with the way it was given away: if it had
3104 * transferred, it only can be transferred back, and if it had
3105 * yielded, it only can be resumed back. After that, it again can
3106 * transfer or yield.
3107 *
3108 * If those rules are broken FiberError is raised.
3109 *
3110 * For an individual Fiber design, yield/resume is easier to use
3111 * (the Fiber just gives away control, it doesn't need to think
3112 * about who the control is given to), while transfer is more flexible
3113 * for complex cases, allowing to build arbitrary graphs of Fibers
3114 * dependent on each other.
3115 *
3116 *
3117 * Example:
3118 *
3119 * manager = nil # For local var to be visible inside worker block
3120 *
3121 * # This fiber would be started with transfer
3122 * # It can't yield, and can't be resumed
3123 * worker = Fiber.new { |work|
3124 * puts "Worker: starts"
3125 * puts "Worker: Performed #{work.inspect}, transferring back"
3126 * # Fiber.yield # this would raise FiberError: attempt to yield on a not resumed fiber
3127 * # manager.resume # this would raise FiberError: attempt to resume a resumed fiber (double resume)
3128 * manager.transfer(work.capitalize)
3129 * }
3130 *
3131 * # This fiber would be started with resume
3132 * # It can yield or transfer, and can be transferred
3133 * # back or resumed
3134 * manager = Fiber.new {
3135 * puts "Manager: starts"
3136 * puts "Manager: transferring 'something' to worker"
3137 * result = worker.transfer('something')
3138 * puts "Manager: worker returned #{result.inspect}"
3139 * # worker.resume # this would raise FiberError: attempt to resume a transferring fiber
3140 * Fiber.yield # this is OK, the fiber transferred from and to, now it can yield
3141 * puts "Manager: finished"
3142 * }
3143 *
3144 * puts "Starting the manager"
3145 * manager.resume
3146 * puts "Resuming the manager"
3147 * # manager.transfer # this would raise FiberError: attempt to transfer to a yielding fiber
3148 * manager.resume
3149 *
3150 * <em>produces</em>
3151 *
3152 * Starting the manager
3153 * Manager: starts
3154 * Manager: transferring 'something' to worker
3155 * Worker: starts
3156 * Worker: Performed "something", transferring back
3157 * Manager: worker returned "Something"
3158 * Resuming the manager
3159 * Manager: finished
3160 *
3161 */
3162static VALUE
3163rb_fiber_m_transfer(int argc, VALUE *argv, VALUE self)
3164{
3165 return rb_fiber_transfer_kw(self, argc, argv, rb_keyword_given_p());
3166}
3167
3168static VALUE
3169fiber_transfer_kw(rb_fiber_t *fiber, int argc, const VALUE *argv, int kw_splat)
3170{
3171 if (fiber->resuming_fiber) {
3172 rb_raise(rb_eFiberError, "attempt to transfer to a resuming fiber");
3173 }
3174
3175 if (fiber->yielding) {
3176 rb_raise(rb_eFiberError, "attempt to transfer to a yielding fiber");
3177 }
3178
3179 return fiber_switch(fiber, argc, argv, kw_splat, NULL, false);
3180}
3181
3182VALUE
3183rb_fiber_transfer_kw(VALUE self, int argc, const VALUE *argv, int kw_splat)
3184{
3185 return fiber_transfer_kw(fiber_ptr(self), argc, argv, kw_splat);
3186}
3187
3188/*
3189 * call-seq:
3190 * Fiber.yield(args, ...) -> obj
3191 *
3192 * Yields control back to the context that resumed the fiber, passing
3193 * along any arguments that were passed to it. The fiber will resume
3194 * processing at this point when #resume is called next.
3195 * Any arguments passed to the next #resume will be the value that
3196 * this Fiber.yield expression evaluates to.
3197 */
3198static VALUE
3199rb_fiber_s_yield(int argc, VALUE *argv, VALUE klass)
3200{
3201 return rb_fiber_yield_kw(argc, argv, rb_keyword_given_p());
3202}
3203
3204static VALUE
3205fiber_raise(rb_fiber_t *fiber, VALUE exception)
3206{
3207 if (fiber == fiber_current()) {
3208 rb_exc_raise(exception);
3209 }
3210 else if (fiber->resuming_fiber) {
3211 return fiber_raise(fiber->resuming_fiber, exception);
3212 }
3213 else if (FIBER_SUSPENDED_P(fiber) && !fiber->yielding) {
3214 return fiber_transfer_kw(fiber, -1, &exception, RB_NO_KEYWORDS);
3215 }
3216 else {
3217 return fiber_resume_kw(fiber, -1, &exception, RB_NO_KEYWORDS);
3218 }
3219}
3220
3221VALUE
3222rb_fiber_raise(VALUE fiber, int argc, VALUE *argv)
3223{
3224 VALUE exception = rb_exception_setup(argc, argv);
3225
3226 return fiber_raise(fiber_ptr(fiber), exception);
3227}
3228
3229/*
3230 * call-seq:
3231 * fiber.raise -> obj
3232 * fiber.raise(string) -> obj
3233 * fiber.raise(exception [, string [, array]]) -> obj
3234 *
3235 * Raises an exception in the fiber at the point at which the last
3236 * +Fiber.yield+ was called. If the fiber has not been started or has
3237 * already run to completion, raises +FiberError+. If the fiber is
3238 * yielding, it is resumed. If it is transferring, it is transferred into.
3239 * But if it is resuming, raises +FiberError+.
3240 *
3241 * With no arguments, raises a +RuntimeError+. With a single +String+
3242 * argument, raises a +RuntimeError+ with the string as a message. Otherwise,
3243 * the first parameter should be the name of an +Exception+ class (or an
3244 * object that returns an +Exception+ object when sent an +exception+
3245 * message). The optional second parameter sets the message associated with
3246 * the exception, and the third parameter is an array of callback information.
3247 * Exceptions are caught by the +rescue+ clause of <code>begin...end</code>
3248 * blocks.
3249 *
3250 * Raises +FiberError+ if called on a Fiber belonging to another +Thread+.
3251 *
3252 * See Kernel#raise for more information.
3253 */
3254static VALUE
3255rb_fiber_m_raise(int argc, VALUE *argv, VALUE self)
3256{
3257 return rb_fiber_raise(self, argc, argv);
3258}
3259
3260/*
3261 * call-seq:
3262 * fiber.kill -> nil
3263 *
3264 * Terminates the fiber by raising an uncatchable exception.
3265 * It only terminates the given fiber and no other fiber, returning +nil+ to
3266 * another fiber if that fiber was calling #resume or #transfer.
3267 *
3268 * <tt>Fiber#kill</tt> only interrupts another fiber when it is in Fiber.yield.
3269 * If called on the current fiber then it raises that exception at the <tt>Fiber#kill</tt> call site.
3270 *
3271 * If the fiber has not been started, transition directly to the terminated state.
3272 *
3273 * If the fiber is already terminated, does nothing.
3274 *
3275 * Raises FiberError if called on a fiber belonging to another thread.
3276 */
3277static VALUE
3278rb_fiber_m_kill(VALUE self)
3279{
3280 rb_fiber_t *fiber = fiber_ptr(self);
3281
3282 if (fiber->killed) return Qfalse;
3283 fiber->killed = 1;
3284
3285 if (fiber->status == FIBER_CREATED) {
3286 fiber->status = FIBER_TERMINATED;
3287 }
3288 else if (fiber->status != FIBER_TERMINATED) {
3289 if (fiber_current() == fiber) {
3290 fiber_check_killed(fiber);
3291 }
3292 else {
3293 fiber_raise(fiber_ptr(self), Qnil);
3294 }
3295 }
3296
3297 return self;
3298}
3299
3300/*
3301 * call-seq:
3302 * Fiber.current -> fiber
3303 *
3304 * Returns the current fiber. If you are not running in the context of
3305 * a fiber this method will return the root fiber.
3306 */
3307static VALUE
3308rb_fiber_s_current(VALUE klass)
3309{
3310 return rb_fiber_current();
3311}
3312
3313static VALUE
3314fiber_to_s(VALUE fiber_value)
3315{
3316 const rb_fiber_t *fiber = fiber_ptr(fiber_value);
3317 const rb_proc_t *proc;
3318 char status_info[0x20];
3319
3320 if (fiber->resuming_fiber) {
3321 snprintf(status_info, 0x20, " (%s by resuming)", fiber_status_name(fiber->status));
3322 }
3323 else {
3324 snprintf(status_info, 0x20, " (%s)", fiber_status_name(fiber->status));
3325 }
3326
3327 if (!rb_obj_is_proc(fiber->first_proc)) {
3328 VALUE str = rb_any_to_s(fiber_value);
3329 strlcat(status_info, ">", sizeof(status_info));
3330 rb_str_set_len(str, RSTRING_LEN(str)-1);
3331 rb_str_cat_cstr(str, status_info);
3332 return str;
3333 }
3334 GetProcPtr(fiber->first_proc, proc);
3335 return rb_block_to_s(fiber_value, &proc->block, status_info);
3336}
3337
3338#ifdef HAVE_WORKING_FORK
3339void
3340rb_fiber_atfork(rb_thread_t *th)
3341{
3342 if (th->root_fiber) {
3343 if (&th->root_fiber->cont.saved_ec != th->ec) {
3344 th->root_fiber = th->ec->fiber_ptr;
3345 }
3346 th->root_fiber->prev = 0;
3347 }
3348}
3349#endif
3350
3351#ifdef RB_EXPERIMENTAL_FIBER_POOL
3352static void
3353fiber_pool_free(void *ptr)
3354{
3355 struct fiber_pool * fiber_pool = ptr;
3356 RUBY_FREE_ENTER("fiber_pool");
3357
3358 fiber_pool_allocation_free(fiber_pool->allocations);
3359 ruby_xfree(fiber_pool);
3360
3361 RUBY_FREE_LEAVE("fiber_pool");
3362}
3363
3364static size_t
3365fiber_pool_memsize(const void *ptr)
3366{
3367 const struct fiber_pool * fiber_pool = ptr;
3368 size_t size = sizeof(*fiber_pool);
3369
3370 size += fiber_pool->count * fiber_pool->size;
3371
3372 return size;
3373}
3374
3375static const rb_data_type_t FiberPoolDataType = {
3376 "fiber_pool",
3377 {NULL, fiber_pool_free, fiber_pool_memsize,},
3378 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
3379};
3380
3381static VALUE
3382fiber_pool_alloc(VALUE klass)
3383{
3384 struct fiber_pool *fiber_pool;
3385
3386 return TypedData_Make_Struct(klass, struct fiber_pool, &FiberPoolDataType, fiber_pool);
3387}
3388
3389static VALUE
3390rb_fiber_pool_initialize(int argc, VALUE* argv, VALUE self)
3391{
3392 rb_thread_t *th = GET_THREAD();
3393 VALUE size = Qnil, count = Qnil, vm_stack_size = Qnil;
3394 struct fiber_pool * fiber_pool = NULL;
3395
3396 // Maybe these should be keyword arguments.
3397 rb_scan_args(argc, argv, "03", &size, &count, &vm_stack_size);
3398
3399 if (NIL_P(size)) {
3400 size = SIZET2NUM(th->vm->default_params.fiber_machine_stack_size);
3401 }
3402
3403 if (NIL_P(count)) {
3404 count = INT2NUM(128);
3405 }
3406
3407 if (NIL_P(vm_stack_size)) {
3408 vm_stack_size = SIZET2NUM(th->vm->default_params.fiber_vm_stack_size);
3409 }
3410
3411 TypedData_Get_Struct(self, struct fiber_pool, &FiberPoolDataType, fiber_pool);
3412
3413 fiber_pool_initialize(fiber_pool, NUM2SIZET(size), NUM2SIZET(count), NUM2SIZET(vm_stack_size));
3414
3415 return self;
3416}
3417#endif
3418
3419/*
3420 * Document-class: FiberError
3421 *
3422 * Raised when an invalid operation is attempted on a Fiber, in
3423 * particular when attempting to call/resume a dead fiber,
3424 * attempting to yield from the root fiber, or calling a fiber across
3425 * threads.
3426 *
3427 * fiber = Fiber.new{}
3428 * fiber.resume #=> nil
3429 * fiber.resume #=> FiberError: dead fiber called
3430 */
3431
3432void
3433Init_Cont(void)
3434{
3435 rb_thread_t *th = GET_THREAD();
3436 size_t vm_stack_size = th->vm->default_params.fiber_vm_stack_size;
3437 size_t machine_stack_size = th->vm->default_params.fiber_machine_stack_size;
3438 size_t stack_size = machine_stack_size + vm_stack_size;
3439
3440#ifdef _WIN32
3441 SYSTEM_INFO info;
3442 GetSystemInfo(&info);
3443 pagesize = info.dwPageSize;
3444#else /* not WIN32 */
3445 pagesize = sysconf(_SC_PAGESIZE);
3446#endif
3447 SET_MACHINE_STACK_END(&th->ec->machine.stack_end);
3448
3449 fiber_pool_initialize(&shared_fiber_pool, stack_size, FIBER_POOL_INITIAL_SIZE, vm_stack_size);
3450
3451 fiber_initialize_keywords[0] = rb_intern_const("blocking");
3452 fiber_initialize_keywords[1] = rb_intern_const("pool");
3453 fiber_initialize_keywords[2] = rb_intern_const("storage");
3454
3455 const char *fiber_shared_fiber_pool_free_stacks = getenv("RUBY_SHARED_FIBER_POOL_FREE_STACKS");
3456 if (fiber_shared_fiber_pool_free_stacks) {
3457 shared_fiber_pool.free_stacks = atoi(fiber_shared_fiber_pool_free_stacks);
3458
3459 if (shared_fiber_pool.free_stacks < 0) {
3460 rb_warn("Setting RUBY_SHARED_FIBER_POOL_FREE_STACKS to a negative value is not allowed.");
3461 shared_fiber_pool.free_stacks = 0;
3462 }
3463
3464 if (shared_fiber_pool.free_stacks > 1) {
3465 rb_warn("Setting RUBY_SHARED_FIBER_POOL_FREE_STACKS to a value greater than 1 is operating system specific, and may cause crashes.");
3466 }
3467 }
3468
3469 rb_cFiber = rb_define_class("Fiber", rb_cObject);
3470 rb_define_alloc_func(rb_cFiber, fiber_alloc);
3471 rb_eFiberError = rb_define_class("FiberError", rb_eStandardError);
3472 rb_define_singleton_method(rb_cFiber, "yield", rb_fiber_s_yield, -1);
3473 rb_define_singleton_method(rb_cFiber, "current", rb_fiber_s_current, 0);
3474 rb_define_singleton_method(rb_cFiber, "blocking", rb_fiber_blocking, 0);
3475 rb_define_singleton_method(rb_cFiber, "[]", rb_fiber_storage_aref, 1);
3476 rb_define_singleton_method(rb_cFiber, "[]=", rb_fiber_storage_aset, 2);
3477
3478 rb_define_method(rb_cFiber, "initialize", rb_fiber_initialize, -1);
3479 rb_define_method(rb_cFiber, "blocking?", rb_fiber_blocking_p, 0);
3480 rb_define_method(rb_cFiber, "storage", rb_fiber_storage_get, 0);
3481 rb_define_method(rb_cFiber, "storage=", rb_fiber_storage_set, 1);
3482 rb_define_method(rb_cFiber, "resume", rb_fiber_m_resume, -1);
3483 rb_define_method(rb_cFiber, "raise", rb_fiber_m_raise, -1);
3484 rb_define_method(rb_cFiber, "kill", rb_fiber_m_kill, 0);
3485 rb_define_method(rb_cFiber, "backtrace", rb_fiber_backtrace, -1);
3486 rb_define_method(rb_cFiber, "backtrace_locations", rb_fiber_backtrace_locations, -1);
3487 rb_define_method(rb_cFiber, "to_s", fiber_to_s, 0);
3488 rb_define_alias(rb_cFiber, "inspect", "to_s");
3489 rb_define_method(rb_cFiber, "transfer", rb_fiber_m_transfer, -1);
3490 rb_define_method(rb_cFiber, "alive?", rb_fiber_alive_p, 0);
3491
3492 rb_define_singleton_method(rb_cFiber, "blocking?", rb_fiber_s_blocking_p, 0);
3493 rb_define_singleton_method(rb_cFiber, "scheduler", rb_fiber_s_scheduler, 0);
3494 rb_define_singleton_method(rb_cFiber, "set_scheduler", rb_fiber_set_scheduler, 1);
3495 rb_define_singleton_method(rb_cFiber, "current_scheduler", rb_fiber_current_scheduler, 0);
3496
3497 rb_define_singleton_method(rb_cFiber, "schedule", rb_fiber_s_schedule, -1);
3498
3499#ifdef RB_EXPERIMENTAL_FIBER_POOL
3500 /*
3501 * Document-class: Fiber::Pool
3502 * :nodoc: experimental
3503 */
3504 rb_cFiberPool = rb_define_class_under(rb_cFiber, "Pool", rb_cObject);
3505 rb_define_alloc_func(rb_cFiberPool, fiber_pool_alloc);
3506 rb_define_method(rb_cFiberPool, "initialize", rb_fiber_pool_initialize, -1);
3507#endif
3508
3509 rb_provide("fiber.so");
3510}
3511
3512RUBY_SYMBOL_EXPORT_BEGIN
3513
3514void
3515ruby_Init_Continuation_body(void)
3516{
3517 rb_cContinuation = rb_define_class("Continuation", rb_cObject);
3518 rb_undef_alloc_func(rb_cContinuation);
3519 rb_undef_method(CLASS_OF(rb_cContinuation), "new");
3520 rb_define_method(rb_cContinuation, "call", rb_cont_call, -1);
3521 rb_define_method(rb_cContinuation, "[]", rb_cont_call, -1);
3522 rb_define_global_function("callcc", rb_callcc, 0);
3523}
3524
3525RUBY_SYMBOL_EXPORT_END
#define rb_define_method(klass, mid, func, arity)
Defines klass#mid.
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
#define rb_define_global_function(mid, func, arity)
Defines rb_mKernel #mid.
#define RUBY_EVENT_FIBER_SWITCH
Encountered a Fiber#yield.
Definition event.h:59
static bool RB_OBJ_FROZEN(VALUE obj)
Checks if an object is frozen.
Definition fl_type.h:892
VALUE rb_define_class(const char *name, VALUE super)
Defines a top-level class.
Definition class.c:1474
VALUE rb_define_class_under(VALUE outer, const char *name, VALUE super)
Defines a class under the namespace of outer.
Definition class.c:1510
void rb_define_alias(VALUE klass, const char *name1, const char *name2)
Defines an alias of a method.
Definition class.c:2843
void rb_undef_method(VALUE klass, const char *name)
Defines an undef of a method.
Definition class.c:2663
int rb_scan_args_kw(int kw_flag, int argc, const VALUE *argv, const char *fmt,...)
Identical to rb_scan_args(), except it also accepts kw_splat.
Definition class.c:3146
int rb_scan_args(int argc, const VALUE *argv, const char *fmt,...)
Retrieves argument from argc and argv to given VALUE references according to the format string.
Definition class.c:3133
int rb_keyword_given_p(void)
Determines if the current method is given a keyword argument.
Definition eval.c:1049
int rb_get_kwargs(VALUE keyword_hash, const ID *table, int required, int optional, VALUE *values)
Keyword argument deconstructor.
Definition class.c:2922
#define REALLOC_N
Old name of RB_REALLOC_N.
Definition memory.h:403
#define xfree
Old name of ruby_xfree.
Definition xmalloc.h:58
#define Qundef
Old name of RUBY_Qundef.
#define UNREACHABLE_RETURN
Old name of RBIMPL_UNREACHABLE_RETURN.
Definition assume.h:29
#define ZALLOC
Old name of RB_ZALLOC.
Definition memory.h:402
#define CLASS_OF
Old name of rb_class_of.
Definition globals.h:206
#define rb_ary_new4
Old name of rb_ary_new_from_values.
Definition array.h:659
#define SIZET2NUM
Old name of RB_SIZE2NUM.
Definition size_t.h:62
#define rb_exc_new2
Old name of rb_exc_new_cstr.
Definition error.h:37
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
#define ALLOC_N
Old name of RB_ALLOC_N.
Definition memory.h:399
#define Qtrue
Old name of RUBY_Qtrue.
#define INT2NUM
Old name of RB_INT2NUM.
Definition int.h:43
#define Qnil
Old name of RUBY_Qnil.
#define Qfalse
Old name of RUBY_Qfalse.
#define NIL_P
Old name of RB_NIL_P.
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
Definition value_type.h:80
#define NUM2SIZET
Old name of RB_NUM2SIZE.
Definition size_t.h:61
void ruby_stop(int ex)
Calls ruby_cleanup() and exits the process.
Definition eval.c:289
void rb_category_warn(rb_warning_category_t category, const char *fmt,...)
Identical to rb_category_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:476
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
Definition eval.c:682
int rb_typeddata_is_kind_of(VALUE obj, const rb_data_type_t *data_type)
Checks if the given object is of given kind.
Definition error.c:1380
void rb_syserr_fail(int e, const char *mesg)
Raises appropriate exception that represents a C errno.
Definition error.c:3905
VALUE rb_eStandardError
StandardError exception.
Definition error.c:1427
VALUE rb_eFrozenError
FrozenError exception.
Definition error.c:1429
VALUE rb_eTypeError
TypeError exception.
Definition error.c:1430
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1428
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:466
@ RB_WARN_CATEGORY_EXPERIMENTAL
Warning is for experimental features.
Definition error.h:51
VALUE rb_any_to_s(VALUE obj)
Generates a textual representation of the given object.
Definition object.c:644
VALUE rb_obj_dup(VALUE obj)
Duplicates the given object.
Definition object.c:551
void rb_provide(const char *feature)
Declares that the given feature is already provided by someone else.
Definition load.c:767
VALUE rb_block_proc(void)
Constructs a Proc object from implicitly passed components.
Definition proc.c:847
VALUE rb_obj_is_proc(VALUE recv)
Queries if the given object is a proc.
Definition proc.c:120
void rb_str_set_len(VALUE str, long len)
Overwrites the length of the string.
Definition string.c:3347
#define rb_str_cat_cstr(buf, str)
Identical to rb_str_cat(), except it assumes the passed pointer is a pointer to a C string.
Definition string.h:1656
void rb_undef_alloc_func(VALUE klass)
Deletes the allocator function of a class.
Definition vm_method.c:1591
void rb_define_alloc_func(VALUE klass, rb_alloc_func_t func)
Sets the allocator function of a class.
static ID rb_intern_const(const char *str)
This is a "tiny optimisation" over rb_intern().
Definition symbol.h:284
VALUE rb_to_symbol(VALUE name)
Identical to rb_intern_str(), except it generates a dynamic symbol if necessary.
Definition string.c:12552
VALUE rb_yield(VALUE val)
Yields the block.
Definition vm_eval.c:1372
rb_block_call_func * rb_block_call_func_t
Shorthand type that represents an iterator-written-in-C function pointer.
Definition iterator.h:88
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition memory.h:372
#define ALLOCA_N(type, n)
Definition memory.h:292
#define RB_ALLOC(type)
Shorthand of RB_ALLOC_N with n=1.
Definition memory.h:213
VALUE rb_proc_new(type *q, VALUE w)
Creates a rb_cProc instance.
void rb_hash_foreach(VALUE q, int_type *w, VALUE e)
Iteration over the given hash.
VALUE rb_ensure(type *q, VALUE w, type *e, VALUE r)
An equivalent of ensure clause.
#define RARRAY_CONST_PTR
Just another name of rb_array_const_ptr.
Definition rarray.h:52
#define DATA_PTR(obj)
Convenient getter macro.
Definition rdata.h:67
#define TypedData_Get_Struct(obj, type, data_type, sval)
Obtains a C struct from inside of a wrapper Ruby object.
Definition rtypeddata.h:515
#define TypedData_Wrap_Struct(klass, data_type, sval)
Converts sval, a pointer to your struct, into a Ruby object.
Definition rtypeddata.h:450
#define TypedData_Make_Struct(klass, type, data_type, sval)
Identical to TypedData_Wrap_Struct, except it allocates a new data region internally instead of takin...
Definition rtypeddata.h:497
#define errno
Ractor-aware version of errno.
Definition ruby.h:388
#define RB_NO_KEYWORDS
Do not pass keywords.
Definition scan_args.h:69
Scheduler APIs.
VALUE rb_fiber_scheduler_current(void)
Identical to rb_fiber_scheduler_get(), except it also returns RUBY_Qnil in case of a blocking fiber.
Definition scheduler.c:463
VALUE rb_fiber_scheduler_set(VALUE scheduler)
Destructively assigns the passed scheduler to that of the current thread that is calling this functio...
Definition scheduler.c:424
VALUE rb_fiber_scheduler_get(void)
Queries the current scheduler of the current thread that is calling this function.
Definition scheduler.c:374
VALUE rb_fiber_scheduler_fiber(VALUE scheduler, int argc, VALUE *argv, int kw_splat)
Create and schedule a non-blocking fiber.
Definition scheduler.c:1109
#define RTEST
This is an old name of RB_TEST.
This is the struct that holds necessary info for a struct.
Definition rtypeddata.h:203
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
static void Check_Type(VALUE v, enum ruby_value_type t)
Identical to RB_TYPE_P(), except it raises exceptions on predication failure.
Definition value_type.h:433
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition value_type.h:376