Ruby 4.1.0dev (2026-11-16 revision 67814cf9b0f3a4bad6f7369b0c8e6fc69220de64)
thread_pthread.c (67814cf9b0f3a4bad6f7369b0c8e6fc69220de64)
1/* -*-c-*- */
2/**********************************************************************
3
4 thread_pthread.c -
5
6 $Author$
7
8 Copyright (C) 2004-2007 Koichi Sasada
9
10**********************************************************************/
11
12#ifdef THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION
13
14#include "internal/gc.h"
15#include "internal/sanitizers.h"
16
17#ifdef HAVE_SYS_RESOURCE_H
18#include <sys/resource.h>
19#endif
20#ifdef HAVE_THR_STKSEGMENT
21#include <thread.h>
22#endif
23#if defined(HAVE_FCNTL_H)
24#include <fcntl.h>
25#elif defined(HAVE_SYS_FCNTL_H)
26#include <sys/fcntl.h>
27#endif
28#ifdef HAVE_SYS_PRCTL_H
29#include <sys/prctl.h>
30#endif
31#if defined(HAVE_SYS_TIME_H)
32#include <sys/time.h>
33#endif
34#if defined(__HAIKU__)
35#include <kernel/OS.h>
36#endif
37#ifdef __linux__
38#include <sys/syscall.h> /* for SYS_gettid */
39#endif
40#include <time.h>
41#include <signal.h>
42
43#include "probes.h"
44
45#if defined(HAVE_SYS_EVENTFD_H) && defined(HAVE_EVENTFD)
46# define USE_EVENTFD (1)
47# include <sys/eventfd.h>
48#else
49# define USE_EVENTFD (0)
50#endif
51
52#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && \
53 defined(CLOCK_REALTIME) && defined(CLOCK_MONOTONIC) && \
54 defined(HAVE_CLOCK_GETTIME)
55static pthread_condattr_t condattr_mono;
56static pthread_condattr_t *condattr_monotonic = &condattr_mono;
57#else
58static const void *const condattr_monotonic = NULL;
59#endif
60
61// Whether native_cond_timedwait() takes an rb_hrtime_t deadline as is. It
62// does when the condvar counts in the same clock rb_hrtime_now() reads;
63// otherwise the caller has to restate the deadline in the condvar's clock.
64#define RB_NATIVE_COND_HRTIME_DEADLINE_P() (condattr_monotonic != NULL)
65
66#include COROUTINE_H
67
68#ifndef HAVE_SYS_EVENT_H
69#define HAVE_SYS_EVENT_H 0
70#endif
71
72#ifndef HAVE_SYS_EPOLL_H
73#define HAVE_SYS_EPOLL_H 0
74#else
75// force setting for debug
76// #undef HAVE_SYS_EPOLL_H
77// #define HAVE_SYS_EPOLL_H 0
78#endif
79
80#ifndef USE_MN_THREADS
81 #if defined(__EMSCRIPTEN__) || defined(COROUTINE_PTHREAD_CONTEXT)
82 // on __EMSCRIPTEN__ provides epoll* declarations, but no implementations.
83 // on COROUTINE_PTHREAD_CONTEXT, it doesn't worth to use it.
84 #define USE_MN_THREADS 0
85 #elif HAVE_SYS_EPOLL_H
86 #include <sys/epoll.h>
87 #ifdef EPOLLONESHOT
88 #define USE_MN_THREADS 1
89 #else
90 // the scheduler arms io fds with EPOLLONESHOT (Linux 2.6.2)
91 #define USE_MN_THREADS 0
92 #endif
93 #elif HAVE_SYS_EVENT_H
94 #include <sys/event.h>
95 #define USE_MN_THREADS 1
96 #else
97 #define USE_MN_THREADS 0
98 #endif
99#endif
100
101#ifdef HAVE_SCHED_YIELD
102#define native_thread_yield() (void)sched_yield()
103#else
104#define native_thread_yield() ((void)0)
105#endif
106
107// native thread wrappers
108
109#define NATIVE_MUTEX_LOCK_DEBUG 0
110#define NATIVE_MUTEX_LOCK_DEBUG_YIELD 0
111
112static void
113mutex_debug(const char *msg, void *lock)
114{
115 if (NATIVE_MUTEX_LOCK_DEBUG) {
116 int r;
117 static pthread_mutex_t dbglock = PTHREAD_MUTEX_INITIALIZER;
118
119 if ((r = pthread_mutex_lock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
120 fprintf(stdout, "%s: %p\n", msg, lock);
121 if ((r = pthread_mutex_unlock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
122 }
123}
124
125void
126rb_native_mutex_lock(pthread_mutex_t *lock)
127{
128 int r;
129#if NATIVE_MUTEX_LOCK_DEBUG_YIELD
130 native_thread_yield();
131#endif
132 mutex_debug("lock", lock);
133 if ((r = pthread_mutex_lock(lock)) != 0) {
134 rb_bug_errno("pthread_mutex_lock", r);
135 }
136}
137
138void
139rb_native_mutex_unlock(pthread_mutex_t *lock)
140{
141 int r;
142 mutex_debug("unlock", lock);
143 if ((r = pthread_mutex_unlock(lock)) != 0) {
144 rb_bug_errno("pthread_mutex_unlock", r);
145 }
146}
147
148int
149rb_native_mutex_trylock(pthread_mutex_t *lock)
150{
151 int r;
152 mutex_debug("trylock", lock);
153 if ((r = pthread_mutex_trylock(lock)) != 0) {
154 if (r == EBUSY) {
155 return EBUSY;
156 }
157 else {
158 rb_bug_errno("pthread_mutex_trylock", r);
159 }
160 }
161 return 0;
162}
163
164void
165rb_native_mutex_initialize(pthread_mutex_t *lock)
166{
167 int r = pthread_mutex_init(lock, 0);
168 mutex_debug("init", lock);
169 if (r != 0) {
170 rb_bug_errno("pthread_mutex_init", r);
171 }
172}
173
174void
175rb_native_mutex_destroy(pthread_mutex_t *lock)
176{
177 int r = pthread_mutex_destroy(lock);
178 mutex_debug("destroy", lock);
179 if (r != 0) {
180 rb_bug_errno("pthread_mutex_destroy", r);
181 }
182}
183
184void
185rb_native_cond_initialize(rb_nativethread_cond_t *cond)
186{
187 int r = pthread_cond_init(cond, condattr_monotonic);
188 if (r != 0) {
189 rb_bug_errno("pthread_cond_init", r);
190 }
191}
192
193void
194rb_native_cond_destroy(rb_nativethread_cond_t *cond)
195{
196 int r = pthread_cond_destroy(cond);
197 if (r != 0) {
198 rb_bug_errno("pthread_cond_destroy", r);
199 }
200}
201
202/*
203 * In OS X 10.7 (Lion), pthread_cond_signal and pthread_cond_broadcast return
204 * EAGAIN after retrying 8192 times. You can see them in the following page:
205 *
206 * http://www.opensource.apple.com/source/Libc/Libc-763.11/pthreads/pthread_cond.c
207 *
208 * The following rb_native_cond_signal and rb_native_cond_broadcast functions
209 * need to retrying until pthread functions don't return EAGAIN.
210 */
211
212void
213rb_native_cond_signal(rb_nativethread_cond_t *cond)
214{
215 int r;
216 do {
217 r = pthread_cond_signal(cond);
218 } while (r == EAGAIN);
219 if (r != 0) {
220 rb_bug_errno("pthread_cond_signal", r);
221 }
222}
223
224void
225rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
226{
227 int r;
228 do {
229 r = pthread_cond_broadcast(cond);
230 } while (r == EAGAIN);
231 if (r != 0) {
232 rb_bug_errno("rb_native_cond_broadcast", r);
233 }
234}
235
236void
237rb_native_cond_wait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex)
238{
239 int r = pthread_cond_wait(cond, mutex);
240 if (r != 0) {
241 rb_bug_errno("pthread_cond_wait", r);
242 }
243}
244
245static int
246native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex, const rb_hrtime_t *abs)
247{
248 int r;
249 struct timespec ts;
250
251 /*
252 * An old Linux may return EINTR. Even though POSIX says
253 * "These functions shall not return an error code of [EINTR]".
254 * http://pubs.opengroup.org/onlinepubs/009695399/functions/pthread_cond_timedwait.html
255 * Let's hide it from arch generic code.
256 */
257 do {
258 rb_hrtime2timespec(&ts, abs);
259 r = pthread_cond_timedwait(cond, mutex, &ts);
260 } while (r == EINTR);
261
262 if (r != 0 && r != ETIMEDOUT) {
263 rb_bug_errno("pthread_cond_timedwait", r);
264 }
265
266 return r;
267}
268
269static rb_hrtime_t
270native_cond_timeout(rb_nativethread_cond_t *cond, const rb_hrtime_t rel)
271{
272 if (condattr_monotonic) {
273 return rb_hrtime_add(rb_hrtime_now(), rel);
274 }
275 else {
276 struct timespec ts;
277
278 rb_timespec_now(&ts);
279 return rb_hrtime_add(rb_timespec2hrtime(&ts), rel);
280 }
281}
282
283void
284rb_native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex, unsigned long msec)
285{
286 rb_hrtime_t hrmsec = native_cond_timeout(cond, RB_HRTIME_PER_MSEC * msec);
287 native_cond_timedwait(cond, mutex, &hrmsec);
288}
289
290// thread scheduling
291
292static rb_internal_thread_event_hook_t *rb_internal_thread_event_hooks = NULL;
293static void rb_thread_execute_hooks(rb_event_flag_t event, rb_thread_t *th);
294
295#if 0
296static const char *
297event_name(rb_event_flag_t event)
298{
299 switch (event) {
301 return "STARTED";
303 return "READY";
305 return "RESUMED";
307 return "SUSPENDED";
309 return "EXITED";
310 }
311 return "no-event";
312}
313
314#define RB_INTERNAL_THREAD_HOOK(event, th) \
315 if (UNLIKELY(rb_internal_thread_event_hooks)) { \
316 fprintf(stderr, "[thread=%"PRIxVALUE"] %s in %s (%s:%d)\n", th->self, event_name(event), __func__, __FILE__, __LINE__); \
317 rb_thread_execute_hooks(event, th); \
318 }
319#else
320#define RB_INTERNAL_THREAD_HOOK(event, th) if (UNLIKELY(rb_internal_thread_event_hooks)) { rb_thread_execute_hooks(event, th); }
321#endif
322
323static rb_serial_t current_fork_gen = 1; /* We can't use GET_VM()->fork_gen */
324
325#if defined(SIGVTALRM) && !defined(__EMSCRIPTEN__)
326# define USE_UBF_LIST 1
327#endif
328
329#if USE_MN_THREADS
330static void nt_machine_stack_atfork(void);
331
332// A coroutine thread's execution context: the coroutine_context (first, so
333// th->sched.context points at the whole block) plus what is needed to free
334// it without the rb_thread_t. Owned by the execution: the dying thread marks
335// it dead before its final transfer, and whichever context RESUMES from that
336// transfer frees it -- the resume itself proves the transfer's register save
337// into this block has completed, so no further synchronization is needed.
338struct rb_thread_context {
339 struct coroutine_context co; // must be first
340 void *stack; // the coroutine machine stack (pool stack)
341 struct rb_native_thread *nt; // final transfer target, stashed at termination
342 bool dead;
343};
344
345static bool thread_sched_reclaim(struct coroutine_context *dead_co);
346#endif
347
348
349#ifdef RB_THREAD_T_HAS_NATIVE_ID
350static int
351get_native_thread_id(void)
352{
353#ifdef __linux__
354 return (int)syscall(SYS_gettid);
355#elif defined(__FreeBSD__)
356 return pthread_getthreadid_np();
357#endif
358}
359#endif
360
361
362#ifdef RB_THREAD_LOCAL_SPECIFIER
363static RB_THREAD_LOCAL_SPECIFIER rb_thread_t *ruby_native_thread;
364#else
365static pthread_key_t ruby_native_thread_key;
366#endif
367
368static void
369null_func(int i)
370{
371 /* null */
372 // This function can be called from signal handler
373 // RUBY_DEBUG_LOG("i:%d", i);
374}
375
376rb_thread_t *
377ruby_thread_from_native(void)
378{
379#ifdef RB_THREAD_LOCAL_SPECIFIER
380 return ruby_native_thread;
381#else
382 return pthread_getspecific(ruby_native_thread_key);
383#endif
384}
385
386int
387ruby_thread_set_native(rb_thread_t *th)
388{
389 if (th) {
390#ifdef USE_UBF_LIST
391 ccan_list_node_init(&th->sched.node.ubf);
392#endif
393 }
394
395 // setup TLS
396
397 if (th && th->ec) {
398 rb_ractor_set_current_ec(th->ractor, th->ec);
399 }
400#ifdef RB_THREAD_LOCAL_SPECIFIER
401 ruby_native_thread = th;
402 return 1;
403#else
404 return pthread_setspecific(ruby_native_thread_key, th) == 0;
405#endif
406}
407
408static void native_thread_setup(struct rb_native_thread *nt);
409static void native_thread_setup_on_thread(struct rb_native_thread *nt);
410
411// Internal cache of page size:
412static size_t RB_THREAD_PAGE_SIZE;
413
414void
415Init_native_thread(rb_thread_t *main_th)
416{
417 // Get the system page size for later use in stack allocation and stack overflow checks:
418 RB_THREAD_PAGE_SIZE = sysconf(_SC_PAGESIZE);
419
420#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK)
421 if (condattr_monotonic) {
422 int r = pthread_condattr_init(condattr_monotonic);
423 if (r == 0) {
424 r = pthread_condattr_setclock(condattr_monotonic, CLOCK_MONOTONIC);
425 }
426 if (r) condattr_monotonic = NULL;
427 }
428#endif
429
430#ifndef RB_THREAD_LOCAL_SPECIFIER
431 if (pthread_key_create(&ruby_native_thread_key, 0) == EAGAIN) {
432 rb_bug("pthread_key_create failed (ruby_native_thread_key)");
433 }
434 if (pthread_key_create(&ruby_current_ec_key, 0) == EAGAIN) {
435 rb_bug("pthread_key_create failed (ruby_current_ec_key)");
436 }
437#endif
438 ruby_posix_signal(SIGVTALRM, null_func);
439
440 // setup vm
441 rb_vm_t *vm = main_th->vm;
442 thread_sched_init_vm(vm);
443
444 // setup main thread
445 main_th->nt->serial = 1;
446#ifdef RUBY_NT_SERIAL
447 ruby_nt_serial = 1;
448#endif
449 ruby_thread_set_native(main_th);
450 native_thread_setup(main_th->nt);
451 native_thread_setup_on_thread(main_th->nt);
452
453 TH_SCHED(main_th)->running = main_th;
454 main_th->has_dedicated_nt = 1;
455
456 // setup main NT (before the record below: its kind decides where it goes)
457 main_th->nt->dedicated = 1;
458 main_th->nt->running_thread = main_th;
459 main_th->nt->vm = vm;
460
461 thread_sched_setup_running_threads(TH_SCHED(main_th), main_th->ractor, vm, main_th, NULL);
462
463 // setup mn
464#if USE_RUBY_DEBUG_LOG
465 vm->ractor.sched.dnt_cnt = 1;
466#endif
467}
468
469
470static void
471native_thread_destroy_atfork(struct rb_native_thread *nt)
472{
473 if (nt) {
474 /* We can't call rb_native_cond_destroy here because according to the
475 * specs of pthread_cond_destroy:
476 *
477 * Attempting to destroy a condition variable upon which other threads
478 * are currently blocked results in undefined behavior.
479 *
480 * Specifically, glibc's pthread_cond_destroy waits on all the other
481 * listeners. Since after forking all the threads are dead, the condition
482 * variable's listeners will never wake up, so it will hang forever.
483 */
484
485 RB_ALTSTACK_FREE(nt->altstack);
486 SIZED_FREE(nt->nt_context);
487 SIZED_FREE(nt);
488 }
489}
490
491static void
492native_thread_destroy(struct rb_native_thread *nt)
493{
494 if (nt) {
495 rb_native_cond_destroy(&nt->readyq);
496 rb_native_mutex_destroy(&nt->running_th_lock);
497
498 native_thread_destroy_atfork(nt);
499 }
500}
501
502// A retiring snt frees its own rb_native_thread as it exits. Disarm the
503// altstack registration first: a signal between the free and the thread's
504// end must not run on the freed block.
505static void
506native_thread_destroy_self(struct rb_native_thread *nt)
507{
508#ifdef USE_SIGALTSTACK
509 stack_t disable = {0};
510 disable.ss_flags = SS_DISABLE;
511 sigaltstack(&disable, NULL);
512#endif
513 native_thread_destroy(nt);
514}
515
516#if defined HAVE_PTHREAD_GETATTR_NP || defined HAVE_PTHREAD_ATTR_GET_NP
517#define STACKADDR_AVAILABLE 1
518# if defined(__linux__) && !defined __GLIBC__
519 /* Musl libc defines `pthread_getattr_np` but it can not get the
520 * correct value of the main thread. */
521# define MAINSTACKADDR_AVAILABLE 0
522# endif
523#elif defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP
524#define STACKADDR_AVAILABLE 1
525#undef MAINSTACKADDR_AVAILABLE
526#define MAINSTACKADDR_AVAILABLE 1
527void *pthread_get_stackaddr_np(pthread_t);
528size_t pthread_get_stacksize_np(pthread_t);
529#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
530#define STACKADDR_AVAILABLE 1
531#elif defined HAVE_PTHREAD_GETTHRDS_NP
532#define STACKADDR_AVAILABLE 1
533#elif defined __HAIKU__
534#define STACKADDR_AVAILABLE 1
535#endif
536
537#ifndef MAINSTACKADDR_AVAILABLE
538# ifdef STACKADDR_AVAILABLE
539# define MAINSTACKADDR_AVAILABLE 1
540# else
541# define MAINSTACKADDR_AVAILABLE 0
542# endif
543#endif
544#if MAINSTACKADDR_AVAILABLE && !defined(get_main_stack)
545# define get_main_stack(addr, size) get_stack(addr, size)
546#endif
547
548#ifdef STACKADDR_AVAILABLE
549/*
550 * Get the initial address and size of current thread's stack
551 */
552static int
553get_stack(void **addr, size_t *size)
554{
555#define CHECK_ERR(expr) \
556 {int err = (expr); if (err) return err;}
557#ifdef HAVE_PTHREAD_GETATTR_NP /* Linux */
558 pthread_attr_t attr;
559 size_t guard = 0;
560 STACK_GROW_DIR_DETECTION;
561 CHECK_ERR(pthread_getattr_np(pthread_self(), &attr));
562# ifdef HAVE_PTHREAD_ATTR_GETSTACK
563 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
564 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
565# else
566 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
567 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
568# endif
569# ifdef HAVE_PTHREAD_ATTR_GETGUARDSIZE
570 CHECK_ERR(pthread_attr_getguardsize(&attr, &guard));
571# else
572 guard = RB_THREAD_PAGE_SIZE;
573# endif
574 *size -= guard;
575 pthread_attr_destroy(&attr);
576#elif defined HAVE_PTHREAD_ATTR_GET_NP /* FreeBSD, DragonFly BSD, NetBSD */
577 pthread_attr_t attr;
578 CHECK_ERR(pthread_attr_init(&attr));
579 CHECK_ERR(pthread_attr_get_np(pthread_self(), &attr));
580# ifdef HAVE_PTHREAD_ATTR_GETSTACK
581 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
582# else
583 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
584 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
585# endif
586 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
587 pthread_attr_destroy(&attr);
588#elif (defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP) /* MacOS X */
589 pthread_t th = pthread_self();
590 *addr = pthread_get_stackaddr_np(th);
591 *size = pthread_get_stacksize_np(th);
592#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
593 stack_t stk;
594# if defined HAVE_THR_STKSEGMENT /* Solaris */
595 CHECK_ERR(thr_stksegment(&stk));
596# else /* OpenBSD */
597 CHECK_ERR(pthread_stackseg_np(pthread_self(), &stk));
598# endif
599 *addr = stk.ss_sp;
600 *size = stk.ss_size;
601#elif defined HAVE_PTHREAD_GETTHRDS_NP /* AIX */
602 pthread_t th = pthread_self();
603 struct __pthrdsinfo thinfo;
604 char reg[256];
605 int regsiz=sizeof(reg);
606 CHECK_ERR(pthread_getthrds_np(&th, PTHRDSINFO_QUERY_ALL,
607 &thinfo, sizeof(thinfo),
608 &reg, &regsiz));
609 *addr = thinfo.__pi_stackaddr;
610 /* Must not use thinfo.__pi_stacksize for size.
611 It is around 3KB smaller than the correct size
612 calculated by thinfo.__pi_stackend - thinfo.__pi_stackaddr. */
613 *size = thinfo.__pi_stackend - thinfo.__pi_stackaddr;
614 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
615#elif defined __HAIKU__
616 thread_info info;
617 STACK_GROW_DIR_DETECTION;
618 CHECK_ERR(get_thread_info(find_thread(NULL), &info));
619 *addr = info.stack_base;
620 *size = (uintptr_t)info.stack_end - (uintptr_t)info.stack_base;
621 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
622#else
623#error STACKADDR_AVAILABLE is defined but not implemented.
624#endif
625 return 0;
626#undef CHECK_ERR
627}
628#endif
629
630// An id not recorded yet is all zero (ZALLOC or a static)
631static bool
632native_thread_id_set_p(rb_nativethread_id_t id)
633{
634#ifdef NON_SCALAR_THREAD_ID
635 static const rb_nativethread_id_t unset;
636 return memcmp(&id, &unset, sizeof(id)) != 0;
637#else
638 return id != 0;
639#endif
640}
641
642static struct {
643 rb_nativethread_id_t id;
644 size_t stack_maxsize;
645 VALUE *stack_start;
646} native_main_thread;
647
648// Whether the calling native thread may leave the shared pool. The process's
649// main pthread cannot: under RUBY_MN_THREADS=2 its loop runs on a stack only
650// it could free (thread_sched_main_to_shared).
651static bool
652native_thread_self_can_retire_p(void)
653{
654 return !pthread_equal(pthread_self(), native_main_thread.id);
655}
656
657#if defined(HAVE_WORKING_FORK)
658// The forking thread's pthread is the child's only one, hence its main one
659static void
660native_main_thread_atfork(void)
661{
662 native_main_thread.id = pthread_self();
663 // The stack recorded here is the parent's initial one; this thread's is
664 // another. Nothing reads it for a thread that is already running, and
665 // saying "unknown" beats saying the wrong bounds.
666 native_main_thread.stack_maxsize = 0;
667}
668#endif
669
670#ifdef STACK_END_ADDRESS
671extern void *STACK_END_ADDRESS;
672#endif
673
674static void
675native_thread_init_main_thread_stack(void *addr)
676{
677 native_main_thread.id = pthread_self();
678#ifdef RUBY_ASAN_ENABLED
679 addr = asan_get_real_stack_addr((void *)addr);
680#endif
681
682#if MAINSTACKADDR_AVAILABLE
683 if (native_main_thread.stack_maxsize) return;
684 {
685 void* stackaddr;
686 size_t size;
687 if (get_main_stack(&stackaddr, &size) == 0) {
688 native_main_thread.stack_maxsize = size;
689 native_main_thread.stack_start = stackaddr;
690 goto bound_check;
691 }
692 }
693#endif
694#ifdef STACK_END_ADDRESS
695 native_main_thread.stack_start = STACK_END_ADDRESS;
696#else
697 if (!native_main_thread.stack_start ||
698 STACK_UPPER((VALUE *)(void *)&addr,
699 native_main_thread.stack_start > (VALUE *)addr,
700 native_main_thread.stack_start < (VALUE *)addr)) {
701 native_main_thread.stack_start = (VALUE *)addr;
702 }
703#endif
704 {
705#if defined(HAVE_GETRLIMIT)
706#if defined(PTHREAD_STACK_DEFAULT)
707 size_t size = PTHREAD_STACK_DEFAULT;
708#else
709 size_t size = RUBY_VM_THREAD_VM_STACK_SIZE;
710#endif
711 size_t space;
712 struct rlimit rlim;
713 STACK_GROW_DIR_DETECTION;
714 if (getrlimit(RLIMIT_STACK, &rlim) == 0) {
715 size = (size_t)rlim.rlim_cur;
716 }
717 addr = native_main_thread.stack_start;
718 if (IS_STACK_DIR_UPPER()) {
719 space = ((size_t)((char *)addr + size) / RB_THREAD_PAGE_SIZE) * RB_THREAD_PAGE_SIZE - (size_t)addr;
720 }
721 else {
722 space = (size_t)addr - ((size_t)((char *)addr - size) / RB_THREAD_PAGE_SIZE + 1) * RB_THREAD_PAGE_SIZE;
723 }
724 native_main_thread.stack_maxsize = space;
725#endif
726 }
727
728#if MAINSTACKADDR_AVAILABLE
729 bound_check:
730#endif
731 /* If addr is out of range of main-thread stack range estimation, */
732 /* it should be on co-routine (alternative stack). [Feature #2294] */
733 {
734 void *start, *end;
735 STACK_GROW_DIR_DETECTION;
736
737 if (IS_STACK_DIR_UPPER()) {
738 start = native_main_thread.stack_start;
739 end = (char *)native_main_thread.stack_start + native_main_thread.stack_maxsize;
740 }
741 else {
742 start = (char *)native_main_thread.stack_start - native_main_thread.stack_maxsize;
743 end = native_main_thread.stack_start;
744 }
745
746 if ((void *)addr < start || (void *)addr > end) {
747 /* out of range */
748 native_main_thread.stack_start = (VALUE *)addr;
749 native_main_thread.stack_maxsize = 0; /* unknown */
750 }
751 }
752}
753
754#define CHECK_ERR(expr) \
755 {int err = (expr); if (err) {rb_bug_errno(#expr, err);}}
756
757static int
758native_thread_init_stack(rb_thread_t *th, void *local_in_parent_frame)
759{
760 rb_nativethread_id_t curr = pthread_self();
761#ifdef RUBY_ASAN_ENABLED
762 local_in_parent_frame = asan_get_real_stack_addr(local_in_parent_frame);
763 th->ec->machine.asan_fake_stack_handle = asan_get_thread_fake_stack_handle();
764#endif
765
766 if (!native_thread_id_set_p(native_main_thread.id)) {
767 /* This thread is the first thread, must be the main thread -
768 * configure the native_main_thread object */
769 native_thread_init_main_thread_stack(local_in_parent_frame);
770 }
771
772 if (th->sched.context != NULL) {
773 // an M:N thread runs on the pool stack native_thread_create_shared
774 // recorded, whichever native thread (the main one included) hosts it.
775 // Not by nt->dedicated: a RESUMED hook may have pinned the nt already.
776 }
777 else if (pthread_equal(curr, native_main_thread.id)) {
778 th->ec->machine.stack_start = native_main_thread.stack_start;
779 th->ec->machine.stack_maxsize = native_main_thread.stack_maxsize;
780 }
781 else {
782#ifdef STACKADDR_AVAILABLE
783 void *start;
784 size_t size;
785
786 if (get_stack(&start, &size) == 0) {
787 uintptr_t diff = (uintptr_t)start - (uintptr_t)local_in_parent_frame;
788 th->ec->machine.stack_start = local_in_parent_frame;
789 th->ec->machine.stack_maxsize = size - diff;
790 }
791#else
792 rb_raise(rb_eNotImpError, "ruby engine can initialize only in the main thread");
793#endif
794 }
795
796 return 0;
797}
798
799struct nt_param {
800 rb_vm_t *vm;
801 struct rb_native_thread *nt;
802};
803
804
805static int
806native_thread_create0(struct rb_native_thread *nt)
807{
808 int err = 0;
809 pthread_attr_t attr;
810
811 const size_t stack_size = nt->vm->default_params.thread_machine_stack_size;
812
813#ifdef USE_SIGALTSTACK
814 nt->altstack = rb_allocate_sigaltstack();
815#endif
816
817 CHECK_ERR(pthread_attr_init(&attr));
818
819# ifdef PTHREAD_STACK_MIN
820 RUBY_DEBUG_LOG("stack size: %lu", (unsigned long)stack_size);
821 CHECK_ERR(pthread_attr_setstacksize(&attr, stack_size));
822# endif
823
824# ifdef HAVE_PTHREAD_ATTR_SETINHERITSCHED
825 CHECK_ERR(pthread_attr_setinheritsched(&attr, PTHREAD_INHERIT_SCHED));
826# endif
827 CHECK_ERR(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED));
828
829 // pthread_create may store the id after the thread has started, and a
830 // shared nt may retire and free itself by then
831 rb_nativethread_id_t thread_id;
832#if USE_RUBY_DEBUG_LOG
833 const rb_atomic_t serial = nt->serial;
834#endif
835 err = pthread_create(&thread_id, &attr, nt_start, nt);
836
837 RUBY_DEBUG_LOG("nt:%d err:%d", (int)serial, err);
838
839 CHECK_ERR(pthread_attr_destroy(&attr));
840
841 return err;
842}
843
844static void
845native_thread_setup(struct rb_native_thread *nt)
846{
847 // init cond
848 rb_native_cond_initialize(&nt->readyq);
849 // also for the main thread's nt: a zero-filled mutex is not usable on macOS
850 rb_native_mutex_initialize(&nt->running_th_lock);
851}
852
853static void
854native_thread_setup_on_thread(struct rb_native_thread *nt)
855{
856 // init thread_id
857 nt->thread_id = pthread_self();
858
859 // init tid
860#ifdef RB_THREAD_T_HAS_NATIVE_ID
861 nt->tid = get_native_thread_id();
862#endif
863
864 // init signal handler
865 RB_ALTSTACK_INIT(nt->altstack, nt->altstack);
866}
867
868static struct rb_native_thread *
869native_thread_alloc(void)
870{
871 struct rb_native_thread *nt = ZALLOC(struct rb_native_thread);
872 native_thread_setup(nt);
873
874#if USE_MN_THREADS
875 nt->nt_context = ruby_xmalloc(sizeof(struct coroutine_context));
876#endif
877
878#if USE_RUBY_DEBUG_LOG
879 static rb_atomic_t nt_serial = 2;
880 nt->serial = RUBY_ATOMIC_FETCH_ADD(nt_serial, 1);
881#endif
882 return nt;
883}
884
885
886
887
888#if USE_NATIVE_THREAD_PRIORITY
889
890static void
891native_thread_apply_priority(rb_thread_t *th)
892{
893#if defined(_POSIX_PRIORITY_SCHEDULING) && (_POSIX_PRIORITY_SCHEDULING > 0)
894 struct sched_param sp;
895 int policy;
896 int priority = 0 - th->priority;
897 int max, min;
898 pthread_getschedparam(th->nt->thread_id, &policy, &sp);
899 max = sched_get_priority_max(policy);
900 min = sched_get_priority_min(policy);
901
902 if (min > priority) {
903 priority = min;
904 }
905 else if (max < priority) {
906 priority = max;
907 }
908
909 sp.sched_priority = priority;
910 pthread_setschedparam(th->nt->thread_id, policy, &sp);
911#else
912 /* not touched */
913#endif
914}
915
916#endif /* USE_NATIVE_THREAD_PRIORITY */
917
918static int
919native_fd_select(int n, rb_fdset_t *readfds, rb_fdset_t *writefds, rb_fdset_t *exceptfds, struct timeval *timeout, rb_thread_t *th)
920{
921 return rb_fd_select(n, readfds, writefds, exceptfds, timeout);
922}
923
924/*
925 * Send a signal to make the target thread return from a blocking syscall.
926 * Maybe any signal is ok, but we chose SIGVTALRM (see null_func).
927 */
928static void
929native_thread_interrupt(rb_thread_t *th)
930{
931 pthread_kill(th->nt->thread_id, SIGVTALRM);
932}
933
934static int
935native_thread_default_max_cpu(void)
936{
937#if defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN)
938 long nprocessors = sysconf(_SC_NPROCESSORS_ONLN);
939 return (nprocessors > 0) ? (int)nprocessors : 8;
940#else
941 return 8;
942#endif
943}
944
945
946#define TT_DEBUG 0
947#define WRITE_CONST(fd, str) (void)(write((fd),(str),sizeof(str)-1)<0)
948
949void
950rb_thread_wakeup_timer_thread(int sig)
951{
952 // This function can be called from signal handlers so that
953 // pthread_mutex_lock() should not be used.
954
955 // wakeup timer thread
956 timer_thread_wakeup_force();
957
958 // interrupt main thread if main thread is available
959 if (RUBY_ATOMIC_LOAD(system_working)) {
960 rb_vm_t *vm = GET_VM();
961 rb_thread_t *main_th = vm->ractor.main_thread;
962
963 if (main_th) {
964 volatile rb_execution_context_t *main_th_ec = ACCESS_ONCE(rb_execution_context_t *, main_th->ec);
965
966 if (main_th_ec) {
967 RUBY_VM_SET_TRAP_INTERRUPT(main_th_ec);
968
969 if (vm->ubf_async_safe && main_th->unblock.func) {
970 (main_th->unblock.func)(main_th->unblock.arg);
971 }
972 }
973 }
974 }
975}
976
977#define CLOSE_INVALIDATE_PAIR(expr) \
978 close_invalidate_pair(expr,"close_invalidate: "#expr)
979static void
980close_invalidate(int *fdp, const char *msg)
981{
982 int fd = *fdp;
983
984 *fdp = -1;
985 if (close(fd) < 0) {
986 async_bug_fd(msg, errno, fd);
987 }
988}
989
990static void
991close_invalidate_pair(int fds[2], const char *msg)
992{
993 if (USE_EVENTFD && fds[0] == fds[1]) {
994 fds[1] = -1; // disable write port first
995 close_invalidate(&fds[0], msg);
996 }
997 else {
998 close_invalidate(&fds[1], msg);
999 close_invalidate(&fds[0], msg);
1000 }
1001}
1002
1003static void
1004set_nonblock(int fd)
1005{
1006 int oflags;
1007 int err;
1008
1009 oflags = fcntl(fd, F_GETFL);
1010 if (oflags == -1)
1011 rb_sys_fail(0);
1012 oflags |= O_NONBLOCK;
1013 err = fcntl(fd, F_SETFL, oflags);
1014 if (err == -1)
1015 rb_sys_fail(0);
1016}
1017
1018/* communication pipe with timer thread and signal handler */
1019static void
1020setup_communication_pipe_internal(int pipes[2])
1021{
1022 int err;
1023
1024 if (pipes[0] > 0 || pipes[1] > 0) {
1025 VM_ASSERT(pipes[0] > 0);
1026 VM_ASSERT(pipes[1] > 0);
1027 return;
1028 }
1029
1030 /*
1031 * Don't bother with eventfd on ancient Linux 2.6.22..2.6.26 which were
1032 * missing EFD_* flags, they can fall back to pipe
1033 */
1034#if USE_EVENTFD && defined(EFD_NONBLOCK) && defined(EFD_CLOEXEC)
1035 pipes[0] = pipes[1] = eventfd(0, EFD_NONBLOCK|EFD_CLOEXEC);
1036
1037 if (pipes[0] >= 0) {
1038 rb_update_max_fd(pipes[0]);
1039 return;
1040 }
1041#endif
1042
1043 err = rb_cloexec_pipe(pipes);
1044 if (err != 0) {
1045 rb_bug("can not create communication pipe");
1046 }
1047 rb_update_max_fd(pipes[0]);
1048 rb_update_max_fd(pipes[1]);
1049 set_nonblock(pipes[0]);
1050 set_nonblock(pipes[1]);
1051}
1052
1053#if !defined(SET_CURRENT_THREAD_NAME) && defined(__linux__) && defined(PR_SET_NAME)
1054# define SET_CURRENT_THREAD_NAME(name) prctl(PR_SET_NAME, name)
1055#endif
1056
1057enum {
1058 THREAD_NAME_MAX =
1059#if defined(__linux__)
1060 16
1061#elif defined(__APPLE__)
1062/* Undocumented, and main thread seems unlimited */
1063 64
1064#else
1065 16
1066#endif
1067};
1068
1069static VALUE threadptr_invoke_proc_location(rb_thread_t *th);
1070
1071static void
1072native_set_thread_name(rb_thread_t *th)
1073{
1074#ifdef SET_CURRENT_THREAD_NAME
1075 VALUE loc;
1076
1077 // An M:N thread does not own the native thread it runs on: naming it here
1078 // would name whichever nt started it (under RUBY_MN_THREADS=2 that can be
1079 // the process's main one, whose name is the process's). Thread#name= is
1080 // skipped for the same reason (rb_thread_setname).
1081 if (!th->has_dedicated_nt) return;
1082
1083 if (!NIL_P(loc = th->name)) {
1084 SET_CURRENT_THREAD_NAME(RSTRING_PTR(loc));
1085 }
1086 else if ((loc = threadptr_invoke_proc_location(th)) != Qnil) {
1087 char *name, *p;
1088 char buf[THREAD_NAME_MAX];
1089 size_t len;
1090 int n;
1091
1092 name = RSTRING_PTR(RARRAY_AREF(loc, 0));
1093 p = strrchr(name, '/'); /* show only the basename of the path. */
1094 if (p && p[1])
1095 name = p + 1;
1096
1097 n = snprintf(buf, sizeof(buf), "%s:%d", name, NUM2INT(RARRAY_AREF(loc, 1)));
1098 RB_GC_GUARD(loc);
1099
1100 len = (size_t)n;
1101 if (len >= sizeof(buf)) {
1102 buf[sizeof(buf)-2] = '*';
1103 buf[sizeof(buf)-1] = '\0';
1104 }
1105 SET_CURRENT_THREAD_NAME(buf);
1106 }
1107#endif
1108}
1109
1110static void
1111native_set_another_thread_name(rb_nativethread_id_t thread_id, VALUE name)
1112{
1113#if defined SET_ANOTHER_THREAD_NAME || defined SET_CURRENT_THREAD_NAME
1114 char buf[THREAD_NAME_MAX];
1115 const char *s = "";
1116 // not recorded until the thread starts; thread_do_start names it then
1117 if (!native_thread_id_set_p(thread_id)) return;
1118# if !defined SET_ANOTHER_THREAD_NAME
1119 if (!pthread_equal(pthread_self(), thread_id)) return;
1120# endif
1121 if (!NIL_P(name)) {
1122 rb_len_t n;
1123 RSTRING_GETMEM(name, s, n);
1124 if (n >= (int)sizeof(buf)) {
1125 memcpy(buf, s, sizeof(buf)-1);
1126 buf[sizeof(buf)-1] = '\0';
1127 s = buf;
1128 }
1129 }
1130# if defined SET_ANOTHER_THREAD_NAME
1131 SET_ANOTHER_THREAD_NAME(thread_id, s);
1132# elif defined SET_CURRENT_THREAD_NAME
1133 SET_CURRENT_THREAD_NAME(s);
1134# endif
1135#endif
1136}
1137
1138#if defined(RB_THREAD_T_HAS_NATIVE_ID) || defined(__APPLE__)
1139static VALUE
1140native_thread_native_thread_id(rb_thread_t *target_th)
1141{
1142 if (!target_th->nt) return Qnil;
1143
1144#ifdef RB_THREAD_T_HAS_NATIVE_ID
1145 int tid = target_th->nt->tid;
1146 if (tid == 0) return Qnil;
1147 return INT2FIX(tid);
1148#elif defined(__APPLE__)
1149 uint64_t tid;
1150 pthread_t thread_id = ACCESS_ONCE(pthread_t, target_th->nt->thread_id);
1151 if (!thread_id) return Qnil;
1152 int e = pthread_threadid_np(thread_id, &tid);
1153 if (e != 0) rb_syserr_fail(e, "pthread_threadid_np");
1154 return ULL2NUM((unsigned long long)tid);
1155#endif
1156}
1157# define USE_NATIVE_THREAD_NATIVE_THREAD_ID 1
1158#else
1159# define USE_NATIVE_THREAD_NATIVE_THREAD_ID 0
1160#endif
1161
1162static struct {
1163 rb_serial_t created_fork_gen;
1164 pthread_t pthread_id;
1165
1166 int comm_fds[2]; // r, w
1167
1168#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
1169 int event_fd; // kernel event queue fd (epoll/kqueue)
1170#endif
1171#if HAVE_SYS_EPOLL_H && USE_MN_THREADS
1172#define EPOLL_EVENTS_MAX 0x10
1173 struct epoll_event finished_events[EPOLL_EVENTS_MAX];
1174#elif HAVE_SYS_EVENT_H && USE_MN_THREADS
1175#define KQUEUE_EVENTS_MAX 0x10
1176 struct kevent finished_events[KQUEUE_EVENTS_MAX];
1177#endif
1178
1179#if USE_MN_THREADS
1180 /* Timed waiters, bucketed by deadline into a hierarchical timer wheel;
1181 * untimed (fd-only) waiters keep a plain list. Both under waiting_lock.
1182 * The wheel operations all live in thread_sched_mn.c (timer_wheel_*). */
1183#define TIMER_WHEEL_LEVELS 4
1184#define TIMER_WHEEL_SLOT_BITS 6
1185#define TIMER_WHEEL_SLOTS (1 << TIMER_WHEEL_SLOT_BITS)
1186 struct timer_wheel_level {
1187 uint64_t occupied; // bit s: slots[s] is non-empty
1188 struct ccan_list_head slots[TIMER_WHEEL_SLOTS];
1189 } wheel[TIMER_WHEEL_LEVELS];
1190 uint64_t wheel_cursor_tick; // slots for ticks <= this are drained
1191 rb_hrtime_t next_expiry; // never later than the earliest deadline
1192 pthread_mutex_t waiting_lock; // the wheel and the flags of fd-less timed waits
1193
1194 /* The fd map entries and the flags of io waits are guarded per fd by a
1195 * shard lock, so unrelated fds register and wake in parallel. Whoever
1196 * clears an io wait's flags under its shard owns the wakeup. Lock order:
1197 * shard -> waiting_lock -> wake_pending_lock, never the other way. */
1198#define IO_WAIT_SHARDS 16
1199 pthread_mutex_t fd_shard_locks[IO_WAIT_SHARDS];
1200
1201 // signaled when wake_pending clears on a thread; see timer_thread_wake_fence
1202 pthread_mutex_t wake_pending_lock;
1203 rb_nativethread_cond_t wake_pending_cond;
1204#endif
1205
1206#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
1207 // fd -> struct rb_fd_waiters, in chunks so entries never move. The chunk
1208 // table installs slots by CAS (chunks span shards); entry state is guarded
1209 // by the fd's shard lock.
1210#define FDMAP_MAX_CHUNKS 1024 // fds up to FDMAP_MAX_CHUNKS * FDMAP_CHUNK_SIZE
1211 struct rb_fd_waiters *fdmap_chunks[FDMAP_MAX_CHUNKS];
1212#endif
1213} timer_th = {
1214 .created_fork_gen = 0,
1215};
1216
1217#define TIMER_THREAD_CREATED_P() (timer_th.created_fork_gen == current_fork_gen)
1218
1219static void timer_thread_check_timeslice(rb_vm_t *vm);
1220static bool timeslice_scan(rb_vm_t *vm, bool interrupt);
1221static int timer_thread_set_timeout(rb_vm_t *vm);
1222
1223#include "thread_sched_mn.c"
1224
1225/* only use signal-safe system calls here */
1226static void
1227signal_communication_pipe(int fd)
1228{
1229#if USE_EVENTFD
1230 const uint64_t buff = 1;
1231#else
1232 const char buff = '!';
1233#endif
1234 ssize_t result;
1235
1236 /* already opened */
1237 if (fd >= 0) {
1238 retry:
1239 if ((result = write(fd, &buff, sizeof(buff))) <= 0) {
1240 int e = errno;
1241 switch (e) {
1242 case EINTR: goto retry;
1243 case EAGAIN:
1244#if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN
1245 case EWOULDBLOCK:
1246#endif
1247 break;
1248 default:
1249 async_bug_fd("rb_thread_wakeup_timer_thread: write", e, fd);
1250 }
1251 }
1252 if (TT_DEBUG) WRITE_CONST(2, "rb_thread_wakeup_timer_thread: write\n");
1253 }
1254 else {
1255 // ignore wakeup
1256 }
1257}
1258
1259static void
1260timer_thread_wakeup_force(void)
1261{
1262 // should not use RUBY_DEBUG_LOG() because it can be called within signal handlers.
1263 signal_communication_pipe(timer_th.comm_fds[1]);
1264}
1265
1266
1267static void
1268rb_thread_create_timer_thread(void)
1269{
1270 rb_serial_t created_fork_gen = timer_th.created_fork_gen;
1271
1272 RUBY_DEBUG_LOG("fork_gen create:%d current:%d", (int)created_fork_gen, (int)current_fork_gen);
1273
1274 timer_th.created_fork_gen = current_fork_gen;
1275
1276 if (created_fork_gen != current_fork_gen) {
1277 if (created_fork_gen != 0) {
1278 RUBY_DEBUG_LOG("forked child process");
1279
1280 CLOSE_INVALIDATE_PAIR(timer_th.comm_fds);
1281#if USE_MN_THREADS
1282 // The parent's waiters do not exist in the child, and the armings
1283 // belong to the closed event backend: a stale entry would satisfy
1284 // fd_waiters_arm's want == armed_flags check and never arm the new
1285 // backend (a lost wake the old dynamic map also had).
1286 for (unsigned int ci = 0; ci < FDMAP_MAX_CHUNKS; ci++) {
1287 struct rb_fd_waiters *chunk = timer_th.fdmap_chunks[ci];
1288 if (chunk == NULL) continue;
1289 for (unsigned int i = 0; i < FDMAP_CHUNK_SIZE; i++) {
1290 ccan_list_head_init(&chunk[i].waiters);
1291 chunk[i].armed_flags = 0;
1292 chunk[i].generation++;
1293 }
1294 }
1295#endif
1296#if HAVE_SYS_EPOLL_H && USE_MN_THREADS
1297 close_invalidate(&timer_th.event_fd, "close event_fd");
1298#elif HAVE_SYS_EVENT_H && USE_MN_THREADS
1299 // A kqueue is not inherited across fork: the number names a closed
1300 // fd in the child, and closing it could hit a reused one.
1301 timer_th.event_fd = -1;
1302#endif
1303 // No mutex_destroy for waiting_lock: glibc returns EBUSY (and
1304 // rb_native_mutex_destroy rb_bugs) for a mutex another ractor's
1305 // M:N thread held at the fork moment. The initialize below
1306 // starts over.
1307 }
1308
1309#if USE_MN_THREADS
1310 for (int lvl = 0; lvl < TIMER_WHEEL_LEVELS; lvl++) {
1311 timer_th.wheel[lvl].occupied = 0;
1312 for (int slot = 0; slot < TIMER_WHEEL_SLOTS; slot++) {
1313 ccan_list_head_init(&timer_th.wheel[lvl].slots[slot]);
1314 }
1315 }
1316 timer_th.wheel_cursor_tick = timer_wheel_tick(rb_hrtime_now());
1317 timer_th.next_expiry = TIMER_WHEEL_NO_EXPIRY;
1318 rb_native_mutex_initialize(&timer_th.waiting_lock);
1319 for (int i = 0; i < IO_WAIT_SHARDS; i++) {
1320 rb_native_mutex_initialize(&timer_th.fd_shard_locks[i]);
1321 }
1322 rb_native_mutex_initialize(&timer_th.wake_pending_lock);
1323 rb_native_cond_initialize(&timer_th.wake_pending_cond);
1324#endif
1325
1326 // open communication channel
1327 setup_communication_pipe_internal(timer_th.comm_fds);
1328
1329 // open event fd
1330 timer_thread_setup_mn();
1331 }
1332
1333 int err = pthread_create(&timer_th.pthread_id, NULL, timer_thread_func, GET_VM());
1334 if (err != 0) {
1335 // The timer thread delivers signals and drives the M:N scheduler;
1336 // running without one only defers the failure to stranger places.
1337 rb_bug_errno("pthread_create (timer thread)", err);
1338 }
1339}
1340
1341static int
1342native_stop_timer_thread(void)
1343{
1344 RUBY_ATOMIC_SET(system_working, 0);
1345
1346 RUBY_DEBUG_LOG("wakeup send %d", timer_th.comm_fds[1]);
1347 timer_thread_wakeup_force();
1348 RUBY_DEBUG_LOG("wakeup sent");
1349 pthread_join(timer_th.pthread_id, NULL);
1350
1351 if (TT_DEBUG) fprintf(stderr, "stop timer thread\n");
1352
1353 return 1;
1354}
1355
1356static void
1357native_reset_timer_thread(void)
1358{
1359 //
1360}
1361
1362#ifdef HAVE_SIGALTSTACK
1363int
1364ruby_stack_overflowed_p(const rb_thread_t *th, const void *addr)
1365{
1366 void *base;
1367 size_t size;
1368 const size_t water_mark = RB_THREAD_PAGE_SIZE;
1369 STACK_GROW_DIR_DETECTION;
1370
1371 if (th) {
1372 size = th->ec->machine.stack_maxsize;
1373 base = (char *)th->ec->machine.stack_start - STACK_DIR_UPPER(0, size);
1374 }
1375#ifdef STACKADDR_AVAILABLE
1376 else if (get_stack(&base, &size) == 0) {
1377# ifdef __APPLE__
1378 // th is NULL in this branch; ask about the calling thread itself.
1379 if (pthread_equal(pthread_self(), native_main_thread.id)) {
1380 struct rlimit rlim;
1381 if (getrlimit(RLIMIT_STACK, &rlim) == 0 && rlim.rlim_cur > size) {
1382 size = (size_t)rlim.rlim_cur;
1383 }
1384 }
1385# endif
1386 base = (char *)base + STACK_DIR_UPPER(+size, -size);
1387 }
1388#endif
1389 else {
1390 return 0;
1391 }
1392
1393 if (size > water_mark) size = water_mark;
1394 if (IS_STACK_DIR_UPPER()) {
1395 if (size > ~(size_t)base+1) size = ~(size_t)base+1;
1396 if (addr > base && addr <= (void *)((char *)base + size)) return 1;
1397 }
1398 else {
1399 if (size > (size_t)base) size = (size_t)base;
1400 if (addr > (void *)((char *)base - size) && addr <= base) return 1;
1401 }
1402 return 0;
1403}
1404#endif
1405
1406int
1407rb_reserved_fd_p(int fd)
1408{
1409 /* no false-positive if out-of-FD at startup */
1410 if (fd < 0) return 0;
1411
1412 if (fd == timer_th.comm_fds[0] ||
1413 fd == timer_th.comm_fds[1]
1414#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
1415 || fd == timer_th.event_fd
1416#endif
1417 ) {
1418 goto check_fork_gen;
1419 }
1420 return 0;
1421
1422 check_fork_gen:
1423 if (timer_th.created_fork_gen == current_fork_gen) {
1424 /* async-signal-safe */
1425 return 1;
1426 }
1427 else {
1428 return 0;
1429 }
1430}
1431
1432rb_nativethread_id_t
1434{
1435 return pthread_self();
1436}
1437
1438#if defined(USE_POLL) && !defined(HAVE_PPOLL)
1439/* TODO: don't ignore sigmask */
1440static int
1441ruby_ppoll(struct pollfd *fds, nfds_t nfds,
1442 const struct timespec *ts, const sigset_t *sigmask)
1443{
1444 int timeout_ms;
1445
1446 if (ts) {
1447 int tmp, tmp2;
1448
1449 if (ts->tv_sec > INT_MAX/1000)
1450 timeout_ms = INT_MAX;
1451 else {
1452 tmp = (int)(ts->tv_sec * 1000);
1453 /* round up 1ns to 1ms to avoid excessive wakeups for <1ms sleep */
1454 tmp2 = (int)((ts->tv_nsec + 999999L) / (1000L * 1000L));
1455 if (INT_MAX - tmp < tmp2)
1456 timeout_ms = INT_MAX;
1457 else
1458 timeout_ms = (int)(tmp + tmp2);
1459 }
1460 }
1461 else
1462 timeout_ms = -1;
1463
1464 return poll(fds, nfds, timeout_ms);
1465}
1466# define ppoll(fds,nfds,ts,sigmask) ruby_ppoll((fds),(nfds),(ts),(sigmask))
1467#endif
1468
1469
1470// fork read-write lock (only for pthread)
1471static pthread_rwlock_t rb_thread_fork_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
1472
1473#ifndef NON_SCALAR_THREAD_ID
1474// the registered thread waiting to be joined, or NULL; no pthread_t is 0
1475static void *last_exiting_thread;
1476STATIC_ASSERT(pthread_t_fits_in_pointer, sizeof(pthread_t) <= sizeof(void *));
1477#endif
1478
1479void
1480rb_thread_release_fork_lock(void)
1481{
1482 int r;
1483 if ((r = pthread_rwlock_unlock(&rb_thread_fork_rw_lock))) {
1484 rb_bug_errno("pthread_rwlock_unlock", r);
1485 }
1486}
1487
1488void
1489rb_thread_reset_fork_lock(void)
1490{
1491 // On macOS the child's copy still counts a thread waiting for this lock
1492 // at fork: unlocking it leaves kernel state that later waiters hang on,
1493 // and destroying or initializing it fails with EBUSY.
1494 rb_thread_fork_rw_lock = (pthread_rwlock_t)PTHREAD_RWLOCK_INITIALIZER;
1495}
1496
1497void *
1498rb_thread_prevent_fork(void *(*func)(void *), void *data)
1499{
1500 int r;
1501 if ((r = pthread_rwlock_rdlock(&rb_thread_fork_rw_lock))) {
1502 rb_bug_errno("pthread_rwlock_rdlock", r);
1503 }
1504 void *result = func(data);
1505 rb_thread_release_fork_lock();
1506 return result;
1507}
1508
1509#ifndef NON_SCALAR_THREAD_ID
1510static void
1511thread_register_exiting(void *thread)
1512{
1513 void *previous_thread = RUBY_ATOMIC_PTR_EXCHANGE(last_exiting_thread, thread);
1514 if (previous_thread) {
1515 int r;
1516 if ((r = pthread_join((pthread_t)(uintptr_t)previous_thread, NULL))) {
1517 rb_bug_errno("pthread_join", r);
1518 }
1519 }
1520}
1521#endif
1522
1523// Called within rb_thread_prevent_fork() by a joinable thread just before it
1524// exits. The thread it replaces in the slot has usually exited by then.
1525void
1526rb_thread_register_exiting(void)
1527{
1528#ifdef NON_SCALAR_THREAD_ID
1529 // last_exiting_thread cannot hold a non-scalar pthread_t; nobody joins it
1530 pthread_detach(pthread_self());
1531#else
1532 thread_register_exiting((void *)(uintptr_t)pthread_self());
1533#endif
1534}
1535
1536void
1537rb_thread_acquire_fork_lock(void)
1538{
1539 int r;
1540 if ((r = pthread_rwlock_wrlock(&rb_thread_fork_rw_lock))) {
1541 rb_bug_errno("pthread_rwlock_wrlock", r);
1542 }
1543
1544 // A registered thread may still be exiting, and glibc's thread exit takes
1545 // the resolver configuration lock, which fork does not reset. None can
1546 // register while this lock is held.
1547#ifndef NON_SCALAR_THREAD_ID
1548 thread_register_exiting(NULL);
1549#endif
1550}
1551
1552// thread internal event hooks (only for pthread)
1553
1554struct rb_internal_thread_event_hook {
1555 rb_internal_thread_event_callback callback;
1556 rb_event_flag_t event;
1557 void *user_data;
1558
1559 struct rb_internal_thread_event_hook *next;
1560};
1561
1562static pthread_rwlock_t rb_internal_thread_event_hooks_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
1563
1564/* For the GC: whether any thread-event hook is registered right now. The rwlock
1565 * makes the answer happen-after any completed registration; a hook registered
1566 * after this read gets no event from the asking thread (rb_thread_execute_hooks
1567 * skips a swept thread), so it never sees the objects the caller may collect. */
1568bool
1569rb_thread_event_hooks_registered_p(void)
1570{
1571 int r;
1572 if ((r = pthread_rwlock_rdlock(&rb_internal_thread_event_hooks_rw_lock))) {
1573 rb_bug_errno("pthread_rwlock_rdlock", r);
1574 }
1575 const bool registered = (rb_internal_thread_event_hooks != NULL);
1576 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
1577 rb_bug_errno("pthread_rwlock_unlock", r);
1578 }
1579 return registered;
1580}
1581
1582#if defined(HAVE_WORKING_FORK)
1583static void
1584rb_internal_thread_event_hooks_rw_lock_atfork(void)
1585{
1586 // After fork(), this rwlock may have been held by a now-dead thread.
1587 //
1588 // pthread_rwlock_destroy() on a held lock is undefined behavior, and
1589 // pthread_rwlock_init() on an already-initialized lock is also undefined
1590 // behavior
1591 //
1592 // Direct assignment of PTHREAD_RWLOCK_INITIALIZER is safe and portable.
1593 rb_internal_thread_event_hooks_rw_lock =
1594 (pthread_rwlock_t)PTHREAD_RWLOCK_INITIALIZER;
1595}
1596#endif
1597
1598rb_internal_thread_event_hook_t *
1599rb_internal_thread_add_event_hook(rb_internal_thread_event_callback callback, rb_event_flag_t internal_event, void *user_data)
1600{
1601 rb_internal_thread_event_hook_t *hook = ALLOC_N(rb_internal_thread_event_hook_t, 1);
1602 hook->callback = callback;
1603 hook->user_data = user_data;
1604 hook->event = internal_event;
1605
1606 int r;
1607 if ((r = pthread_rwlock_wrlock(&rb_internal_thread_event_hooks_rw_lock))) {
1608 rb_bug_errno("pthread_rwlock_wrlock", r);
1609 }
1610
1611 hook->next = rb_internal_thread_event_hooks;
1612 ATOMIC_PTR_EXCHANGE(rb_internal_thread_event_hooks, hook);
1613
1614 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
1615 rb_bug_errno("pthread_rwlock_unlock", r);
1616 }
1617 return hook;
1618}
1619
1620bool
1621rb_internal_thread_remove_event_hook(rb_internal_thread_event_hook_t * hook)
1622{
1623 int r;
1624 if ((r = pthread_rwlock_wrlock(&rb_internal_thread_event_hooks_rw_lock))) {
1625 rb_bug_errno("pthread_rwlock_wrlock", r);
1626 }
1627
1628 bool success = FALSE;
1629
1630 if (rb_internal_thread_event_hooks == hook) {
1631 ATOMIC_PTR_EXCHANGE(rb_internal_thread_event_hooks, hook->next);
1632 success = TRUE;
1633 }
1634 else {
1635 rb_internal_thread_event_hook_t *h = rb_internal_thread_event_hooks;
1636
1637 do {
1638 if (h->next == hook) {
1639 h->next = hook->next;
1640 success = TRUE;
1641 break;
1642 }
1643 } while ((h = h->next));
1644 }
1645
1646 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
1647 rb_bug_errno("pthread_rwlock_unlock", r);
1648 }
1649
1650 if (success) {
1651 SIZED_FREE(hook);
1652 }
1653 return success;
1654}
1655
1656static void
1657rb_thread_execute_hooks(rb_event_flag_t event, rb_thread_t *th)
1658{
1659 int r;
1660
1661 /* th->self == 0: the dying thread's final collection swept its Thread wrapper, so
1662 * no hook existed then; one registered since has no ordering claim to this event. */
1663 if (th->self == 0) return;
1664 if ((r = pthread_rwlock_rdlock(&rb_internal_thread_event_hooks_rw_lock))) {
1665 rb_bug_errno("pthread_rwlock_rdlock", r);
1666 }
1667
1668 if (rb_internal_thread_event_hooks) {
1669 rb_internal_thread_event_hook_t *h = rb_internal_thread_event_hooks;
1670 do {
1671 if (h->event & event) {
1672 rb_internal_thread_event_data_t event_data = {
1673 .thread = th->self,
1674 };
1675 (*h->callback)(event, &event_data, h->user_data);
1676 }
1677 } while((h = h->next));
1678 }
1679 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
1680 rb_bug_errno("pthread_rwlock_unlock", r);
1681 }
1682}
1683
1684#endif /* THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION */
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
Definition atomic.h:69
#define RUBY_ATOMIC_PTR_EXCHANGE(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except it expects its arguments are void*.
Definition atomic.h:337
#define RUBY_ATOMIC_FETCH_ADD(var, val)
Atomically replaces the value pointed by var with the result of addition of val to the old value of v...
Definition atomic.h:118
#define RUBY_ATOMIC_LOAD(var)
Atomic load.
Definition atomic.h:175
#define RUBY_ATOMIC_SET(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except for the return type.
Definition atomic.h:185
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
#define ZALLOC
Old name of RB_ZALLOC.
Definition memory.h:405
#define ALLOC_N
Old name of RB_ALLOC_N.
Definition memory.h:402
#define ULL2NUM
Old name of RB_ULL2NUM.
Definition long_long.h:31
#define NUM2INT
Old name of RB_NUM2INT.
Definition int.h:44
#define Qnil
Old name of RUBY_Qnil.
#define NIL_P
Old name of RB_NIL_P.
VALUE rb_eNotImpError
NotImplementedError exception.
Definition error.c:1467
void rb_syserr_fail(int e, const char *mesg)
Raises appropriate exception that represents a C errno.
Definition error.c:4053
void rb_bug_errno(const char *mesg, int errno_arg)
This is a wrapper of rb_bug() which automatically constructs appropriate message from the passed errn...
Definition error.c:1177
int rb_cloexec_pipe(int fildes[2])
Opens a pipe with closing on exec.
Definition io.c:478
void rb_update_max_fd(int fd)
Informs the interpreter that the passed fd can be the max.
Definition io.c:274
int rb_reserved_fd_p(int fd)
Queries if the given FD is reserved or not.
void rb_timespec_now(struct timespec *ts)
Fills the current time into the given struct.
Definition time.c:2034
int len
Length of the buffer.
Definition io.h:8
#define RUBY_INTERNAL_THREAD_EVENT_RESUMED
Triggered when a thread successfully acquired the GVL.
Definition thread.h:249
rb_internal_thread_event_hook_t * rb_internal_thread_add_event_hook(rb_internal_thread_event_callback func, rb_event_flag_t events, void *data)
Registers a thread event hook function.
#define RUBY_INTERNAL_THREAD_EVENT_EXITED
Triggered when a thread exits.
Definition thread.h:263
#define RUBY_INTERNAL_THREAD_EVENT_SUSPENDED
Triggered when a thread released the GVL.
Definition thread.h:256
#define RUBY_INTERNAL_THREAD_EVENT_STARTED
Triggered when a new thread is started.
Definition thread.h:235
bool rb_internal_thread_remove_event_hook(rb_internal_thread_event_hook_t *hook)
Unregister the passed hook.
#define RUBY_INTERNAL_THREAD_EVENT_READY
Triggered when a thread attempt to acquire the GVL.
Definition thread.h:242
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:170
#define rb_fd_select
Waits for multiple file descriptors at once.
Definition posix.h:66
#define RARRAY_AREF(a, i)
References an object in an array.
Definition rarray.h:421
#define RSTRING_GETMEM(str, ptrvar, lenvar)
Convenient macro to obtain the contents and length at once.
Definition rstring.h:450
#define errno
Ractor-aware version of errno.
Definition ruby.h:388
The data structure which wraps the fd_set bitmap used by select(2).
Definition largesize.h:71
rb_nativethread_id_t rb_nativethread_self(void)
Queries the ID of the native thread that is calling this function.
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_cond_initialize(rb_nativethread_cond_t *cond)
Fills the passed condition variable with an initial value.
int rb_native_mutex_trylock(rb_nativethread_lock_t *lock)
Identical to rb_native_mutex_lock(), except it doesn't block in case rb_native_mutex_lock() would.
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
void rb_native_cond_destroy(rb_nativethread_cond_t *cond)
Destroys the passed condition variable.
void rb_native_cond_signal(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
Waits for the passed condition variable to be signalled.
void rb_native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, unsigned long msec)
Identical to rb_native_cond_wait(), except it additionally takes timeout in msec resolution.
long rb_len_t
A signed integer type for lengths and indices of objects such as String and Array.
Definition value.h:142
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40