12#ifdef THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION
14#include "internal/gc.h"
15#include "internal/sanitizers.h"
17#ifdef HAVE_SYS_RESOURCE_H
18#include <sys/resource.h>
20#ifdef HAVE_THR_STKSEGMENT
23#if defined(HAVE_FCNTL_H)
25#elif defined(HAVE_SYS_FCNTL_H)
28#ifdef HAVE_SYS_PRCTL_H
31#if defined(HAVE_SYS_TIME_H)
38#include <sys/syscall.h>
45#if defined(HAVE_SYS_EVENTFD_H) && defined(HAVE_EVENTFD)
46# define USE_EVENTFD (1)
47# include <sys/eventfd.h>
49# define USE_EVENTFD (0)
52#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && \
53 defined(CLOCK_REALTIME) && defined(CLOCK_MONOTONIC) && \
54 defined(HAVE_CLOCK_GETTIME)
55static pthread_condattr_t condattr_mono;
56static pthread_condattr_t *condattr_monotonic = &condattr_mono;
58static const void *
const condattr_monotonic = NULL;
64#define RB_NATIVE_COND_HRTIME_DEADLINE_P() (condattr_monotonic != NULL)
68#ifndef HAVE_SYS_EVENT_H
69#define HAVE_SYS_EVENT_H 0
72#ifndef HAVE_SYS_EPOLL_H
73#define HAVE_SYS_EPOLL_H 0
81 #if defined(__EMSCRIPTEN__) || defined(COROUTINE_PTHREAD_CONTEXT)
84 #define USE_MN_THREADS 0
85 #elif HAVE_SYS_EPOLL_H
86 #include <sys/epoll.h>
88 #define USE_MN_THREADS 1
91 #define USE_MN_THREADS 0
93 #elif HAVE_SYS_EVENT_H
94 #include <sys/event.h>
95 #define USE_MN_THREADS 1
97 #define USE_MN_THREADS 0
101#ifdef HAVE_SCHED_YIELD
102#define native_thread_yield() (void)sched_yield()
104#define native_thread_yield() ((void)0)
109#define NATIVE_MUTEX_LOCK_DEBUG 0
110#define NATIVE_MUTEX_LOCK_DEBUG_YIELD 0
113mutex_debug(
const char *msg,
void *lock)
115 if (NATIVE_MUTEX_LOCK_DEBUG) {
117 static pthread_mutex_t dbglock = PTHREAD_MUTEX_INITIALIZER;
119 if ((r = pthread_mutex_lock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
120 fprintf(stdout,
"%s: %p\n", msg, lock);
121 if ((r = pthread_mutex_unlock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
129#if NATIVE_MUTEX_LOCK_DEBUG_YIELD
130 native_thread_yield();
132 mutex_debug(
"lock", lock);
133 if ((r = pthread_mutex_lock(lock)) != 0) {
142 mutex_debug(
"unlock", lock);
143 if ((r = pthread_mutex_unlock(lock)) != 0) {
152 mutex_debug(
"trylock", lock);
153 if ((r = pthread_mutex_trylock(lock)) != 0) {
167 int r = pthread_mutex_init(lock, 0);
168 mutex_debug(
"init", lock);
177 int r = pthread_mutex_destroy(lock);
178 mutex_debug(
"destroy", lock);
187 int r = pthread_cond_init(cond, condattr_monotonic);
196 int r = pthread_cond_destroy(cond);
217 r = pthread_cond_signal(cond);
218 }
while (r == EAGAIN);
229 r = pthread_cond_broadcast(cond);
230 }
while (r == EAGAIN);
239 int r = pthread_cond_wait(cond, mutex);
246native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex,
const rb_hrtime_t *abs)
258 rb_hrtime2timespec(&ts, abs);
259 r = pthread_cond_timedwait(cond, mutex, &ts);
260 }
while (r == EINTR);
262 if (r != 0 && r != ETIMEDOUT) {
270native_cond_timeout(rb_nativethread_cond_t *cond,
const rb_hrtime_t rel)
272 if (condattr_monotonic) {
273 return rb_hrtime_add(rb_hrtime_now(), rel);
279 return rb_hrtime_add(rb_timespec2hrtime(&ts), rel);
286 rb_hrtime_t hrmsec = native_cond_timeout(cond, RB_HRTIME_PER_MSEC * msec);
287 native_cond_timedwait(cond, mutex, &hrmsec);
292static rb_internal_thread_event_hook_t *rb_internal_thread_event_hooks = NULL;
293static void rb_thread_execute_hooks(
rb_event_flag_t event, rb_thread_t *th);
314#define RB_INTERNAL_THREAD_HOOK(event, th) \
315 if (UNLIKELY(rb_internal_thread_event_hooks)) { \
316 fprintf(stderr, "[thread=%"PRIxVALUE"] %s in %s (%s:%d)\n", th->self, event_name(event), __func__, __FILE__, __LINE__); \
317 rb_thread_execute_hooks(event, th); \
320#define RB_INTERNAL_THREAD_HOOK(event, th) if (UNLIKELY(rb_internal_thread_event_hooks)) { rb_thread_execute_hooks(event, th); }
323static rb_serial_t current_fork_gen = 1;
325#if defined(SIGVTALRM) && !defined(__EMSCRIPTEN__)
326# define USE_UBF_LIST 1
330static void nt_machine_stack_atfork(
void);
338struct rb_thread_context {
339 struct coroutine_context co;
341 struct rb_native_thread *nt;
349#ifdef RB_THREAD_T_HAS_NATIVE_ID
351get_native_thread_id(
void)
354 return (
int)syscall(SYS_gettid);
355#elif defined(__FreeBSD__)
356 return pthread_getthreadid_np();
362#ifdef RB_THREAD_LOCAL_SPECIFIER
363static RB_THREAD_LOCAL_SPECIFIER rb_thread_t *ruby_native_thread;
365static pthread_key_t ruby_native_thread_key;
377ruby_thread_from_native(
void)
379#ifdef RB_THREAD_LOCAL_SPECIFIER
380 return ruby_native_thread;
382 return pthread_getspecific(ruby_native_thread_key);
387ruby_thread_set_native(rb_thread_t *th)
391 ccan_list_node_init(&th->sched.node.ubf);
398 rb_ractor_set_current_ec(th->ractor, th->ec);
400#ifdef RB_THREAD_LOCAL_SPECIFIER
401 ruby_native_thread = th;
404 return pthread_setspecific(ruby_native_thread_key, th) == 0;
412static size_t RB_THREAD_PAGE_SIZE;
415Init_native_thread(rb_thread_t *main_th)
418 RB_THREAD_PAGE_SIZE = sysconf(_SC_PAGESIZE);
420#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK)
421 if (condattr_monotonic) {
422 int r = pthread_condattr_init(condattr_monotonic);
424 r = pthread_condattr_setclock(condattr_monotonic, CLOCK_MONOTONIC);
426 if (r) condattr_monotonic = NULL;
430#ifndef RB_THREAD_LOCAL_SPECIFIER
431 if (pthread_key_create(&ruby_native_thread_key, 0) == EAGAIN) {
432 rb_bug(
"pthread_key_create failed (ruby_native_thread_key)");
434 if (pthread_key_create(&ruby_current_ec_key, 0) == EAGAIN) {
435 rb_bug(
"pthread_key_create failed (ruby_current_ec_key)");
438 ruby_posix_signal(SIGVTALRM, null_func);
441 rb_vm_t *vm = main_th->vm;
442 thread_sched_init_vm(vm);
445 main_th->nt->serial = 1;
449 ruby_thread_set_native(main_th);
450 native_thread_setup(main_th->nt);
451 native_thread_setup_on_thread(main_th->nt);
453 TH_SCHED(main_th)->running = main_th;
454 main_th->has_dedicated_nt = 1;
457 main_th->nt->dedicated = 1;
458 main_th->nt->running_thread = main_th;
459 main_th->nt->vm = vm;
461 thread_sched_setup_running_threads(TH_SCHED(main_th), main_th->ractor, vm, main_th, NULL);
464#if USE_RUBY_DEBUG_LOG
465 vm->ractor.sched.dnt_cnt = 1;
485 RB_ALTSTACK_FREE(nt->altstack);
486 SIZED_FREE(nt->nt_context);
498 native_thread_destroy_atfork(nt);
508#ifdef USE_SIGALTSTACK
509 stack_t disable = {0};
510 disable.ss_flags = SS_DISABLE;
511 sigaltstack(&disable, NULL);
513 native_thread_destroy(nt);
516#if defined HAVE_PTHREAD_GETATTR_NP || defined HAVE_PTHREAD_ATTR_GET_NP
517#define STACKADDR_AVAILABLE 1
518# if defined(__linux__) && !defined __GLIBC__
521# define MAINSTACKADDR_AVAILABLE 0
523#elif defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP
524#define STACKADDR_AVAILABLE 1
525#undef MAINSTACKADDR_AVAILABLE
526#define MAINSTACKADDR_AVAILABLE 1
527void *pthread_get_stackaddr_np(pthread_t);
528size_t pthread_get_stacksize_np(pthread_t);
529#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
530#define STACKADDR_AVAILABLE 1
531#elif defined HAVE_PTHREAD_GETTHRDS_NP
532#define STACKADDR_AVAILABLE 1
533#elif defined __HAIKU__
534#define STACKADDR_AVAILABLE 1
537#ifndef MAINSTACKADDR_AVAILABLE
538# ifdef STACKADDR_AVAILABLE
539# define MAINSTACKADDR_AVAILABLE 1
541# define MAINSTACKADDR_AVAILABLE 0
544#if MAINSTACKADDR_AVAILABLE && !defined(get_main_stack)
545# define get_main_stack(addr, size) get_stack(addr, size)
548#ifdef STACKADDR_AVAILABLE
553get_stack(
void **addr,
size_t *size)
555#define CHECK_ERR(expr) \
556 {int err = (expr); if (err) return err;}
557#ifdef HAVE_PTHREAD_GETATTR_NP
560 STACK_GROW_DIR_DETECTION;
561 CHECK_ERR(pthread_getattr_np(pthread_self(), &attr));
562# ifdef HAVE_PTHREAD_ATTR_GETSTACK
563 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
564 STACK_DIR_UPPER((
void)0, (
void)(*addr = (
char *)*addr + *size));
566 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
567 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
569# ifdef HAVE_PTHREAD_ATTR_GETGUARDSIZE
570 CHECK_ERR(pthread_attr_getguardsize(&attr, &guard));
572 guard = RB_THREAD_PAGE_SIZE;
575 pthread_attr_destroy(&attr);
576#elif defined HAVE_PTHREAD_ATTR_GET_NP
578 CHECK_ERR(pthread_attr_init(&attr));
579 CHECK_ERR(pthread_attr_get_np(pthread_self(), &attr));
580# ifdef HAVE_PTHREAD_ATTR_GETSTACK
581 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
583 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
584 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
586 STACK_DIR_UPPER((
void)0, (
void)(*addr = (
char *)*addr + *size));
587 pthread_attr_destroy(&attr);
588#elif (defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP)
589 pthread_t th = pthread_self();
590 *addr = pthread_get_stackaddr_np(th);
591 *size = pthread_get_stacksize_np(th);
592#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
594# if defined HAVE_THR_STKSEGMENT
595 CHECK_ERR(thr_stksegment(&stk));
597 CHECK_ERR(pthread_stackseg_np(pthread_self(), &stk));
601#elif defined HAVE_PTHREAD_GETTHRDS_NP
602 pthread_t th = pthread_self();
603 struct __pthrdsinfo thinfo;
605 int regsiz=
sizeof(reg);
606 CHECK_ERR(pthread_getthrds_np(&th, PTHRDSINFO_QUERY_ALL,
607 &thinfo,
sizeof(thinfo),
609 *addr = thinfo.__pi_stackaddr;
613 *size = thinfo.__pi_stackend - thinfo.__pi_stackaddr;
614 STACK_DIR_UPPER((
void)0, (
void)(*addr = (
char *)*addr + *size));
615#elif defined __HAIKU__
617 STACK_GROW_DIR_DETECTION;
618 CHECK_ERR(get_thread_info(find_thread(NULL), &info));
619 *addr = info.stack_base;
620 *size = (uintptr_t)info.stack_end - (uintptr_t)info.stack_base;
621 STACK_DIR_UPPER((
void)0, (
void)(*addr = (
char *)*addr + *size));
623#error STACKADDR_AVAILABLE is defined but not implemented.
632native_thread_id_set_p(rb_nativethread_id_t
id)
634#ifdef NON_SCALAR_THREAD_ID
635 static const rb_nativethread_id_t unset;
636 return memcmp(&
id, &unset,
sizeof(
id)) != 0;
643 rb_nativethread_id_t id;
644 size_t stack_maxsize;
652native_thread_self_can_retire_p(
void)
654 return !pthread_equal(pthread_self(), native_main_thread.id);
657#if defined(HAVE_WORKING_FORK)
660native_main_thread_atfork(
void)
662 native_main_thread.id = pthread_self();
666 native_main_thread.stack_maxsize = 0;
670#ifdef STACK_END_ADDRESS
671extern void *STACK_END_ADDRESS;
675native_thread_init_main_thread_stack(
void *addr)
677 native_main_thread.id = pthread_self();
678#ifdef RUBY_ASAN_ENABLED
679 addr = asan_get_real_stack_addr((
void *)addr);
682#if MAINSTACKADDR_AVAILABLE
683 if (native_main_thread.stack_maxsize)
return;
687 if (get_main_stack(&stackaddr, &size) == 0) {
688 native_main_thread.stack_maxsize = size;
689 native_main_thread.stack_start = stackaddr;
694#ifdef STACK_END_ADDRESS
695 native_main_thread.stack_start = STACK_END_ADDRESS;
697 if (!native_main_thread.stack_start ||
698 STACK_UPPER((
VALUE *)(
void *)&addr,
699 native_main_thread.stack_start > (
VALUE *)addr,
700 native_main_thread.stack_start < (
VALUE *)addr)) {
701 native_main_thread.stack_start = (
VALUE *)addr;
705#if defined(HAVE_GETRLIMIT)
706#if defined(PTHREAD_STACK_DEFAULT)
707 size_t size = PTHREAD_STACK_DEFAULT;
709 size_t size = RUBY_VM_THREAD_VM_STACK_SIZE;
713 STACK_GROW_DIR_DETECTION;
714 if (getrlimit(RLIMIT_STACK, &rlim) == 0) {
715 size = (size_t)rlim.rlim_cur;
717 addr = native_main_thread.stack_start;
718 if (IS_STACK_DIR_UPPER()) {
719 space = ((size_t)((
char *)addr + size) / RB_THREAD_PAGE_SIZE) * RB_THREAD_PAGE_SIZE - (size_t)addr;
722 space = (size_t)addr - ((
size_t)((
char *)addr - size) / RB_THREAD_PAGE_SIZE + 1) * RB_THREAD_PAGE_SIZE;
724 native_main_thread.stack_maxsize = space;
728#if MAINSTACKADDR_AVAILABLE
735 STACK_GROW_DIR_DETECTION;
737 if (IS_STACK_DIR_UPPER()) {
738 start = native_main_thread.stack_start;
739 end = (
char *)native_main_thread.stack_start + native_main_thread.stack_maxsize;
742 start = (
char *)native_main_thread.stack_start - native_main_thread.stack_maxsize;
743 end = native_main_thread.stack_start;
746 if ((
void *)addr < start || (
void *)addr > end) {
748 native_main_thread.stack_start = (
VALUE *)addr;
749 native_main_thread.stack_maxsize = 0;
754#define CHECK_ERR(expr) \
755 {int err = (expr); if (err) {rb_bug_errno(#expr, err);}}
758native_thread_init_stack(rb_thread_t *th,
void *local_in_parent_frame)
760 rb_nativethread_id_t curr = pthread_self();
761#ifdef RUBY_ASAN_ENABLED
762 local_in_parent_frame = asan_get_real_stack_addr(local_in_parent_frame);
763 th->ec->machine.asan_fake_stack_handle = asan_get_thread_fake_stack_handle();
766 if (!native_thread_id_set_p(native_main_thread.id)) {
769 native_thread_init_main_thread_stack(local_in_parent_frame);
772 if (th->sched.context != NULL) {
777 else if (pthread_equal(curr, native_main_thread.id)) {
778 th->ec->machine.stack_start = native_main_thread.stack_start;
779 th->ec->machine.stack_maxsize = native_main_thread.stack_maxsize;
782#ifdef STACKADDR_AVAILABLE
786 if (get_stack(&start, &size) == 0) {
787 uintptr_t diff = (uintptr_t)start - (uintptr_t)local_in_parent_frame;
788 th->ec->machine.stack_start = local_in_parent_frame;
789 th->ec->machine.stack_maxsize = size - diff;
792 rb_raise(
rb_eNotImpError,
"ruby engine can initialize only in the main thread");
801 struct rb_native_thread *nt;
811 const size_t stack_size = nt->vm->default_params.thread_machine_stack_size;
813#ifdef USE_SIGALTSTACK
814 nt->altstack = rb_allocate_sigaltstack();
817 CHECK_ERR(pthread_attr_init(&attr));
819# ifdef PTHREAD_STACK_MIN
820 RUBY_DEBUG_LOG(
"stack size: %lu", (
unsigned long)stack_size);
821 CHECK_ERR(pthread_attr_setstacksize(&attr, stack_size));
824# ifdef HAVE_PTHREAD_ATTR_SETINHERITSCHED
825 CHECK_ERR(pthread_attr_setinheritsched(&attr, PTHREAD_INHERIT_SCHED));
827 CHECK_ERR(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED));
831 rb_nativethread_id_t thread_id;
832#if USE_RUBY_DEBUG_LOG
835 err = pthread_create(&thread_id, &attr, nt_start, nt);
837 RUBY_DEBUG_LOG(
"nt:%d err:%d", (
int)serial, err);
839 CHECK_ERR(pthread_attr_destroy(&attr));
857 nt->thread_id = pthread_self();
860#ifdef RB_THREAD_T_HAS_NATIVE_ID
861 nt->tid = get_native_thread_id();
865 RB_ALTSTACK_INIT(nt->altstack, nt->altstack);
869native_thread_alloc(
void)
872 native_thread_setup(nt);
878#if USE_RUBY_DEBUG_LOG
888#if USE_NATIVE_THREAD_PRIORITY
891native_thread_apply_priority(rb_thread_t *th)
893#if defined(_POSIX_PRIORITY_SCHEDULING) && (_POSIX_PRIORITY_SCHEDULING > 0)
894 struct sched_param sp;
896 int priority = 0 - th->priority;
898 pthread_getschedparam(th->nt->thread_id, &policy, &sp);
899 max = sched_get_priority_max(policy);
900 min = sched_get_priority_min(policy);
902 if (min > priority) {
905 else if (max < priority) {
909 sp.sched_priority = priority;
910 pthread_setschedparam(th->nt->thread_id, policy, &sp);
921 return rb_fd_select(n, readfds, writefds, exceptfds, timeout);
929native_thread_interrupt(rb_thread_t *th)
931 pthread_kill(th->nt->thread_id, SIGVTALRM);
935native_thread_default_max_cpu(
void)
937#if defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN)
938 long nprocessors = sysconf(_SC_NPROCESSORS_ONLN);
939 return (nprocessors > 0) ? (int)nprocessors : 8;
947#define WRITE_CONST(fd, str) (void)(write((fd),(str),sizeof(str)-1)<0)
950rb_thread_wakeup_timer_thread(
int sig)
956 timer_thread_wakeup_force();
960 rb_vm_t *vm = GET_VM();
961 rb_thread_t *main_th = vm->ractor.main_thread;
964 volatile rb_execution_context_t *main_th_ec = ACCESS_ONCE(rb_execution_context_t *, main_th->ec);
967 RUBY_VM_SET_TRAP_INTERRUPT(main_th_ec);
969 if (vm->ubf_async_safe && main_th->unblock.func) {
970 (main_th->unblock.func)(main_th->unblock.arg);
977#define CLOSE_INVALIDATE_PAIR(expr) \
978 close_invalidate_pair(expr,"close_invalidate: "#expr)
980close_invalidate(
int *fdp,
const char *msg)
986 async_bug_fd(msg,
errno, fd);
991close_invalidate_pair(
int fds[2],
const char *msg)
993 if (USE_EVENTFD && fds[0] == fds[1]) {
995 close_invalidate(&fds[0], msg);
998 close_invalidate(&fds[1], msg);
999 close_invalidate(&fds[0], msg);
1009 oflags = fcntl(fd, F_GETFL);
1012 oflags |= O_NONBLOCK;
1013 err = fcntl(fd, F_SETFL, oflags);
1020setup_communication_pipe_internal(
int pipes[2])
1024 if (pipes[0] > 0 || pipes[1] > 0) {
1025 VM_ASSERT(pipes[0] > 0);
1026 VM_ASSERT(pipes[1] > 0);
1034#if USE_EVENTFD && defined(EFD_NONBLOCK) && defined(EFD_CLOEXEC)
1035 pipes[0] = pipes[1] = eventfd(0, EFD_NONBLOCK|EFD_CLOEXEC);
1037 if (pipes[0] >= 0) {
1045 rb_bug(
"can not create communication pipe");
1049 set_nonblock(pipes[0]);
1050 set_nonblock(pipes[1]);
1053#if !defined(SET_CURRENT_THREAD_NAME) && defined(__linux__) && defined(PR_SET_NAME)
1054# define SET_CURRENT_THREAD_NAME(name) prctl(PR_SET_NAME, name)
1059#if defined(__linux__)
1061#elif defined(__APPLE__)
1069static VALUE threadptr_invoke_proc_location(rb_thread_t *th);
1072native_set_thread_name(rb_thread_t *th)
1074#ifdef SET_CURRENT_THREAD_NAME
1081 if (!th->has_dedicated_nt)
return;
1083 if (!
NIL_P(loc = th->name)) {
1084 SET_CURRENT_THREAD_NAME(RSTRING_PTR(loc));
1086 else if ((loc = threadptr_invoke_proc_location(th)) !=
Qnil) {
1088 char buf[THREAD_NAME_MAX];
1093 p = strrchr(name,
'/');
1101 if (
len >=
sizeof(buf)) {
1102 buf[
sizeof(buf)-2] =
'*';
1103 buf[
sizeof(buf)-1] =
'\0';
1105 SET_CURRENT_THREAD_NAME(buf);
1111native_set_another_thread_name(rb_nativethread_id_t thread_id,
VALUE name)
1113#if defined SET_ANOTHER_THREAD_NAME || defined SET_CURRENT_THREAD_NAME
1114 char buf[THREAD_NAME_MAX];
1117 if (!native_thread_id_set_p(thread_id))
return;
1118# if !defined SET_ANOTHER_THREAD_NAME
1119 if (!pthread_equal(pthread_self(), thread_id))
return;
1124 if (n >= (
int)
sizeof(buf)) {
1125 memcpy(buf, s,
sizeof(buf)-1);
1126 buf[
sizeof(buf)-1] =
'\0';
1130# if defined SET_ANOTHER_THREAD_NAME
1131 SET_ANOTHER_THREAD_NAME(thread_id, s);
1132# elif defined SET_CURRENT_THREAD_NAME
1133 SET_CURRENT_THREAD_NAME(s);
1138#if defined(RB_THREAD_T_HAS_NATIVE_ID) || defined(__APPLE__)
1140native_thread_native_thread_id(rb_thread_t *target_th)
1142 if (!target_th->nt)
return Qnil;
1144#ifdef RB_THREAD_T_HAS_NATIVE_ID
1145 int tid = target_th->nt->tid;
1146 if (tid == 0)
return Qnil;
1148#elif defined(__APPLE__)
1150 pthread_t thread_id = ACCESS_ONCE(pthread_t, target_th->nt->thread_id);
1151 if (!thread_id)
return Qnil;
1152 int e = pthread_threadid_np(thread_id, &tid);
1154 return ULL2NUM((
unsigned long long)tid);
1157# define USE_NATIVE_THREAD_NATIVE_THREAD_ID 1
1159# define USE_NATIVE_THREAD_NATIVE_THREAD_ID 0
1163 rb_serial_t created_fork_gen;
1164 pthread_t pthread_id;
1168#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
1171#if HAVE_SYS_EPOLL_H && USE_MN_THREADS
1172#define EPOLL_EVENTS_MAX 0x10
1173 struct epoll_event finished_events[EPOLL_EVENTS_MAX];
1174#elif HAVE_SYS_EVENT_H && USE_MN_THREADS
1175#define KQUEUE_EVENTS_MAX 0x10
1176 struct kevent finished_events[KQUEUE_EVENTS_MAX];
1183#define TIMER_WHEEL_LEVELS 4
1184#define TIMER_WHEEL_SLOT_BITS 6
1185#define TIMER_WHEEL_SLOTS (1 << TIMER_WHEEL_SLOT_BITS)
1186 struct timer_wheel_level {
1188 struct ccan_list_head slots[TIMER_WHEEL_SLOTS];
1189 } wheel[TIMER_WHEEL_LEVELS];
1190 uint64_t wheel_cursor_tick;
1191 rb_hrtime_t next_expiry;
1192 pthread_mutex_t waiting_lock;
1198#define IO_WAIT_SHARDS 16
1199 pthread_mutex_t fd_shard_locks[IO_WAIT_SHARDS];
1202 pthread_mutex_t wake_pending_lock;
1203 rb_nativethread_cond_t wake_pending_cond;
1206#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
1210#define FDMAP_MAX_CHUNKS 1024
1211 struct rb_fd_waiters *fdmap_chunks[FDMAP_MAX_CHUNKS];
1214 .created_fork_gen = 0,
1217#define TIMER_THREAD_CREATED_P() (timer_th.created_fork_gen == current_fork_gen)
1219static void timer_thread_check_timeslice(rb_vm_t *vm);
1220static bool timeslice_scan(rb_vm_t *vm,
bool interrupt);
1221static int timer_thread_set_timeout(rb_vm_t *vm);
1223#include "thread_sched_mn.c"
1227signal_communication_pipe(
int fd)
1230 const uint64_t buff = 1;
1232 const char buff =
'!';
1239 if ((result = write(fd, &buff,
sizeof(buff))) <= 0) {
1242 case EINTR:
goto retry;
1244#if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN
1249 async_bug_fd(
"rb_thread_wakeup_timer_thread: write", e, fd);
1252 if (TT_DEBUG) WRITE_CONST(2,
"rb_thread_wakeup_timer_thread: write\n");
1260timer_thread_wakeup_force(
void)
1263 signal_communication_pipe(timer_th.comm_fds[1]);
1268rb_thread_create_timer_thread(
void)
1270 rb_serial_t created_fork_gen = timer_th.created_fork_gen;
1272 RUBY_DEBUG_LOG(
"fork_gen create:%d current:%d", (
int)created_fork_gen, (
int)current_fork_gen);
1274 timer_th.created_fork_gen = current_fork_gen;
1276 if (created_fork_gen != current_fork_gen) {
1277 if (created_fork_gen != 0) {
1278 RUBY_DEBUG_LOG(
"forked child process");
1280 CLOSE_INVALIDATE_PAIR(timer_th.comm_fds);
1286 for (
unsigned int ci = 0; ci < FDMAP_MAX_CHUNKS; ci++) {
1288 if (chunk == NULL)
continue;
1289 for (
unsigned int i = 0; i < FDMAP_CHUNK_SIZE; i++) {
1290 ccan_list_head_init(&chunk[i].waiters);
1291 chunk[i].armed_flags = 0;
1292 chunk[i].generation++;
1296#if HAVE_SYS_EPOLL_H && USE_MN_THREADS
1297 close_invalidate(&timer_th.event_fd,
"close event_fd");
1298#elif HAVE_SYS_EVENT_H && USE_MN_THREADS
1301 timer_th.event_fd = -1;
1310 for (
int lvl = 0; lvl < TIMER_WHEEL_LEVELS; lvl++) {
1311 timer_th.wheel[lvl].occupied = 0;
1312 for (
int slot = 0; slot < TIMER_WHEEL_SLOTS; slot++) {
1313 ccan_list_head_init(&timer_th.wheel[lvl].slots[slot]);
1316 timer_th.wheel_cursor_tick = timer_wheel_tick(rb_hrtime_now());
1317 timer_th.next_expiry = TIMER_WHEEL_NO_EXPIRY;
1319 for (
int i = 0; i < IO_WAIT_SHARDS; i++) {
1327 setup_communication_pipe_internal(timer_th.comm_fds);
1330 timer_thread_setup_mn();
1333 int err = pthread_create(&timer_th.pthread_id, NULL, timer_thread_func, GET_VM());
1342native_stop_timer_thread(
void)
1346 RUBY_DEBUG_LOG(
"wakeup send %d", timer_th.comm_fds[1]);
1347 timer_thread_wakeup_force();
1348 RUBY_DEBUG_LOG(
"wakeup sent");
1349 pthread_join(timer_th.pthread_id, NULL);
1351 if (TT_DEBUG) fprintf(stderr,
"stop timer thread\n");
1357native_reset_timer_thread(
void)
1362#ifdef HAVE_SIGALTSTACK
1364ruby_stack_overflowed_p(
const rb_thread_t *th,
const void *addr)
1368 const size_t water_mark = RB_THREAD_PAGE_SIZE;
1369 STACK_GROW_DIR_DETECTION;
1372 size = th->ec->machine.stack_maxsize;
1373 base = (
char *)th->ec->machine.stack_start - STACK_DIR_UPPER(0, size);
1375#ifdef STACKADDR_AVAILABLE
1376 else if (get_stack(&base, &size) == 0) {
1379 if (pthread_equal(pthread_self(), native_main_thread.id)) {
1381 if (getrlimit(RLIMIT_STACK, &rlim) == 0 && rlim.rlim_cur > size) {
1382 size = (size_t)rlim.rlim_cur;
1386 base = (
char *)base + STACK_DIR_UPPER(+size, -size);
1393 if (size > water_mark) size = water_mark;
1394 if (IS_STACK_DIR_UPPER()) {
1395 if (size > ~(
size_t)base+1) size = ~(size_t)base+1;
1396 if (addr > base && addr <= (
void *)((
char *)base + size))
return 1;
1399 if (size > (
size_t)base) size = (size_t)base;
1400 if (addr > (
void *)((
char *)base - size) && addr <= base)
return 1;
1410 if (fd < 0)
return 0;
1412 if (fd == timer_th.comm_fds[0] ||
1413 fd == timer_th.comm_fds[1]
1414#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
1415 || fd == timer_th.event_fd
1418 goto check_fork_gen;
1423 if (timer_th.created_fork_gen == current_fork_gen) {
1435 return pthread_self();
1438#if defined(USE_POLL) && !defined(HAVE_PPOLL)
1441ruby_ppoll(
struct pollfd *fds, nfds_t nfds,
1442 const struct timespec *ts,
const sigset_t *sigmask)
1449 if (ts->tv_sec > INT_MAX/1000)
1450 timeout_ms = INT_MAX;
1452 tmp = (int)(ts->tv_sec * 1000);
1454 tmp2 = (int)((ts->tv_nsec + 999999L) / (1000L * 1000L));
1455 if (INT_MAX - tmp < tmp2)
1456 timeout_ms = INT_MAX;
1458 timeout_ms = (int)(tmp + tmp2);
1464 return poll(fds, nfds, timeout_ms);
1466# define ppoll(fds,nfds,ts,sigmask) ruby_ppoll((fds),(nfds),(ts),(sigmask))
1471static pthread_rwlock_t rb_thread_fork_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
1473#ifndef NON_SCALAR_THREAD_ID
1475static void *last_exiting_thread;
1476STATIC_ASSERT(pthread_t_fits_in_pointer,
sizeof(pthread_t) <=
sizeof(
void *));
1480rb_thread_release_fork_lock(
void)
1483 if ((r = pthread_rwlock_unlock(&rb_thread_fork_rw_lock))) {
1489rb_thread_reset_fork_lock(
void)
1494 rb_thread_fork_rw_lock = (pthread_rwlock_t)PTHREAD_RWLOCK_INITIALIZER;
1498rb_thread_prevent_fork(
void *(*func)(
void *),
void *data)
1501 if ((r = pthread_rwlock_rdlock(&rb_thread_fork_rw_lock))) {
1504 void *result = func(data);
1505 rb_thread_release_fork_lock();
1509#ifndef NON_SCALAR_THREAD_ID
1511thread_register_exiting(
void *thread)
1514 if (previous_thread) {
1516 if ((r = pthread_join((pthread_t)(uintptr_t)previous_thread, NULL))) {
1526rb_thread_register_exiting(
void)
1528#ifdef NON_SCALAR_THREAD_ID
1530 pthread_detach(pthread_self());
1532 thread_register_exiting((
void *)(uintptr_t)pthread_self());
1537rb_thread_acquire_fork_lock(
void)
1540 if ((r = pthread_rwlock_wrlock(&rb_thread_fork_rw_lock))) {
1547#ifndef NON_SCALAR_THREAD_ID
1548 thread_register_exiting(NULL);
1554struct rb_internal_thread_event_hook {
1555 rb_internal_thread_event_callback callback;
1559 struct rb_internal_thread_event_hook *next;
1562static pthread_rwlock_t rb_internal_thread_event_hooks_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
1569rb_thread_event_hooks_registered_p(
void)
1572 if ((r = pthread_rwlock_rdlock(&rb_internal_thread_event_hooks_rw_lock))) {
1575 const bool registered = (rb_internal_thread_event_hooks != NULL);
1576 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
1582#if defined(HAVE_WORKING_FORK)
1584rb_internal_thread_event_hooks_rw_lock_atfork(
void)
1593 rb_internal_thread_event_hooks_rw_lock =
1594 (pthread_rwlock_t)PTHREAD_RWLOCK_INITIALIZER;
1598rb_internal_thread_event_hook_t *
1601 rb_internal_thread_event_hook_t *hook =
ALLOC_N(rb_internal_thread_event_hook_t, 1);
1602 hook->callback = callback;
1603 hook->user_data = user_data;
1604 hook->event = internal_event;
1607 if ((r = pthread_rwlock_wrlock(&rb_internal_thread_event_hooks_rw_lock))) {
1611 hook->next = rb_internal_thread_event_hooks;
1612 ATOMIC_PTR_EXCHANGE(rb_internal_thread_event_hooks, hook);
1614 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
1624 if ((r = pthread_rwlock_wrlock(&rb_internal_thread_event_hooks_rw_lock))) {
1628 bool success = FALSE;
1630 if (rb_internal_thread_event_hooks == hook) {
1631 ATOMIC_PTR_EXCHANGE(rb_internal_thread_event_hooks, hook->next);
1635 rb_internal_thread_event_hook_t *h = rb_internal_thread_event_hooks;
1638 if (h->next == hook) {
1639 h->next = hook->next;
1643 }
while ((h = h->next));
1646 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
1663 if (th->self == 0)
return;
1664 if ((r = pthread_rwlock_rdlock(&rb_internal_thread_event_hooks_rw_lock))) {
1668 if (rb_internal_thread_event_hooks) {
1669 rb_internal_thread_event_hook_t *h = rb_internal_thread_event_hooks;
1671 if (h->event & event) {
1672 rb_internal_thread_event_data_t event_data = {
1675 (*h->callback)(event, &event_data, h->user_data);
1677 }
while((h = h->next));
1679 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
#define RUBY_ATOMIC_PTR_EXCHANGE(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except it expects its arguments are void*.
#define RUBY_ATOMIC_FETCH_ADD(var, val)
Atomically replaces the value pointed by var with the result of addition of val to the old value of v...
#define RUBY_ATOMIC_LOAD(var)
Atomic load.
#define RUBY_ATOMIC_SET(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except for the return type.
uint32_t rb_event_flag_t
Represents event(s).
#define INT2FIX
Old name of RB_INT2FIX.
#define ZALLOC
Old name of RB_ZALLOC.
#define ALLOC_N
Old name of RB_ALLOC_N.
#define ULL2NUM
Old name of RB_ULL2NUM.
#define NUM2INT
Old name of RB_NUM2INT.
#define Qnil
Old name of RUBY_Qnil.
#define NIL_P
Old name of RB_NIL_P.
VALUE rb_eNotImpError
NotImplementedError exception.
void rb_syserr_fail(int e, const char *mesg)
Raises appropriate exception that represents a C errno.
void rb_bug_errno(const char *mesg, int errno_arg)
This is a wrapper of rb_bug() which automatically constructs appropriate message from the passed errn...
int rb_cloexec_pipe(int fildes[2])
Opens a pipe with closing on exec.
void rb_update_max_fd(int fd)
Informs the interpreter that the passed fd can be the max.
int rb_reserved_fd_p(int fd)
Queries if the given FD is reserved or not.
void rb_timespec_now(struct timespec *ts)
Fills the current time into the given struct.
int len
Length of the buffer.
#define RUBY_INTERNAL_THREAD_EVENT_RESUMED
Triggered when a thread successfully acquired the GVL.
rb_internal_thread_event_hook_t * rb_internal_thread_add_event_hook(rb_internal_thread_event_callback func, rb_event_flag_t events, void *data)
Registers a thread event hook function.
#define RUBY_INTERNAL_THREAD_EVENT_EXITED
Triggered when a thread exits.
#define RUBY_INTERNAL_THREAD_EVENT_SUSPENDED
Triggered when a thread released the GVL.
#define RUBY_INTERNAL_THREAD_EVENT_STARTED
Triggered when a new thread is started.
bool rb_internal_thread_remove_event_hook(rb_internal_thread_event_hook_t *hook)
Unregister the passed hook.
#define RUBY_INTERNAL_THREAD_EVENT_READY
Triggered when a thread attempt to acquire the GVL.
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
#define rb_fd_select
Waits for multiple file descriptors at once.
#define RARRAY_AREF(a, i)
References an object in an array.
#define RSTRING_GETMEM(str, ptrvar, lenvar)
Convenient macro to obtain the contents and length at once.
#define errno
Ractor-aware version of errno.
The data structure which wraps the fd_set bitmap used by select(2).
rb_nativethread_id_t rb_nativethread_self(void)
Queries the ID of the native thread that is calling this function.
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_cond_initialize(rb_nativethread_cond_t *cond)
Fills the passed condition variable with an initial value.
int rb_native_mutex_trylock(rb_nativethread_lock_t *lock)
Identical to rb_native_mutex_lock(), except it doesn't block in case rb_native_mutex_lock() would.
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
void rb_native_cond_destroy(rb_nativethread_cond_t *cond)
Destroys the passed condition variable.
void rb_native_cond_signal(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
Waits for the passed condition variable to be signalled.
void rb_native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, unsigned long msec)
Identical to rb_native_cond_wait(), except it additionally takes timeout in msec resolution.
long rb_len_t
A signed integer type for lengths and indices of objects such as String and Array.
uintptr_t VALUE
Type that represents a Ruby object.