27#include "ruby/internal/config.h"
33#ifdef HAVE_SYS_TYPES_H
34# include <sys/types.h>
37#if RBIMPL_COMPILER_SINCE(MSVC, 13, 0, 0)
38# pragma intrinsic(_InterlockedOr)
39#elif defined(__sun) && defined(HAVE_ATOMIC_H)
44#include "ruby/backward/2/limits.h"
49#include "ruby/internal/cast.h"
59#if defined(__DOXYGEN__) || defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
60# define RUBY_ATOMIC_GENERIC_MACRO 1
68#if defined(__DOXYGEN__)
70#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
72#elif defined(HAVE_GCC_SYNC_BUILTINS)
78#elif defined(__sun) && defined(HAVE_ATOMIC_H)
80#elif defined(HAVE_STDATOMIC_H)
81# include <stdatomic.h>
84# error No atomic operation found
96#define RUBY_ATOMIC_FETCH_ADD(var, val) rbimpl_atomic_fetch_add(&(var), (val))
107#define RUBY_ATOMIC_FETCH_SUB(var, val) rbimpl_atomic_fetch_sub(&(var), (val))
119#define RUBY_ATOMIC_OR(var, val) rbimpl_atomic_or(&(var), (val))
130#define RUBY_ATOMIC_EXCHANGE(var, val) rbimpl_atomic_exchange(&(var), (val))
143#define RUBY_ATOMIC_CAS(var, oldval, newval) \
144 rbimpl_atomic_cas(&(var), (oldval), (newval))
153#define RUBY_ATOMIC_LOAD(var) rbimpl_atomic_load(&(var))
163#define RUBY_ATOMIC_SET(var, val) rbimpl_atomic_set(&(var), (val))
173#define RUBY_ATOMIC_ADD(var, val) rbimpl_atomic_add(&(var), (val))
183#define RUBY_ATOMIC_SUB(var, val) rbimpl_atomic_sub(&(var), (val))
192#define RUBY_ATOMIC_INC(var) rbimpl_atomic_inc(&(var))
201#define RUBY_ATOMIC_DEC(var) rbimpl_atomic_dec(&(var))
213#define RUBY_ATOMIC_SIZE_FETCH_ADD(var, val) rbimpl_atomic_size_fetch_add(&(var), (val))
224#define RUBY_ATOMIC_SIZE_INC(var) rbimpl_atomic_size_inc(&(var))
235#define RUBY_ATOMIC_SIZE_DEC(var) rbimpl_atomic_size_dec(&(var))
248#define RUBY_ATOMIC_SIZE_EXCHANGE(var, val) \
249 rbimpl_atomic_size_exchange(&(var), (val))
262#define RUBY_ATOMIC_SIZE_CAS(var, oldval, newval) \
263 rbimpl_atomic_size_cas(&(var), (oldval), (newval))
275#define RUBY_ATOMIC_SIZE_ADD(var, val) rbimpl_atomic_size_add(&(var), (val))
287#define RUBY_ATOMIC_SIZE_SUB(var, val) rbimpl_atomic_size_sub(&(var), (val))
305#define RUBY_ATOMIC_PTR_EXCHANGE(var, val) \
306 RBIMPL_CAST(rbimpl_atomic_ptr_exchange((void **)&(var), (void *)val))
316#define RUBY_ATOMIC_PTR_LOAD(var) \
317 RBIMPL_CAST(rbimpl_atomic_ptr_load((void **)&var))
329#define RUBY_ATOMIC_PTR_SET(var, val) \
330 rbimpl_atomic_ptr_set((volatile void **)&(var), (val))
343#define RUBY_ATOMIC_PTR_CAS(var, oldval, newval) \
344 RBIMPL_CAST(rbimpl_atomic_ptr_cas((void **)&(var), (void *)(oldval), (void *)(newval)))
356#define RUBY_ATOMIC_VALUE_SET(var, val) \
357 rbimpl_atomic_value_set(&(var), (val))
370#define RUBY_ATOMIC_VALUE_EXCHANGE(var, val) \
371 rbimpl_atomic_value_exchange(&(var), (val))
384#define RUBY_ATOMIC_VALUE_CAS(var, oldval, newval) \
385 rbimpl_atomic_value_cas(&(var), (oldval), (newval))
396#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
397 return __atomic_fetch_add(ptr, val, __ATOMIC_SEQ_CST);
399#elif defined(HAVE_GCC_SYNC_BUILTINS)
400 return __sync_fetch_and_add(ptr, val);
403 return InterlockedExchangeAdd(ptr, val);
405#elif defined(__sun) && defined(HAVE_ATOMIC_H)
412 return atomic_add_int_nv(ptr, val) - val;
414#elif defined(HAVE_STDATOMIC_H)
415 return atomic_fetch_add((_Atomic
volatile rb_atomic_t *)ptr, val);
418# error Unsupported platform.
427rbimpl_atomic_size_fetch_add(volatile
size_t *ptr,
size_t val)
431#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
432 return __atomic_fetch_add(ptr, val, __ATOMIC_SEQ_CST);
434#elif defined(HAVE_GCC_SYNC_BUILTINS)
435 return __sync_fetch_and_add(ptr, val);
438 return InterlockedExchangeAdd64(ptr, val);
440#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
443 atomic_add_long(ptr, val);
445#elif defined(__sun) && defined(HAVE_ATOMIC_H)
449 rbimpl_atomic_fetch_add(tmp, val);
451#elif defined(HAVE_STDATOMIC_H)
452 return atomic_fetch_add((_Atomic
volatile size_t *)ptr, val);
455# error Unsupported platform.
467#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
473 __atomic_add_fetch(ptr, val, __ATOMIC_SEQ_CST);
475#elif defined(HAVE_GCC_SYNC_BUILTINS)
476 __sync_add_and_fetch(ptr, val);
484 InterlockedExchangeAdd(ptr, val);
486#elif defined(__sun) && defined(HAVE_ATOMIC_H)
489 atomic_add_int(ptr, val);
491#elif defined(HAVE_STDATOMIC_H)
495# error Unsupported platform.
503rbimpl_atomic_size_add(volatile
size_t *ptr,
size_t val)
507#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
508 __atomic_add_fetch(ptr, val, __ATOMIC_SEQ_CST);
510#elif defined(HAVE_GCC_SYNC_BUILTINS)
511 __sync_add_and_fetch(ptr, val);
515 InterlockedExchangeAdd64(ptr, val);
517#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
520 atomic_add_long(ptr, val);
522#elif defined(_WIN32) || (defined(__sun) && defined(HAVE_ATOMIC_H))
526 rbimpl_atomic_add(tmp, val);
528#elif defined(HAVE_STDATOMIC_H)
529 *(_Atomic
volatile size_t *)ptr += val;
532# error Unsupported platform.
544#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
545 rbimpl_atomic_add(ptr, 1);
548 InterlockedIncrement(ptr);
550#elif defined(__sun) && defined(HAVE_ATOMIC_H)
551 atomic_inc_uint(ptr);
553#elif defined(HAVE_STDATOMIC_H)
554 rbimpl_atomic_add(ptr, 1);
557# error Unsupported platform.
565rbimpl_atomic_size_inc(volatile
size_t *ptr)
569#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
570 rbimpl_atomic_size_add(ptr, 1);
573 InterlockedIncrement64(ptr);
575#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
576 atomic_inc_ulong(ptr);
578#elif defined(_WIN32) || (defined(__sun) && defined(HAVE_ATOMIC_H))
581 rbimpl_atomic_size_add(ptr, 1);
583#elif defined(HAVE_STDATOMIC_H)
584 rbimpl_atomic_size_add(ptr, 1);
587# error Unsupported platform.
599#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
600 return __atomic_fetch_sub(ptr, val, __ATOMIC_SEQ_CST);
602#elif defined(HAVE_GCC_SYNC_BUILTINS)
603 return __sync_fetch_and_sub(ptr, val);
607 return InterlockedExchangeAdd(ptr, -val);
609#elif defined(__sun) && defined(HAVE_ATOMIC_H)
611 const signed neg = -1;
613 return atomic_add_int_nv(ptr, neg * val) + val;
615#elif defined(HAVE_STDATOMIC_H)
616 return atomic_fetch_sub((_Atomic
volatile rb_atomic_t *)ptr, val);
619# error Unsupported platform.
631#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
632 __atomic_sub_fetch(ptr, val, __ATOMIC_SEQ_CST);
634#elif defined(HAVE_GCC_SYNC_BUILTINS)
635 __sync_sub_and_fetch(ptr, val);
638 InterlockedExchangeAdd(ptr, -val);
640#elif defined(__sun) && defined(HAVE_ATOMIC_H)
641 const signed neg = -1;
643 atomic_add_int(ptr, neg * val);
645#elif defined(HAVE_STDATOMIC_H)
649# error Unsupported platform.
657rbimpl_atomic_size_sub(volatile
size_t *ptr,
size_t val)
661#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
662 __atomic_sub_fetch(ptr, val, __ATOMIC_SEQ_CST);
664#elif defined(HAVE_GCC_SYNC_BUILTINS)
665 __sync_sub_and_fetch(ptr, val);
668 const ssize_t neg = -1;
669 InterlockedExchangeAdd64(ptr, neg * val);
671#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
672 const signed neg = -1;
674 atomic_add_long(ptr, neg * val);
676#elif defined(_WIN32) || (defined(__sun) && defined(HAVE_ATOMIC_H))
680 rbimpl_atomic_sub(tmp, val);
682#elif defined(HAVE_STDATOMIC_H)
683 *(_Atomic
volatile size_t *)ptr -= val;
686# error Unsupported platform.
698#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
699 rbimpl_atomic_sub(ptr, 1);
702 InterlockedDecrement(ptr);
704#elif defined(__sun) && defined(HAVE_ATOMIC_H)
705 atomic_dec_uint(ptr);
707#elif defined(HAVE_STDATOMIC_H)
708 rbimpl_atomic_sub(ptr, 1);
711# error Unsupported platform.
719rbimpl_atomic_size_dec(volatile
size_t *ptr)
723#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
724 rbimpl_atomic_size_sub(ptr, 1);
727 InterlockedDecrement64(ptr);
729#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
730 atomic_dec_ulong(ptr);
732#elif defined(_WIN32) || (defined(__sun) && defined(HAVE_ATOMIC_H))
735 rbimpl_atomic_size_sub(ptr, 1);
737#elif defined(HAVE_STDATOMIC_H)
738 rbimpl_atomic_size_sub(ptr, 1);
741# error Unsupported platform.
753#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
754 __atomic_or_fetch(ptr, val, __ATOMIC_SEQ_CST);
756#elif defined(HAVE_GCC_SYNC_BUILTINS)
757 __sync_or_and_fetch(ptr, val);
759#elif RBIMPL_COMPILER_SINCE(MSVC, 13, 0, 0)
760 _InterlockedOr(ptr, val);
762#elif defined(_WIN32) && defined(__GNUC__)
770#elif defined(_WIN32) && defined(_M_IX86)
773 __asm lock or [eax], ecx;
775#elif defined(__sun) && defined(HAVE_ATOMIC_H)
776 atomic_or_uint(ptr, val);
778#elif !defined(_WIN32) && defined(HAVE_STDATOMIC_H)
782# error Unsupported platform.
787#if RBIMPL_COMPILER_BEFORE(MSVC, 13, 0, 0)
791 return rbimpl_atomic_or(var, val);
803#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
804 return __atomic_exchange_n(ptr, val, __ATOMIC_SEQ_CST);
806#elif defined(HAVE_GCC_SYNC_BUILTINS)
807 return __sync_lock_test_and_set(ptr, val);
810 return InterlockedExchange(ptr, val);
812#elif defined(__sun) && defined(HAVE_ATOMIC_H)
813 return atomic_swap_uint(ptr, val);
815#elif defined(HAVE_STDATOMIC_H)
816 return atomic_exchange((_Atomic
volatile rb_atomic_t *)ptr, val);
819# error Unsupported platform.
827rbimpl_atomic_size_exchange(volatile
size_t *ptr,
size_t val)
831#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
832 return __atomic_exchange_n(ptr, val, __ATOMIC_SEQ_CST);
834#elif defined(HAVE_GCC_SYNC_BUILTINS)
835 return __sync_lock_test_and_set(ptr, val);
838 return InterlockedExchange64(ptr, val);
840#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
841 return atomic_swap_ulong(ptr, val);
843#elif defined(_WIN32) || (defined(__sun) && defined(HAVE_ATOMIC_H))
847 const rb_atomic_t ret = rbimpl_atomic_exchange(tmp, val);
848 return RBIMPL_CAST((
size_t)ret);
850#elif defined(HAVE_STDATOMIC_H)
851 return atomic_exchange((_Atomic
volatile size_t *)ptr, val);
854# error Unsupported platform.
862rbimpl_atomic_size_set(volatile
size_t *ptr,
size_t val)
866#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
867 __atomic_store_n(ptr, val, __ATOMIC_SEQ_CST);
870 rbimpl_atomic_size_exchange(ptr, val);
879rbimpl_atomic_ptr_exchange(
void *volatile *ptr, const
void *val)
883#elif defined(InterlockedExchangePointer)
885 PVOID *pptr = RBIMPL_CAST((PVOID *)ptr);
886 PVOID pval = RBIMPL_CAST((PVOID)val);
887 return InterlockedExchangePointer(pptr, pval);
889#elif defined(__sun) && defined(HAVE_ATOMIC_H)
890 return atomic_swap_ptr(ptr, RBIMPL_CAST((
void *)val));
895 const size_t sval = RBIMPL_CAST((
size_t)val);
896 volatile size_t *
const sptr = RBIMPL_CAST((
volatile size_t *)ptr);
897 const size_t sret = rbimpl_atomic_size_exchange(sptr, sval);
898 return RBIMPL_CAST((
void *)sret);
907rbimpl_atomic_ptr_set(volatile
void **ptr,
void *val)
911 const size_t sval = RBIMPL_CAST((
size_t)val);
912 volatile size_t *
const sptr = RBIMPL_CAST((
volatile size_t *)ptr);
913 rbimpl_atomic_size_set(sptr, sval);
920rbimpl_atomic_value_exchange(volatile
VALUE *ptr,
VALUE val)
924 const size_t sval = RBIMPL_CAST((
size_t)val);
925 volatile size_t *
const sptr = RBIMPL_CAST((
volatile size_t *)ptr);
926 const size_t sret = rbimpl_atomic_size_exchange(sptr, sval);
927 return RBIMPL_CAST((
VALUE)sret);
934rbimpl_atomic_value_set(volatile
VALUE *ptr,
VALUE val)
938 const size_t sval = RBIMPL_CAST((
size_t)val);
939 volatile size_t *
const sptr = RBIMPL_CAST((
volatile size_t *)ptr);
940 rbimpl_atomic_size_set(sptr, sval);
951#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
952 return __atomic_load_n(ptr, __ATOMIC_SEQ_CST);
954 return rbimpl_atomic_fetch_add(ptr, 0);
966#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
967 __atomic_store_n(ptr, val, __ATOMIC_SEQ_CST);
971 rbimpl_atomic_exchange(ptr, val);
984#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
985 __atomic_compare_exchange_n(
986 ptr, &oldval, newval, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
989#elif defined(HAVE_GCC_SYNC_BUILTINS)
990 return __sync_val_compare_and_swap(ptr, oldval, newval);
992#elif RBIMPL_COMPILER_SINCE(MSVC, 13, 0, 0)
993 return InterlockedCompareExchange(ptr, newval, oldval);
996 PVOID *pptr = RBIMPL_CAST((PVOID *)ptr);
997 PVOID pold = RBIMPL_CAST((PVOID)oldval);
998 PVOID pnew = RBIMPL_CAST((PVOID)newval);
999 PVOID pret = InterlockedCompareExchange(pptr, pnew, pold);
1002#elif defined(__sun) && defined(HAVE_ATOMIC_H)
1003 return atomic_cas_uint(ptr, oldval, newval);
1005#elif defined(HAVE_STDATOMIC_H)
1006 atomic_compare_exchange_strong(
1007 (_Atomic
volatile rb_atomic_t *)ptr, &oldval, newval);
1011# error Unsupported platform.
1016#if RBIMPL_COMPILER_BEFORE(MSVC, 13, 0, 0)
1020 return rbimpl_atomic_cas(var, oldval, newval);
1028rbimpl_atomic_size_cas(volatile
size_t *ptr,
size_t oldval,
size_t newval)
1032#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
1033 __atomic_compare_exchange_n(
1034 ptr, &oldval, newval, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
1037#elif defined(HAVE_GCC_SYNC_BUILTINS)
1038 return __sync_val_compare_and_swap(ptr, oldval, newval);
1040#elif defined(_WIN64)
1041 return InterlockedCompareExchange64(ptr, newval, oldval);
1043#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
1044 return atomic_cas_ulong(ptr, oldval, newval);
1046#elif defined(_WIN32) || (defined(__sun) && defined(HAVE_ATOMIC_H))
1050 return rbimpl_atomic_cas(tmp, oldval, newval);
1052#elif defined(HAVE_STDATOMIC_H)
1053 atomic_compare_exchange_strong(
1054 (_Atomic
volatile size_t *)ptr, &oldval, newval);
1058# error Unsupported platform.
1066rbimpl_atomic_ptr_cas(
void **ptr, const
void *oldval, const
void *newval)
1070#elif defined(InterlockedExchangePointer)
1073 PVOID *pptr = RBIMPL_CAST((PVOID *)ptr);
1074 PVOID pold = RBIMPL_CAST((PVOID)oldval);
1075 PVOID pnew = RBIMPL_CAST((PVOID)newval);
1076 return InterlockedCompareExchangePointer(pptr, pnew, pold);
1078#elif defined(__sun) && defined(HAVE_ATOMIC_H)
1079 void *pold = RBIMPL_CAST((
void *)oldval);
1080 void *pnew = RBIMPL_CAST((
void *)newval);
1081 return atomic_cas_ptr(ptr, pold, pnew);
1087 const size_t snew = RBIMPL_CAST((
size_t)newval);
1088 const size_t sold = RBIMPL_CAST((
size_t)oldval);
1089 volatile size_t *
const sptr = RBIMPL_CAST((
volatile size_t *)ptr);
1090 const size_t sret = rbimpl_atomic_size_cas(sptr, sold, snew);
1091 return RBIMPL_CAST((
void *)sret);
1100rbimpl_atomic_ptr_load(
void **ptr)
1104#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
1105 return __atomic_load_n(ptr, __ATOMIC_SEQ_CST);
1108 return rbimpl_atomic_ptr_cas(ptr, val, val);
1116rbimpl_atomic_value_cas(volatile
VALUE *ptr,
VALUE oldval,
VALUE newval)
1120 const size_t snew = RBIMPL_CAST((
size_t)newval);
1121 const size_t sold = RBIMPL_CAST((
size_t)oldval);
1122 volatile size_t *
const sptr = RBIMPL_CAST((
volatile size_t *)ptr);
1123 const size_t sret = rbimpl_atomic_size_cas(sptr, sold, snew);
1124 return RBIMPL_CAST((
VALUE)sret);
Defines RBIMPL_ATTR_ARTIFICIAL.
#define RBIMPL_ATTR_ARTIFICIAL()
Wraps (or simulates) __attribute__((artificial))
#define RBIMPL_ASSERT_OR_ASSUME(...)
This is either RUBY_ASSERT or RBIMPL_ASSUME, depending on RUBY_DEBUG.
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
Defines RBIMPL_COMPILER_SINCE.
Defines RBIMPL_STATIC_ASSERT.
#define RBIMPL_STATIC_ASSERT
Wraps (or simulates) static_assert
Defines RBIMPL_ATTR_NOALIAS.
#define RBIMPL_ATTR_NOALIAS()
Wraps (or simulates) __declspec((noalias))
Defines RBIMPL_ATTR_NONNULL.
#define RBIMPL_ATTR_NONNULL(list)
Wraps (or simulates) __attribute__((nonnull))
#define inline
Old Visual Studio versions do not support the inline keyword, so we need to define it to be __inline.
uintptr_t VALUE
Type that represents a Ruby object.