atomic.h

Go to the documentation of this file.

#ifndef RUBY_ATOMIC_H                                /*-*-C++-*-vi:se ft=cpp:*/
#define RUBY_ATOMIC_H
#include "ruby/internal/config.h"
 
#ifdef STDC_HEADERS
# include <stddef.h>            /* size_t */
#endif
 
#ifdef HAVE_SYS_TYPES_H
# include <sys/types.h>         /* ssize_t */
#endif
 
#if defined(_MSC_VER)
# pragma intrinsic(_InterlockedOr)
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
# include <atomic.h>
#endif
 
#include "ruby/assert.h"
#include "ruby/backward/2/limits.h"
#include "ruby/internal/attr/artificial.h"
#include "ruby/internal/attr/noalias.h"
#include "ruby/internal/attr/nonnull.h"
#include "ruby/internal/compiler_since.h"
#include "ruby/internal/cast.h"
#include "ruby/internal/value.h"
#include "ruby/internal/static_assert.h"
#include "ruby/internal/stdbool.h"
 
/*
 * Asserts that  your environment supports  more than one atomic  types.  These
 * days systems tend to have such property  (C11 was a standard of decades ago,
 * right?) but we still support older ones.
 */
#if defined(__DOXYGEN__) || defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
# define RUBY_ATOMIC_GENERIC_MACRO 1
#endif
 
#if defined(__DOXYGEN__)
using rb_atomic_t = std::atomic<unsigned>;
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
typedef unsigned int rb_atomic_t;
#elif defined(HAVE_GCC_SYNC_BUILTINS)
typedef unsigned int rb_atomic_t;
#elif defined(_WIN32)
# include <winsock2.h>       // to prevent macro redefinitions
# include <windows.h>        // for `LONG` and `Interlocked` functions
typedef LONG rb_atomic_t;
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
typedef unsigned int rb_atomic_t;
#elif defined(HAVE_STDATOMIC_H)
# include <stdatomic.h>
typedef unsigned int rb_atomic_t;
#else
# error No atomic operation found
#endif
 
/* Memory ordering constants */
#if defined(HAVE_GCC_ATOMIC_BUILTINS)
# define RBIMPL_ATOMIC_RELAXED __ATOMIC_RELAXED
# define RBIMPL_ATOMIC_ACQUIRE __ATOMIC_ACQUIRE
# define RBIMPL_ATOMIC_RELEASE __ATOMIC_RELEASE
# define RBIMPL_ATOMIC_ACQ_REL __ATOMIC_ACQ_REL
# define RBIMPL_ATOMIC_SEQ_CST __ATOMIC_SEQ_CST
#elif defined(HAVE_STDATOMIC_H)
# define RBIMPL_ATOMIC_RELAXED memory_order_relaxed
# define RBIMPL_ATOMIC_ACQUIRE memory_order_acquire
# define RBIMPL_ATOMIC_RELEASE memory_order_release
# define RBIMPL_ATOMIC_ACQ_REL memory_order_acq_rel
# define RBIMPL_ATOMIC_SEQ_CST memory_order_seq_cst
#else
/* Dummy values for unsupported platforms */
# define RBIMPL_ATOMIC_RELAXED 0
# define RBIMPL_ATOMIC_ACQUIRE 1
# define RBIMPL_ATOMIC_RELEASE 2
# define RBIMPL_ATOMIC_ACQ_REL 3
# define RBIMPL_ATOMIC_SEQ_CST 4
#endif
 
#define RUBY_ATOMIC_FETCH_ADD(var, val) rbimpl_atomic_fetch_add(&(var), (val), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_FETCH_SUB(var, val) rbimpl_atomic_fetch_sub(&(var), (val), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_OR(var, val) rbimpl_atomic_or(&(var), (val), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_EXCHANGE(var, val) rbimpl_atomic_exchange(&(var), (val), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_CAS(var, oldval, newval) \
    rbimpl_atomic_cas(&(var), (oldval), (newval), RBIMPL_ATOMIC_SEQ_CST, RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_LOAD(var) rbimpl_atomic_load(&(var), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_SET(var, val) rbimpl_atomic_store(&(var), (val), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_ADD(var, val) rbimpl_atomic_add(&(var), (val), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_SUB(var, val) rbimpl_atomic_sub(&(var), (val), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_INC(var) rbimpl_atomic_inc(&(var), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_DEC(var) rbimpl_atomic_dec(&(var), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_SIZE_FETCH_ADD(var, val) rbimpl_atomic_size_fetch_add(&(var), (val), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_SIZE_INC(var) rbimpl_atomic_size_inc(&(var), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_SIZE_DEC(var) rbimpl_atomic_size_dec(&(var), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_SIZE_EXCHANGE(var, val) \
    rbimpl_atomic_size_exchange(&(var), (val), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_SIZE_CAS(var, oldval, newval) \
    rbimpl_atomic_size_cas(&(var), (oldval), (newval), RBIMPL_ATOMIC_SEQ_CST, RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_SIZE_LOAD(var) rbimpl_atomic_size_load(&(var), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_SIZE_ADD(var, val) rbimpl_atomic_size_add(&(var), (val), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_SIZE_SUB(var, val) rbimpl_atomic_size_sub(&(var), (val), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_PTR_EXCHANGE(var, val) \
    RBIMPL_CAST(rbimpl_atomic_ptr_exchange((void **)&(var), (void *)val, RBIMPL_ATOMIC_SEQ_CST))
 
#define RUBY_ATOMIC_PTR_LOAD(var) \
    RBIMPL_CAST(rbimpl_atomic_ptr_load((void **)&var, RBIMPL_ATOMIC_SEQ_CST))
 
#define RUBY_ATOMIC_PTR_SET(var, val) \
   rbimpl_atomic_ptr_store((volatile void **)&(var), (val), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_PTR_CAS(var, oldval, newval) \
    RBIMPL_CAST(rbimpl_atomic_ptr_cas((void **)&(var), (void *)(oldval), (void *)(newval), RBIMPL_ATOMIC_SEQ_CST, RBIMPL_ATOMIC_SEQ_CST))
 
#define RUBY_ATOMIC_VALUE_SET(var, val) \
    rbimpl_atomic_value_store(&(var), (val), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_VALUE_EXCHANGE(var, val) \
    rbimpl_atomic_value_exchange(&(var), (val), RBIMPL_ATOMIC_SEQ_CST)
 
#define RUBY_ATOMIC_VALUE_CAS(var, oldval, newval) \
    rbimpl_atomic_value_cas(&(var), (oldval), (newval), RBIMPL_ATOMIC_SEQ_CST, RBIMPL_ATOMIC_SEQ_CST)
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline rb_atomic_t
rbimpl_atomic_fetch_add(volatile rb_atomic_t *ptr, rb_atomic_t val, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    return __atomic_fetch_add(ptr, val, memory_order);
 
#elif defined(HAVE_GCC_SYNC_BUILTINS)
    return __sync_fetch_and_add(ptr, val);
 
#elif defined(_WIN32)
    return InterlockedExchangeAdd(ptr, val);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    /*
     * `atomic_add_int_nv` takes its second argument as `int`!  Meanwhile our
     * `rb_atomic_t` is unsigned.  We cannot pass `val` as-is.  We have to
     * manually check integer overflow.
     */
    RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX);
    return atomic_add_int_nv(ptr, val) - val;
 
#elif defined(HAVE_STDATOMIC_H)
    return atomic_fetch_add_explicit((_Atomic volatile rb_atomic_t *)ptr, val, memory_order);
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline size_t
rbimpl_atomic_size_fetch_add(volatile size_t *ptr, size_t val, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    return __atomic_fetch_add(ptr, val, memory_order);
 
#elif defined(HAVE_GCC_SYNC_BUILTINS)
    return __sync_fetch_and_add(ptr, val);
 
#elif defined(_WIN64)
    return InterlockedExchangeAdd64(RBIMPL_CAST((LONG64 *)ptr), val);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
    /* Ditto for `atomic_add_int_nv`. */
    RBIMPL_ASSERT_OR_ASSUME(val <= LONG_MAX);
    atomic_add_long(ptr, val);
 
#elif defined(_WIN32) || (defined(__sun) && defined(HAVE_ATOMIC_H))
    RBIMPL_STATIC_ASSERT(size_of_rb_atomic_t, sizeof *ptr == sizeof(rb_atomic_t));
 
    volatile rb_atomic_t *const tmp = RBIMPL_CAST((volatile rb_atomic_t *)ptr);
    return rbimpl_atomic_fetch_add(tmp, val, memory_order);
 
#elif defined(HAVE_STDATOMIC_H)
    return atomic_fetch_add_explicit((_Atomic volatile size_t *)ptr, val, memory_order);
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_add(volatile rb_atomic_t *ptr, rb_atomic_t val, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    /*
     * GCC on amd64 is smart enough to detect this `__atomic_add_fetch`'s
     * return value is not used, then compiles it into single `LOCK ADD`
     * instruction.
     */
    __atomic_add_fetch(ptr, val, memory_order);
 
#elif defined(HAVE_GCC_SYNC_BUILTINS)
    __sync_add_and_fetch(ptr, val);
 
#elif defined(_WIN32)
    /*
     * `InterlockedExchangeAdd` is `LOCK XADD`.  It seems there also is
     * `_InterlockedAdd` intrinsic in ARM Windows but not for x86?  Sticking to
     * `InterlockedExchangeAdd` for better portability.
     */
    InterlockedExchangeAdd(ptr, val);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    /* Ditto for `atomic_add_int_nv`. */
    RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX);
    atomic_add_int(ptr, val);
 
#elif defined(HAVE_STDATOMIC_H)
    atomic_fetch_add_explicit((_Atomic volatile rb_atomic_t *)ptr, val, memory_order);
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_size_add(volatile size_t *ptr, size_t val, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_add_fetch(ptr, val, memory_order);
 
#elif defined(HAVE_GCC_SYNC_BUILTINS)
    __sync_add_and_fetch(ptr, val);
 
#elif defined(_WIN64)
    /* Ditto for `InterlockeExchangedAdd`. */
    InterlockedExchangeAdd64(RBIMPL_CAST((LONG64 *)ptr), val);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
    /* Ditto for `atomic_add_int_nv`. */
    RBIMPL_ASSERT_OR_ASSUME(val <= LONG_MAX);
    atomic_add_long(ptr, val);
 
#elif defined(_WIN32) || (defined(__sun) && defined(HAVE_ATOMIC_H))
    RBIMPL_STATIC_ASSERT(size_of_rb_atomic_t, sizeof *ptr == sizeof(rb_atomic_t));
 
    volatile rb_atomic_t *const tmp = RBIMPL_CAST((volatile rb_atomic_t *)ptr);
    rbimpl_atomic_add(tmp, val, memory_order);
 
#elif defined(HAVE_STDATOMIC_H)
    atomic_fetch_add_explicit((_Atomic volatile size_t *)ptr, val, memory_order);
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_inc(volatile rb_atomic_t *ptr, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
    rbimpl_atomic_add(ptr, 1, memory_order);
 
#elif defined(_WIN32)
    InterlockedIncrement(ptr);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    atomic_inc_uint(ptr);
 
#elif defined(HAVE_STDATOMIC_H)
    rbimpl_atomic_add(ptr, 1, memory_order);
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_size_inc(volatile size_t *ptr, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
    rbimpl_atomic_size_add(ptr, 1, memory_order);
 
#elif defined(_WIN64)
    InterlockedIncrement64(RBIMPL_CAST((LONG64 *)ptr));
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
    atomic_inc_ulong(ptr);
 
#elif defined(_WIN32) || (defined(__sun) && defined(HAVE_ATOMIC_H))
    RBIMPL_STATIC_ASSERT(size_of_size_t, sizeof *ptr == sizeof(rb_atomic_t));
 
    rbimpl_atomic_size_add(ptr, 1, memory_order);
 
#elif defined(HAVE_STDATOMIC_H)
    rbimpl_atomic_size_add(ptr, 1, memory_order);
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline rb_atomic_t
rbimpl_atomic_fetch_sub(volatile rb_atomic_t *ptr, rb_atomic_t val, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    return __atomic_fetch_sub(ptr, val, memory_order);
 
#elif defined(HAVE_GCC_SYNC_BUILTINS)
    return __sync_fetch_and_sub(ptr, val);
 
#elif defined(_WIN32)
    /* rb_atomic_t is signed here! Safe to do `-val`. */
    return InterlockedExchangeAdd(ptr, -val);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    /* Ditto for `rbimpl_atomic_fetch_add`. */
    const signed neg = -1;
    RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX);
    return atomic_add_int_nv(ptr, neg * val) + val;
 
#elif defined(HAVE_STDATOMIC_H)
    return atomic_fetch_sub_explicit((_Atomic volatile rb_atomic_t *)ptr, val, memory_order);
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_sub(volatile rb_atomic_t *ptr, rb_atomic_t val, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_sub_fetch(ptr, val, memory_order);
 
#elif defined(HAVE_GCC_SYNC_BUILTINS)
    __sync_sub_and_fetch(ptr, val);
 
#elif defined(_WIN32)
    InterlockedExchangeAdd(ptr, -val);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    const signed neg = -1;
    RBIMPL_ASSERT_OR_ASSUME(val <= INT_MAX);
    atomic_add_int(ptr, neg * val);
 
#elif defined(HAVE_STDATOMIC_H)
    atomic_fetch_sub_explicit((_Atomic volatile rb_atomic_t *)ptr, val, memory_order);
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_size_sub(volatile size_t *ptr, size_t val, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_sub_fetch(ptr, val, memory_order);
 
#elif defined(HAVE_GCC_SYNC_BUILTINS)
    __sync_sub_and_fetch(ptr, val);
 
#elif defined(_WIN64)
    const ssize_t neg = -1;
    InterlockedExchangeAdd64(RBIMPL_CAST((LONG64 *)ptr), neg * val);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
    const signed neg = -1;
    RBIMPL_ASSERT_OR_ASSUME(val <= LONG_MAX);
    atomic_add_long(ptr, neg * val);
 
#elif defined(_WIN32) || (defined(__sun) && defined(HAVE_ATOMIC_H))
    RBIMPL_STATIC_ASSERT(size_of_rb_atomic_t, sizeof *ptr == sizeof(rb_atomic_t));
 
    volatile rb_atomic_t *const tmp = RBIMPL_CAST((volatile rb_atomic_t *)ptr);
    rbimpl_atomic_sub(tmp, val, memory_order);
 
#elif defined(HAVE_STDATOMIC_H)
    atomic_fetch_sub_explicit((_Atomic volatile size_t *)ptr, val, memory_order);
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_dec(volatile rb_atomic_t *ptr, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
    rbimpl_atomic_sub(ptr, 1, memory_order);
 
#elif defined(_WIN32)
    InterlockedDecrement(ptr);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    atomic_dec_uint(ptr);
 
#elif defined(HAVE_STDATOMIC_H)
    rbimpl_atomic_sub(ptr, 1, memory_order);
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_size_dec(volatile size_t *ptr, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_SYNC_BUILTINS)
    rbimpl_atomic_size_sub(ptr, 1, memory_order);
 
#elif defined(_WIN64)
    InterlockedDecrement64(RBIMPL_CAST((LONG64 *)ptr));
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
    atomic_dec_ulong(ptr);
 
#elif defined(_WIN32) || (defined(__sun) && defined(HAVE_ATOMIC_H))
    RBIMPL_STATIC_ASSERT(size_of_size_t, sizeof *ptr == sizeof(rb_atomic_t));
 
    rbimpl_atomic_size_sub(ptr, 1, memory_order);
 
#elif defined(HAVE_STDATOMIC_H)
    rbimpl_atomic_size_sub(ptr, 1, memory_order);
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_or(volatile rb_atomic_t *ptr, rb_atomic_t val, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_or_fetch(ptr, val, memory_order);
 
#elif defined(HAVE_GCC_SYNC_BUILTINS)
    __sync_or_and_fetch(ptr, val);
 
#elif defined(_MSC_VER)
    _InterlockedOr(ptr, val);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    atomic_or_uint(ptr, val);
 
#elif !defined(_WIN32) && defined(HAVE_STDATOMIC_H)
    atomic_fetch_or_explicit((_Atomic volatile rb_atomic_t *)ptr, val, memory_order);
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline rb_atomic_t
rbimpl_atomic_exchange(volatile rb_atomic_t *ptr, rb_atomic_t val, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    return __atomic_exchange_n(ptr, val, memory_order);
 
#elif defined(HAVE_GCC_SYNC_BUILTINS)
    return __sync_lock_test_and_set(ptr, val);
 
#elif defined(_WIN32)
    return InterlockedExchange(ptr, val);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    return atomic_swap_uint(ptr, val);
 
#elif defined(HAVE_STDATOMIC_H)
    return atomic_exchange_explicit((_Atomic volatile rb_atomic_t *)ptr, val, memory_order);
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline size_t
rbimpl_atomic_size_exchange(volatile size_t *ptr, size_t val, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    return __atomic_exchange_n(ptr, val, memory_order);
 
#elif defined(HAVE_GCC_SYNC_BUILTINS)
    return __sync_lock_test_and_set(ptr, val);
 
#elif defined(_WIN64)
    return InterlockedExchange64(RBIMPL_CAST((LONG64 *)ptr), val);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
    return atomic_swap_ulong(ptr, val);
 
#elif defined(_WIN32) || (defined(__sun) && defined(HAVE_ATOMIC_H))
    RBIMPL_STATIC_ASSERT(size_of_size_t, sizeof *ptr == sizeof(rb_atomic_t));
 
    volatile rb_atomic_t *const tmp = RBIMPL_CAST((volatile rb_atomic_t *)ptr);
    const rb_atomic_t ret = rbimpl_atomic_exchange(tmp, val, memory_order);
    return RBIMPL_CAST((size_t)ret);
 
#elif defined(HAVE_STDATOMIC_H)
    return atomic_exchange_explicit((_Atomic volatile size_t *)ptr, val, memory_order);
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_size_store(volatile size_t *ptr, size_t val, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_store_n(ptr, val, memory_order);
 
#else
    rbimpl_atomic_size_exchange(ptr, val, memory_order);
 
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void *
rbimpl_atomic_ptr_exchange(void *volatile *ptr, const void *val, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(InterlockedExchangePointer)
    /* const_cast */
    PVOID *pptr = RBIMPL_CAST((PVOID *)ptr);
    PVOID pval = RBIMPL_CAST((PVOID)val);
    return InterlockedExchangePointer(pptr, pval);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    return atomic_swap_ptr(ptr, RBIMPL_CAST((void *)val));
 
#else
    RBIMPL_STATIC_ASSERT(sizeof_voidp, sizeof *ptr == sizeof(size_t));
 
    const size_t sval = RBIMPL_CAST((size_t)val);
    volatile size_t *const sptr = RBIMPL_CAST((volatile size_t *)ptr);
    const size_t sret = rbimpl_atomic_size_exchange(sptr, sval, memory_order);
    return RBIMPL_CAST((void *)sret);
 
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_ptr_store(volatile void **ptr, void *val, int memory_order)
{
    RBIMPL_STATIC_ASSERT(sizeof_value, sizeof *ptr == sizeof(size_t));
 
    const size_t sval = RBIMPL_CAST((size_t)val);
    volatile size_t *const sptr = RBIMPL_CAST((volatile size_t *)ptr);
    rbimpl_atomic_size_store(sptr, sval, memory_order);
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline VALUE
rbimpl_atomic_value_exchange(volatile VALUE *ptr, VALUE val, int memory_order)
{
    RBIMPL_STATIC_ASSERT(sizeof_value, sizeof *ptr == sizeof(size_t));
 
    const size_t sval = RBIMPL_CAST((size_t)val);
    volatile size_t *const sptr = RBIMPL_CAST((volatile size_t *)ptr);
    const size_t sret = rbimpl_atomic_size_exchange(sptr, sval, memory_order);
    return RBIMPL_CAST((VALUE)sret);
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_value_store(volatile VALUE *ptr, VALUE val, int memory_order)
{
    RBIMPL_STATIC_ASSERT(sizeof_value, sizeof *ptr == sizeof(size_t));
 
    const size_t sval = RBIMPL_CAST((size_t)val);
    volatile size_t *const sptr = RBIMPL_CAST((volatile size_t *)ptr);
    rbimpl_atomic_size_store(sptr, sval, memory_order);
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline rb_atomic_t
rbimpl_atomic_load(volatile rb_atomic_t *ptr, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    return __atomic_load_n(ptr, memory_order);
#elif defined(_M_X64) && !defined(_M_ARM64EC)
    /* A plain load is seq_cst on x64 because the stores here use xchg. */
    return *ptr;
#else
    return rbimpl_atomic_fetch_add(ptr, 0, memory_order);
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_store(volatile rb_atomic_t *ptr, rb_atomic_t val, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_store_n(ptr, val, memory_order);
 
#else
    /* Maybe std::atomic<rb_atomic_t>::store can be faster? */
    rbimpl_atomic_exchange(ptr, val, memory_order);
 
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline rb_atomic_t
rbimpl_atomic_cas(volatile rb_atomic_t *ptr, rb_atomic_t oldval, rb_atomic_t newval, int success_memorder, int failure_memorder)
{
    (void)success_memorder;
    (void)failure_memorder;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_compare_exchange_n(
        ptr, &oldval, newval, 0, success_memorder, failure_memorder);
    return oldval;
 
#elif defined(HAVE_GCC_SYNC_BUILTINS)
    return __sync_val_compare_and_swap(ptr, oldval, newval);
 
#elif defined(_MSC_VER)
    return InterlockedCompareExchange(ptr, newval, oldval);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    return atomic_cas_uint(ptr, oldval, newval);
 
#elif defined(HAVE_STDATOMIC_H)
    atomic_compare_exchange_strong_explicit(
        (_Atomic volatile rb_atomic_t *)ptr, &oldval, newval, success_memorder, failure_memorder);
    return oldval;
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline size_t
rbimpl_atomic_size_cas(volatile size_t *ptr, size_t oldval, size_t newval, int success_memorder, int failure_memorder)
{
    (void)success_memorder;
    (void)failure_memorder;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    __atomic_compare_exchange_n(
        ptr, &oldval, newval, 0, success_memorder, failure_memorder);
    return oldval;
 
#elif defined(HAVE_GCC_SYNC_BUILTINS)
    return __sync_val_compare_and_swap(ptr, oldval, newval);
 
#elif defined(_WIN64)
    return InterlockedCompareExchange64(RBIMPL_CAST((LONG64 *)ptr), newval, oldval);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx))
    return atomic_cas_ulong(ptr, oldval, newval);
 
#elif defined(_WIN32) || (defined(__sun) && defined(HAVE_ATOMIC_H))
    RBIMPL_STATIC_ASSERT(size_of_size_t, sizeof *ptr == sizeof(rb_atomic_t));
 
    volatile rb_atomic_t *tmp = RBIMPL_CAST((volatile rb_atomic_t *)ptr);
    return rbimpl_atomic_cas(tmp, oldval, newval, success_memorder, failure_memorder);
 
#elif defined(HAVE_STDATOMIC_H)
    atomic_compare_exchange_strong_explicit(
        (_Atomic volatile size_t *)ptr, &oldval, newval, success_memorder, failure_memorder);
    return oldval;
 
#else
# error Unsupported platform.
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline size_t
rbimpl_atomic_size_load(volatile size_t *ptr, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    return __atomic_load_n(ptr, memory_order);
#elif defined(_M_X64) && !defined(_M_ARM64EC)
    /* A plain load is seq_cst on x64 because the stores here use xchg. */
    return *ptr;
#else
    size_t val = *ptr;
    return rbimpl_atomic_size_cas(ptr, val, val, memory_order, memory_order);
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void *
rbimpl_atomic_ptr_cas(void **ptr, const void *oldval, const void *newval, int success_memorder, int failure_memorder)
{
    (void)success_memorder;
    (void)failure_memorder;
#if 0
 
#elif defined(InterlockedExchangePointer)
    /* ... Can we say that InterlockedCompareExchangePtr surly exists when
     * InterlockedExchangePointer is defined?  Seems so but...?*/
    PVOID *pptr = RBIMPL_CAST((PVOID *)ptr);
    PVOID pold = RBIMPL_CAST((PVOID)oldval);
    PVOID pnew = RBIMPL_CAST((PVOID)newval);
    return InterlockedCompareExchangePointer(pptr, pnew, pold);
 
#elif defined(__sun) && defined(HAVE_ATOMIC_H)
    void *pold = RBIMPL_CAST((void *)oldval);
    void *pnew = RBIMPL_CAST((void *)newval);
    return atomic_cas_ptr(ptr, pold, pnew);
 
 
#else
    RBIMPL_STATIC_ASSERT(sizeof_voidp, sizeof *ptr == sizeof(size_t));
 
    const size_t snew = RBIMPL_CAST((size_t)newval);
    const size_t sold = RBIMPL_CAST((size_t)oldval);
    volatile size_t *const sptr = RBIMPL_CAST((volatile size_t *)ptr);
    const size_t sret = rbimpl_atomic_size_cas(sptr, sold, snew, success_memorder, failure_memorder);
    return RBIMPL_CAST((void *)sret);
 
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void *
rbimpl_atomic_ptr_load(void **ptr, int memory_order)
{
    (void)memory_order;
#if 0
 
#elif defined(HAVE_GCC_ATOMIC_BUILTINS)
    return __atomic_load_n(ptr, memory_order);
#elif defined(_M_X64) && !defined(_M_ARM64EC)
    return *RBIMPL_CAST((void *volatile *)ptr);
#else
    void *val = *ptr;
    return rbimpl_atomic_ptr_cas(ptr, val, val, memory_order, memory_order);
#endif
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline VALUE
rbimpl_atomic_value_load(volatile VALUE *ptr, int memory_order)
{
    return RBIMPL_CAST((VALUE)rbimpl_atomic_ptr_load((void **)ptr, memory_order));
}
 
RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline VALUE
rbimpl_atomic_value_cas(volatile VALUE *ptr, VALUE oldval, VALUE newval, int success_memorder, int failure_memorder)
{
    RBIMPL_STATIC_ASSERT(sizeof_value, sizeof *ptr == sizeof(size_t));
 
    const size_t snew = RBIMPL_CAST((size_t)newval);
    const size_t sold = RBIMPL_CAST((size_t)oldval);
    volatile size_t *const sptr = RBIMPL_CAST((volatile size_t *)ptr);
    const size_t sret = rbimpl_atomic_size_cas(sptr, sold, snew, success_memorder, failure_memorder);
    return RBIMPL_CAST((VALUE)sret);
}
#endif /* RUBY_ATOMIC_H */