28#include "ruby/internal/config.h"
31#include "internal/compilers.h"
37#if defined(HAVE_X86INTRIN_H)
38# include <x86intrin.h>
39#elif defined(_MSC_VER)
43#if defined(_MSC_VER) && defined(__AVX__)
44# pragma intrinsic(__popcnt)
45# pragma intrinsic(__popcnt64)
48#if defined(_MSC_VER) && defined(__AVX2__)
49# pragma intrinsic(__lzcnt)
50# pragma intrinsic(__lzcnt64)
54# pragma intrinsic(_rotl)
55# pragma intrinsic(_rotr)
57# pragma intrinsic(_rotl64)
58# pragma intrinsic(_rotr64)
60# pragma intrinsic(_BitScanForward)
61# pragma intrinsic(_BitScanReverse)
63# pragma intrinsic(_BitScanForward64)
64# pragma intrinsic(_BitScanReverse64)
69#include "internal/static_assert.h"
75#define HALF_LONG_MSB ((SIGNED_VALUE)1<<((SIZEOF_LONG*CHAR_BIT-1)/2))
77#define SIGNED_INTEGER_TYPE_P(T) (0 > ((T)0)-1)
79#define SIGNED_INTEGER_MIN(T) \
80 ((sizeof(T) == sizeof(int8_t)) ? ((T)INT8_MIN) : \
81 ((sizeof(T) == sizeof(int16_t)) ? ((T)INT16_MIN) : \
82 ((sizeof(T) == sizeof(int32_t)) ? ((T)INT32_MIN) : \
83 ((sizeof(T) == sizeof(int64_t)) ? ((T)INT64_MIN) : \
86#define SIGNED_INTEGER_MAX(T) ((T)(SIGNED_INTEGER_MIN(T) ^ ((T)~(T)0)))
88#define UNSIGNED_INTEGER_MAX(T) ((T)~(T)0)
90#ifndef MUL_OVERFLOW_SIGNED_INTEGER_P
91#if __has_builtin(__builtin_mul_overflow_p)
92# define MUL_OVERFLOW_P(a, b) \
93 __builtin_mul_overflow_p((a), (b), (__typeof__(a * b))0)
94#elif __has_builtin(__builtin_mul_overflow)
95# define MUL_OVERFLOW_P(a, b) \
96 __extension__ ({ __typeof__(a) c; __builtin_mul_overflow((a), (b), &c); })
99#define MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, min, max) ( \
101 (a) == -1 ? (b) < -(max) : \
103 ((b) > 0 ? (max) / (a) < (b) : (min) / (a) > (b)) : \
104 ((b) > 0 ? (min) / (a) < (b) : (max) / (a) > (b)))
106#if __has_builtin(__builtin_mul_overflow_p)
109# define MUL_OVERFLOW_FIXNUM_P(a, b) \
111 struct { long fixnum : sizeof(long) * CHAR_BIT - 1; } c = { 0 }; \
112 __builtin_mul_overflow_p((a), (b), c.fixnum); \
115# define MUL_OVERFLOW_FIXNUM_P(a, b) \
116 MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, FIXNUM_MIN, FIXNUM_MAX)
119#if defined(MUL_OVERFLOW_P) && defined(USE___BUILTIN_MUL_OVERFLOW_LONG_LONG)
120# define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_P(a, b)
122# define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX)
126# define MUL_OVERFLOW_LONG_P(a, b) MUL_OVERFLOW_P(a, b)
127# define MUL_OVERFLOW_INT_P(a, b) MUL_OVERFLOW_P(a, b)
129# define MUL_OVERFLOW_LONG_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LONG_MIN, LONG_MAX)
130# define MUL_OVERFLOW_INT_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, INT_MIN, INT_MAX)
134#ifndef ADD_OVERFLOW_SIGNED_INTEGER_P
135#if __has_builtin(__builtin_add_overflow_p)
136# define ADD_OVERFLOW_P(a, b) \
137 __builtin_add_overflow_p((a), (b), (__typeof__(a * b))0)
138#elif __has_builtin(__builtin_add_overflow)
139# define ADD_OVERFLOW_P(a, b) \
140 __extension__ ({ __typeof__(a) c; __builtin_add_overflow((a), (b), &c); })
143#define ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, min, max) ( \
144 (a) > 0 ? (b) > (max) - (a) : (b) < (min) - (a))
146#if __has_builtin(__builtin_add_overflow_p)
149# define ADD_OVERFLOW_FIXNUM_P(a, b) \
151 struct { long fixnum : sizeof(long) * CHAR_BIT - 1; } c = { 0 }; \
152 __builtin_add_overflow_p((a), (b), c.fixnum); \
155# define ADD_OVERFLOW_FIXNUM_P(a, b) \
156 ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, FIXNUM_MIN, FIXNUM_MAX)
159#if defined(ADD_OVERFLOW_P) && defined(USE___BUILTIN_ADD_OVERFLOW_LONG_LONG)
160# define ADD_OVERFLOW_LONG_LONG_P(a, b) ADD_OVERFLOW_P(a, b)
162# define ADD_OVERFLOW_LONG_LONG_P(a, b) ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX)
166# define ADD_OVERFLOW_LONG_P(a, b) ADD_OVERFLOW_P(a, b)
167# define ADD_OVERFLOW_INT_P(a, b) ADD_OVERFLOW_P(a, b)
169# define ADD_OVERFLOW_LONG_P(a, b) ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, LONG_MIN, LONG_MAX)
170# define ADD_OVERFLOW_INT_P(a, b) ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, INT_MIN, INT_MAX)
174#ifndef SUB_OVERFLOW_SIGNED_INTEGER_P
175#if __has_builtin(__builtin_sub_overflow_p)
176# define SUB_OVERFLOW_P(a, b) \
177 __builtin_sub_overflow_p((a), (b), (__typeof__(a * b))0)
178#elif __has_builtin(__builtin_sub_overflow)
179# define SUB_OVERFLOW_P(a, b) \
180 __extension__ ({ __typeof__(a) c; __builtin_sub_overflow((a), (b), &c); })
183#define SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, min, max) ( \
184 (b) > 0 ? (a) < (min) + (b) : (a) > (max) + (b))
186#if __has_builtin(__builtin_sub_overflow_p)
189# define SUB_OVERFLOW_FIXNUM_P(a, b) \
191 struct { long fixnum : sizeof(long) * CHAR_BIT - 1; } c = { 0 }; \
192 __builtin_sub_overflow_p((a), (b), c.fixnum); \
195# define SUB_OVERFLOW_FIXNUM_P(a, b) \
196 SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, FIXNUM_MIN, FIXNUM_MAX)
199#if defined(SUB_OVERFLOW_P) && defined(USE___BUILTIN_SUB_OVERFLOW_LONG_LONG)
200# define SUB_OVERFLOW_LONG_LONG_P(a, b) SUB_OVERFLOW_P(a, b)
202# define SUB_OVERFLOW_LONG_LONG_P(a, b) SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX)
206# define SUB_OVERFLOW_LONG_P(a, b) SUB_OVERFLOW_P(a, b)
207# define SUB_OVERFLOW_INT_P(a, b) SUB_OVERFLOW_P(a, b)
209# define SUB_OVERFLOW_LONG_P(a, b) SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, LONG_MIN, LONG_MAX)
210# define SUB_OVERFLOW_INT_P(a, b) SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, INT_MIN, INT_MAX)
215# define bit_length(x) \
217 (sizeof(x) <= sizeof(int32_t) ? 32 - nlz_int32((uint32_t)(x)) : \
218 sizeof(x) <= sizeof(int64_t) ? 64 - nlz_int64((uint64_t)(x)) : \
219 128 - nlz_int128((uint128_t)(x)))
221# define bit_length(x) \
223 (sizeof(x) <= sizeof(int32_t) ? 32 - nlz_int32((uint32_t)(x)) : \
224 64 - nlz_int64((uint64_t)(x)))
228# define swap16 ruby_swap16
232# define swap32 ruby_swap32
236# define swap64 ruby_swap64
239static inline uint16_t ruby_swap16(uint16_t);
240static inline uint32_t ruby_swap32(uint32_t);
241static inline uint64_t ruby_swap64(uint64_t);
242static inline unsigned nlz_int(
unsigned x);
243static inline unsigned nlz_long(
unsigned long x);
244static inline unsigned nlz_long_long(
unsigned long long x);
245static inline unsigned nlz_intptr(uintptr_t x);
246static inline unsigned nlz_int32(uint32_t x);
247static inline unsigned nlz_int64(uint64_t x);
249static inline unsigned nlz_int128(uint128_t x);
251static inline unsigned rb_popcount32(uint32_t x);
252static inline unsigned rb_popcount64(uint64_t x);
253static inline unsigned rb_popcount_intptr(uintptr_t x);
254static inline int ntz_int32(uint32_t x);
255static inline int ntz_int64(uint64_t x);
256static inline int ntz_intptr(uintptr_t x);
260static inline uint16_t
261ruby_swap16(uint16_t x)
263#if __has_builtin(__builtin_bswap16)
264 return __builtin_bswap16(x);
266#elif defined(_MSC_VER)
267 return _byteswap_ushort(x);
270 return (x << 8) | (x >> 8);
275static inline uint32_t
276ruby_swap32(uint32_t x)
278#if __has_builtin(__builtin_bswap32)
279 return __builtin_bswap32(x);
281#elif defined(_MSC_VER)
282 return _byteswap_ulong(x);
285 x = ((x & 0x0000FFFF) << 16) | ((x & 0xFFFF0000) >> 16);
286 x = ((x & 0x00FF00FF) << 8) | ((x & 0xFF00FF00) >> 8);
292static inline uint64_t
293ruby_swap64(uint64_t x)
295#if __has_builtin(__builtin_bswap64)
296 return __builtin_bswap64(x);
298#elif defined(_MSC_VER)
299 return _byteswap_uint64(x);
302 x = ((x & 0x00000000FFFFFFFFULL) << 32) | ((x & 0xFFFFFFFF00000000ULL) >> 32);
303 x = ((x & 0x0000FFFF0000FFFFULL) << 16) | ((x & 0xFFFF0000FFFF0000ULL) >> 16);
304 x = ((x & 0x00FF00FF00FF00FFULL) << 8) | ((x & 0xFF00FF00FF00FF00ULL) >> 8);
310static inline unsigned int
313#if defined(_MSC_VER) && defined(__AVX2__)
318 return (
unsigned int)__lzcnt(x);
320#elif defined(__x86_64__) && defined(__LZCNT__)
321 return (
unsigned int)_lzcnt_u32(x);
323#elif defined(_MSC_VER)
325 return _BitScanReverse(&r, x) ? (31 - (int)r) : 32;
327#elif __has_builtin(__builtin_clz)
328 STATIC_ASSERT(sizeof_int,
sizeof(
int) * CHAR_BIT == 32);
329 return x ? (
unsigned int)__builtin_clz(x) : 32;
334 y = x >> 16;
if (y) {n -= 16; x = y;}
335 y = x >> 8;
if (y) {n -= 8; x = y;}
336 y = x >> 4;
if (y) {n -= 4; x = y;}
337 y = x >> 2;
if (y) {n -= 2; x = y;}
338 y = x >> 1;
if (y) {
return n - 2;}
339 return (
unsigned int)(n - x);
343static inline unsigned int
346#if defined(_MSC_VER) && defined(__AVX2__)
347 return (
unsigned int)__lzcnt64(x);
349#elif defined(__x86_64__) && defined(__LZCNT__)
350 return (
unsigned int)_lzcnt_u64(x);
352#elif defined(_WIN64) && defined(_MSC_VER)
354 return _BitScanReverse64(&r, x) ? (63u - (
unsigned int)r) : 64;
356#elif __has_builtin(__builtin_clzl)
360 else if (
sizeof(
long) * CHAR_BIT == 64) {
361 return (
unsigned int)__builtin_clzl((
unsigned long)x);
363 else if (
sizeof(
long long) * CHAR_BIT == 64) {
364 return (
unsigned int)__builtin_clzll((
unsigned long long)x);
374 y = x >> 32;
if (y) {n -= 32; x = y;}
375 y = x >> 16;
if (y) {n -= 16; x = y;}
376 y = x >> 8;
if (y) {n -= 8; x = y;}
377 y = x >> 4;
if (y) {n -= 4; x = y;}
378 y = x >> 2;
if (y) {n -= 2; x = y;}
379 y = x >> 1;
if (y) {
return n - 2;}
380 return (
unsigned int)(n - x);
386static inline unsigned int
387nlz_int128(uint128_t x)
389 uint64_t y = (uint64_t)(x >> 64);
395 return (
unsigned int)nlz_int64(x) + 64;
398 return (
unsigned int)nlz_int64(y);
403static inline unsigned int
404nlz_int(
unsigned int x)
406 if (
sizeof(
unsigned int) * CHAR_BIT == 32) {
407 return nlz_int32((uint32_t)x);
409 else if (
sizeof(
unsigned int) * CHAR_BIT == 64) {
410 return nlz_int64((uint64_t)x);
417static inline unsigned int
418nlz_long(
unsigned long x)
420 if (
sizeof(
unsigned long) * CHAR_BIT == 32) {
421 return nlz_int32((uint32_t)x);
423 else if (
sizeof(
unsigned long) * CHAR_BIT == 64) {
424 return nlz_int64((uint64_t)x);
431static inline unsigned int
432nlz_long_long(
unsigned long long x)
434 if (
sizeof(
unsigned long long) * CHAR_BIT == 64) {
435 return nlz_int64((uint64_t)x);
438 else if (
sizeof(
unsigned long long) * CHAR_BIT == 128) {
439 return nlz_int128((uint128_t)x);
447static inline unsigned int
448nlz_intptr(uintptr_t x)
450 if (
sizeof(uintptr_t) ==
sizeof(
unsigned int)) {
451 return nlz_int((
unsigned int)x);
453 if (
sizeof(uintptr_t) ==
sizeof(
unsigned long)) {
454 return nlz_long((
unsigned long)x);
456 if (
sizeof(uintptr_t) ==
sizeof(
unsigned long long)) {
457 return nlz_long_long((
unsigned long long)x);
464static inline unsigned int
465rb_popcount32(uint32_t x)
467#if defined(_MSC_VER) && defined(__AVX__)
471 return (
unsigned int)__popcnt(x);
473#elif __has_builtin(__builtin_popcount)
474 STATIC_ASSERT(sizeof_int,
sizeof(
int) * CHAR_BIT >= 32);
475 return (
unsigned int)__builtin_popcount(x);
478 x = (x & 0x55555555) + (x >> 1 & 0x55555555);
479 x = (x & 0x33333333) + (x >> 2 & 0x33333333);
480 x = (x & 0x07070707) + (x >> 4 & 0x07070707);
481 x = (x & 0x000f000f) + (x >> 8 & 0x000f000f);
482 x = (x & 0x0000001f) + (x >>16 & 0x0000001f);
483 return (
unsigned int)x;
488static inline unsigned int
489rb_popcount64(uint64_t x)
491#if defined(_MSC_VER) && defined(__AVX__)
492 return (
unsigned int)__popcnt64(x);
494#elif __has_builtin(__builtin_popcount)
495 if (
sizeof(
long) * CHAR_BIT == 64) {
496 return (
unsigned int)__builtin_popcountl((
unsigned long)x);
498 else if (
sizeof(
long long) * CHAR_BIT == 64) {
499 return (
unsigned int)__builtin_popcountll((
unsigned long long)x);
507 x = (x & 0x5555555555555555) + (x >> 1 & 0x5555555555555555);
508 x = (x & 0x3333333333333333) + (x >> 2 & 0x3333333333333333);
509 x = (x & 0x0707070707070707) + (x >> 4 & 0x0707070707070707);
510 x = (x & 0x000f000f000f000f) + (x >> 8 & 0x000f000f000f000f);
511 x = (x & 0x0000001f0000001f) + (x >>16 & 0x0000001f0000001f);
512 x = (x & 0x000000000000003f) + (x >>32 & 0x000000000000003f);
513 return (
unsigned int)x;
518static inline unsigned int
519rb_popcount_intptr(uintptr_t x)
521 if (
sizeof(uintptr_t) * CHAR_BIT == 64) {
522 return rb_popcount64((uint64_t)x);
524 else if (
sizeof(uintptr_t) * CHAR_BIT == 32) {
525 return rb_popcount32((uint32_t)x);
535#if defined(__x86_64__) && defined(__BMI__)
536 return (
unsigned)_tzcnt_u32(x);
538#elif defined(_MSC_VER)
542 return _BitScanForward(&r, x) ? (int)r : 32;
544#elif __has_builtin(__builtin_ctz)
545 STATIC_ASSERT(sizeof_int,
sizeof(
int) * CHAR_BIT == 32);
546 return x ? (unsigned)__builtin_ctz(x) : 32;
549 return rb_popcount32((~x) & (x-1));
557#if defined(__x86_64__) && defined(__BMI__)
558 return (
unsigned)_tzcnt_u64(x);
560#elif defined(_WIN64) && defined(_MSC_VER)
562 return _BitScanForward64(&r, x) ? (int)r : 64;
564#elif __has_builtin(__builtin_ctzl)
568 else if (
sizeof(
long) * CHAR_BIT == 64) {
569 return (
unsigned)__builtin_ctzl((
unsigned long)x);
571 else if (
sizeof(
long long) * CHAR_BIT == 64) {
572 return (
unsigned)__builtin_ctzll((
unsigned long long)x);
580 return rb_popcount64((~x) & (x-1));
586ntz_intptr(uintptr_t x)
588 if (
sizeof(uintptr_t) * CHAR_BIT == 64) {
589 return ntz_int64((uint64_t)x);
591 else if (
sizeof(uintptr_t) * CHAR_BIT == 32) {
592 return ntz_int32((uint32_t)x);
600RUBY_BIT_ROTL(
VALUE v,
int n)
602#if __has_builtin(__builtin_rotateleft32) && (SIZEOF_VALUE * CHAR_BIT == 32)
603 return __builtin_rotateleft32(v, n);
605#elif __has_builtin(__builtin_rotateleft64) && (SIZEOF_VALUE * CHAR_BIT == 64)
606 return __builtin_rotateleft64(v, n);
608#elif defined(_MSC_VER) && (SIZEOF_VALUE * CHAR_BIT == 32)
611#elif defined(_MSC_VER) && (SIZEOF_VALUE * CHAR_BIT == 64)
612 return _rotl64(v, n);
614#elif defined(_lrotl) && (SIZEOF_VALUE == SIZEOF_LONG)
618 const int m = (
sizeof(
VALUE) * CHAR_BIT) - 1;
619 return (v << (n & m)) | (v >> (-n & m));
624RUBY_BIT_ROTR(
VALUE v,
int n)
626#if __has_builtin(__builtin_rotateright32) && (SIZEOF_VALUE * CHAR_BIT == 32)
627 return __builtin_rotateright32(v, n);
629#elif __has_builtin(__builtin_rotateright64) && (SIZEOF_VALUE * CHAR_BIT == 64)
630 return __builtin_rotateright64(v, n);
632#elif defined(_MSC_VER) && (SIZEOF_VALUE * CHAR_BIT == 32)
635#elif defined(_MSC_VER) && (SIZEOF_VALUE * CHAR_BIT == 64)
636 return _rotr64(v, n);
638#elif defined(_lrotr) && (SIZEOF_VALUE == SIZEOF_LONG)
642 const int m = (
sizeof(
VALUE) * CHAR_BIT) - 1;
643 return (v << (-n & m)) | (v >> (n & m));
#define UNREACHABLE_RETURN
Old name of RBIMPL_UNREACHABLE_RETURN.
uintptr_t VALUE
Type that represents a Ruby object.