28#include "ruby/internal/config.h"
31#include "internal/compilers.h"
33#if MSC_VERSION_SINCE(1310)
37#if defined(HAVE_X86INTRIN_H)
38# include <x86intrin.h>
39#elif MSC_VERSION_SINCE(1310)
43#if defined(_MSC_VER) && defined(__AVX__)
44# pragma intrinsic(__popcnt)
45# pragma intrinsic(__popcnt64)
48#if defined(_MSC_VER) && defined(__AVX2__)
49# pragma intrinsic(__lzcnt)
50# pragma intrinsic(__lzcnt64)
53#if MSC_VERSION_SINCE(1310)
54# pragma intrinsic(_rotl)
55# pragma intrinsic(_rotr)
57# pragma intrinsic(_rotl64)
58# pragma intrinsic(_rotr64)
62#if MSC_VERSION_SINCE(1400)
63# pragma intrinsic(_BitScanForward)
64# pragma intrinsic(_BitScanReverse)
66# pragma intrinsic(_BitScanForward64)
67# pragma intrinsic(_BitScanReverse64)
72#include "internal/static_assert.h"
78#define HALF_LONG_MSB ((SIGNED_VALUE)1<<((SIZEOF_LONG*CHAR_BIT-1)/2))
80#define SIGNED_INTEGER_TYPE_P(T) (0 > ((T)0)-1)
82#define SIGNED_INTEGER_MIN(T) \
83 ((sizeof(T) == sizeof(int8_t)) ? ((T)INT8_MIN) : \
84 ((sizeof(T) == sizeof(int16_t)) ? ((T)INT16_MIN) : \
85 ((sizeof(T) == sizeof(int32_t)) ? ((T)INT32_MIN) : \
86 ((sizeof(T) == sizeof(int64_t)) ? ((T)INT64_MIN) : \
89#define SIGNED_INTEGER_MAX(T) ((T)(SIGNED_INTEGER_MIN(T) ^ ((T)~(T)0)))
91#define UNSIGNED_INTEGER_MAX(T) ((T)~(T)0)
93#ifndef MUL_OVERFLOW_SIGNED_INTEGER_P
94#if __has_builtin(__builtin_mul_overflow_p)
95# define MUL_OVERFLOW_P(a, b) \
96 __builtin_mul_overflow_p((a), (b), (__typeof__(a * b))0)
97#elif __has_builtin(__builtin_mul_overflow)
98# define MUL_OVERFLOW_P(a, b) \
99 __extension__ ({ __typeof__(a) c; __builtin_mul_overflow((a), (b), &c); })
102#define MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, min, max) ( \
104 (a) == -1 ? (b) < -(max) : \
106 ((b) > 0 ? (max) / (a) < (b) : (min) / (a) > (b)) : \
107 ((b) > 0 ? (min) / (a) < (b) : (max) / (a) > (b)))
109#if __has_builtin(__builtin_mul_overflow_p)
112# define MUL_OVERFLOW_FIXNUM_P(a, b) \
114 struct { long fixnum : sizeof(long) * CHAR_BIT - 1; } c = { 0 }; \
115 __builtin_mul_overflow_p((a), (b), c.fixnum); \
118# define MUL_OVERFLOW_FIXNUM_P(a, b) \
119 MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, FIXNUM_MIN, FIXNUM_MAX)
122#if defined(MUL_OVERFLOW_P) && defined(USE___BUILTIN_MUL_OVERFLOW_LONG_LONG)
123# define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_P(a, b)
125# define MUL_OVERFLOW_LONG_LONG_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX)
129# define MUL_OVERFLOW_LONG_P(a, b) MUL_OVERFLOW_P(a, b)
130# define MUL_OVERFLOW_INT_P(a, b) MUL_OVERFLOW_P(a, b)
132# define MUL_OVERFLOW_LONG_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, LONG_MIN, LONG_MAX)
133# define MUL_OVERFLOW_INT_P(a, b) MUL_OVERFLOW_SIGNED_INTEGER_P(a, b, INT_MIN, INT_MAX)
137#ifndef ADD_OVERFLOW_SIGNED_INTEGER_P
138#if __has_builtin(__builtin_add_overflow_p)
139# define ADD_OVERFLOW_P(a, b) \
140 __builtin_add_overflow_p((a), (b), (__typeof__(a * b))0)
141#elif __has_builtin(__builtin_add_overflow)
142# define ADD_OVERFLOW_P(a, b) \
143 __extension__ ({ __typeof__(a) c; __builtin_add_overflow((a), (b), &c); })
146#define ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, min, max) ( \
147 (a) > 0 ? (b) > (max) - (a) : (b) < (min) - (a))
149#if __has_builtin(__builtin_add_overflow_p)
152# define ADD_OVERFLOW_FIXNUM_P(a, b) \
154 struct { long fixnum : sizeof(long) * CHAR_BIT - 1; } c = { 0 }; \
155 __builtin_add_overflow_p((a), (b), c.fixnum); \
158# define ADD_OVERFLOW_FIXNUM_P(a, b) \
159 ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, FIXNUM_MIN, FIXNUM_MAX)
162#if defined(ADD_OVERFLOW_P) && defined(USE___BUILTIN_ADD_OVERFLOW_LONG_LONG)
163# define ADD_OVERFLOW_LONG_LONG_P(a, b) ADD_OVERFLOW_P(a, b)
165# define ADD_OVERFLOW_LONG_LONG_P(a, b) ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX)
169# define ADD_OVERFLOW_LONG_P(a, b) ADD_OVERFLOW_P(a, b)
170# define ADD_OVERFLOW_INT_P(a, b) ADD_OVERFLOW_P(a, b)
172# define ADD_OVERFLOW_LONG_P(a, b) ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, LONG_MIN, LONG_MAX)
173# define ADD_OVERFLOW_INT_P(a, b) ADD_OVERFLOW_SIGNED_INTEGER_P(a, b, INT_MIN, INT_MAX)
177#ifndef SUB_OVERFLOW_SIGNED_INTEGER_P
178#if __has_builtin(__builtin_sub_overflow_p)
179# define SUB_OVERFLOW_P(a, b) \
180 __builtin_sub_overflow_p((a), (b), (__typeof__(a * b))0)
181#elif __has_builtin(__builtin_sub_overflow)
182# define SUB_OVERFLOW_P(a, b) \
183 __extension__ ({ __typeof__(a) c; __builtin_sub_overflow((a), (b), &c); })
186#define SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, min, max) ( \
187 (b) > 0 ? (a) < (min) + (b) : (a) > (max) + (b))
189#if __has_builtin(__builtin_sub_overflow_p)
192# define SUB_OVERFLOW_FIXNUM_P(a, b) \
194 struct { long fixnum : sizeof(long) * CHAR_BIT - 1; } c = { 0 }; \
195 __builtin_sub_overflow_p((a), (b), c.fixnum); \
198# define SUB_OVERFLOW_FIXNUM_P(a, b) \
199 SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, FIXNUM_MIN, FIXNUM_MAX)
202#if defined(SUB_OVERFLOW_P) && defined(USE___BUILTIN_SUB_OVERFLOW_LONG_LONG)
203# define SUB_OVERFLOW_LONG_LONG_P(a, b) SUB_OVERFLOW_P(a, b)
205# define SUB_OVERFLOW_LONG_LONG_P(a, b) SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, LLONG_MIN, LLONG_MAX)
209# define SUB_OVERFLOW_LONG_P(a, b) SUB_OVERFLOW_P(a, b)
210# define SUB_OVERFLOW_INT_P(a, b) SUB_OVERFLOW_P(a, b)
212# define SUB_OVERFLOW_LONG_P(a, b) SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, LONG_MIN, LONG_MAX)
213# define SUB_OVERFLOW_INT_P(a, b) SUB_OVERFLOW_SIGNED_INTEGER_P(a, b, INT_MIN, INT_MAX)
218# define bit_length(x) \
220 (sizeof(x) <= sizeof(int32_t) ? 32 - nlz_int32((uint32_t)(x)) : \
221 sizeof(x) <= sizeof(int64_t) ? 64 - nlz_int64((uint64_t)(x)) : \
222 128 - nlz_int128((uint128_t)(x)))
224# define bit_length(x) \
226 (sizeof(x) <= sizeof(int32_t) ? 32 - nlz_int32((uint32_t)(x)) : \
227 64 - nlz_int64((uint64_t)(x)))
231# define swap16 ruby_swap16
235# define swap32 ruby_swap32
239# define swap64 ruby_swap64
242static inline uint16_t ruby_swap16(uint16_t);
243static inline uint32_t ruby_swap32(uint32_t);
244static inline uint64_t ruby_swap64(uint64_t);
245static inline unsigned nlz_int(
unsigned x);
246static inline unsigned nlz_long(
unsigned long x);
247static inline unsigned nlz_long_long(
unsigned long long x);
248static inline unsigned nlz_intptr(uintptr_t x);
249static inline unsigned nlz_int32(uint32_t x);
250static inline unsigned nlz_int64(uint64_t x);
252static inline unsigned nlz_int128(uint128_t x);
254static inline unsigned rb_popcount32(uint32_t x);
255static inline unsigned rb_popcount64(uint64_t x);
256static inline unsigned rb_popcount_intptr(uintptr_t x);
257static inline int ntz_int32(uint32_t x);
258static inline int ntz_int64(uint64_t x);
259static inline int ntz_intptr(uintptr_t x);
263static inline uint16_t
264ruby_swap16(uint16_t x)
266#if __has_builtin(__builtin_bswap16)
267 return __builtin_bswap16(x);
269#elif MSC_VERSION_SINCE(1310)
270 return _byteswap_ushort(x);
273 return (x << 8) | (x >> 8);
278static inline uint32_t
279ruby_swap32(uint32_t x)
281#if __has_builtin(__builtin_bswap32)
282 return __builtin_bswap32(x);
284#elif MSC_VERSION_SINCE(1310)
285 return _byteswap_ulong(x);
288 x = ((x & 0x0000FFFF) << 16) | ((x & 0xFFFF0000) >> 16);
289 x = ((x & 0x00FF00FF) << 8) | ((x & 0xFF00FF00) >> 8);
295static inline uint64_t
296ruby_swap64(uint64_t x)
298#if __has_builtin(__builtin_bswap64)
299 return __builtin_bswap64(x);
301#elif MSC_VERSION_SINCE(1310)
302 return _byteswap_uint64(x);
305 x = ((x & 0x00000000FFFFFFFFULL) << 32) | ((x & 0xFFFFFFFF00000000ULL) >> 32);
306 x = ((x & 0x0000FFFF0000FFFFULL) << 16) | ((x & 0xFFFF0000FFFF0000ULL) >> 16);
307 x = ((x & 0x00FF00FF00FF00FFULL) << 8) | ((x & 0xFF00FF00FF00FF00ULL) >> 8);
313static inline unsigned int
316#if defined(_MSC_VER) && defined(__AVX2__)
321 return (
unsigned int)__lzcnt(x);
323#elif defined(__x86_64__) && defined(__LZCNT__)
324 return (
unsigned int)_lzcnt_u32(x);
326#elif MSC_VERSION_SINCE(1400)
328 return _BitScanReverse(&r, x) ? (31 - (int)r) : 32;
330#elif __has_builtin(__builtin_clz)
331 STATIC_ASSERT(sizeof_int,
sizeof(
int) * CHAR_BIT == 32);
332 return x ? (
unsigned int)__builtin_clz(x) : 32;
337 y = x >> 16;
if (y) {n -= 16; x = y;}
338 y = x >> 8;
if (y) {n -= 8; x = y;}
339 y = x >> 4;
if (y) {n -= 4; x = y;}
340 y = x >> 2;
if (y) {n -= 2; x = y;}
341 y = x >> 1;
if (y) {
return n - 2;}
342 return (
unsigned int)(n - x);
346static inline unsigned int
349#if defined(_MSC_VER) && defined(__AVX2__)
350 return (
unsigned int)__lzcnt64(x);
352#elif defined(__x86_64__) && defined(__LZCNT__)
353 return (
unsigned int)_lzcnt_u64(x);
355#elif defined(_WIN64) && MSC_VERSION_SINCE(1400)
357 return _BitScanReverse64(&r, x) ? (63u - (
unsigned int)r) : 64;
359#elif __has_builtin(__builtin_clzl)
363 else if (
sizeof(
long) * CHAR_BIT == 64) {
364 return (
unsigned int)__builtin_clzl((
unsigned long)x);
366 else if (
sizeof(
long long) * CHAR_BIT == 64) {
367 return (
unsigned int)__builtin_clzll((
unsigned long long)x);
377 y = x >> 32;
if (y) {n -= 32; x = y;}
378 y = x >> 16;
if (y) {n -= 16; x = y;}
379 y = x >> 8;
if (y) {n -= 8; x = y;}
380 y = x >> 4;
if (y) {n -= 4; x = y;}
381 y = x >> 2;
if (y) {n -= 2; x = y;}
382 y = x >> 1;
if (y) {
return n - 2;}
383 return (
unsigned int)(n - x);
389static inline unsigned int
390nlz_int128(uint128_t x)
392 uint64_t y = (uint64_t)(x >> 64);
398 return (
unsigned int)nlz_int64(x) + 64;
401 return (
unsigned int)nlz_int64(y);
406static inline unsigned int
407nlz_int(
unsigned int x)
409 if (
sizeof(
unsigned int) * CHAR_BIT == 32) {
410 return nlz_int32((uint32_t)x);
412 else if (
sizeof(
unsigned int) * CHAR_BIT == 64) {
413 return nlz_int64((uint64_t)x);
420static inline unsigned int
421nlz_long(
unsigned long x)
423 if (
sizeof(
unsigned long) * CHAR_BIT == 32) {
424 return nlz_int32((uint32_t)x);
426 else if (
sizeof(
unsigned long) * CHAR_BIT == 64) {
427 return nlz_int64((uint64_t)x);
434static inline unsigned int
435nlz_long_long(
unsigned long long x)
437 if (
sizeof(
unsigned long long) * CHAR_BIT == 64) {
438 return nlz_int64((uint64_t)x);
441 else if (
sizeof(
unsigned long long) * CHAR_BIT == 128) {
442 return nlz_int128((uint128_t)x);
450static inline unsigned int
451nlz_intptr(uintptr_t x)
453 if (
sizeof(uintptr_t) ==
sizeof(
unsigned int)) {
454 return nlz_int((
unsigned int)x);
456 if (
sizeof(uintptr_t) ==
sizeof(
unsigned long)) {
457 return nlz_long((
unsigned long)x);
459 if (
sizeof(uintptr_t) ==
sizeof(
unsigned long long)) {
460 return nlz_long_long((
unsigned long long)x);
467static inline unsigned int
468rb_popcount32(uint32_t x)
470#if defined(_MSC_VER) && defined(__AVX__)
474 return (
unsigned int)__popcnt(x);
476#elif __has_builtin(__builtin_popcount)
477 STATIC_ASSERT(sizeof_int,
sizeof(
int) * CHAR_BIT >= 32);
478 return (
unsigned int)__builtin_popcount(x);
481 x = (x & 0x55555555) + (x >> 1 & 0x55555555);
482 x = (x & 0x33333333) + (x >> 2 & 0x33333333);
483 x = (x & 0x07070707) + (x >> 4 & 0x07070707);
484 x = (x & 0x000f000f) + (x >> 8 & 0x000f000f);
485 x = (x & 0x0000001f) + (x >>16 & 0x0000001f);
486 return (
unsigned int)x;
491static inline unsigned int
492rb_popcount64(uint64_t x)
494#if defined(_MSC_VER) && defined(__AVX__)
495 return (
unsigned int)__popcnt64(x);
497#elif __has_builtin(__builtin_popcount)
498 if (
sizeof(
long) * CHAR_BIT == 64) {
499 return (
unsigned int)__builtin_popcountl((
unsigned long)x);
501 else if (
sizeof(
long long) * CHAR_BIT == 64) {
502 return (
unsigned int)__builtin_popcountll((
unsigned long long)x);
510 x = (x & 0x5555555555555555) + (x >> 1 & 0x5555555555555555);
511 x = (x & 0x3333333333333333) + (x >> 2 & 0x3333333333333333);
512 x = (x & 0x0707070707070707) + (x >> 4 & 0x0707070707070707);
513 x = (x & 0x000f000f000f000f) + (x >> 8 & 0x000f000f000f000f);
514 x = (x & 0x0000001f0000001f) + (x >>16 & 0x0000001f0000001f);
515 x = (x & 0x000000000000003f) + (x >>32 & 0x000000000000003f);
516 return (
unsigned int)x;
521static inline unsigned int
522rb_popcount_intptr(uintptr_t x)
524 if (
sizeof(uintptr_t) * CHAR_BIT == 64) {
525 return rb_popcount64((uint64_t)x);
527 else if (
sizeof(uintptr_t) * CHAR_BIT == 32) {
528 return rb_popcount32((uint32_t)x);
538#if defined(__x86_64__) && defined(__BMI__)
539 return (
unsigned)_tzcnt_u32(x);
541#elif MSC_VERSION_SINCE(1400)
545 return _BitScanForward(&r, x) ? (int)r : 32;
547#elif __has_builtin(__builtin_ctz)
548 STATIC_ASSERT(sizeof_int,
sizeof(
int) * CHAR_BIT == 32);
549 return x ? (unsigned)__builtin_ctz(x) : 32;
552 return rb_popcount32((~x) & (x-1));
560#if defined(__x86_64__) && defined(__BMI__)
561 return (
unsigned)_tzcnt_u64(x);
563#elif defined(_WIN64) && MSC_VERSION_SINCE(1400)
565 return _BitScanForward64(&r, x) ? (int)r : 64;
567#elif __has_builtin(__builtin_ctzl)
571 else if (
sizeof(
long) * CHAR_BIT == 64) {
572 return (
unsigned)__builtin_ctzl((
unsigned long)x);
574 else if (
sizeof(
long long) * CHAR_BIT == 64) {
575 return (
unsigned)__builtin_ctzll((
unsigned long long)x);
583 return rb_popcount64((~x) & (x-1));
589ntz_intptr(uintptr_t x)
591 if (
sizeof(uintptr_t) * CHAR_BIT == 64) {
592 return ntz_int64((uint64_t)x);
594 else if (
sizeof(uintptr_t) * CHAR_BIT == 32) {
595 return ntz_int32((uint32_t)x);
603RUBY_BIT_ROTL(
VALUE v,
int n)
605#if __has_builtin(__builtin_rotateleft32) && (SIZEOF_VALUE * CHAR_BIT == 32)
606 return __builtin_rotateleft32(v, n);
608#elif __has_builtin(__builtin_rotateleft64) && (SIZEOF_VALUE * CHAR_BIT == 64)
609 return __builtin_rotateleft64(v, n);
611#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 32)
614#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 64)
615 return _rotl64(v, n);
617#elif defined(_lrotl) && (SIZEOF_VALUE == SIZEOF_LONG)
621 const int m = (
sizeof(
VALUE) * CHAR_BIT) - 1;
622 return (v << (n & m)) | (v >> (-n & m));
627RUBY_BIT_ROTR(
VALUE v,
int n)
629#if __has_builtin(__builtin_rotateright32) && (SIZEOF_VALUE * CHAR_BIT == 32)
630 return __builtin_rotateright32(v, n);
632#elif __has_builtin(__builtin_rotateright64) && (SIZEOF_VALUE * CHAR_BIT == 64)
633 return __builtin_rotateright64(v, n);
635#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 32)
638#elif MSC_VERSION_SINCE(1310) && (SIZEOF_VALUE * CHAR_BIT == 64)
639 return _rotr64(v, n);
641#elif defined(_lrotr) && (SIZEOF_VALUE == SIZEOF_LONG)
645 const int m = (
sizeof(
VALUE) * CHAR_BIT) - 1;
646 return (v << (-n & m)) | (v >> (n & m));
#define UNREACHABLE_RETURN
Old name of RBIMPL_UNREACHABLE_RETURN.
uintptr_t VALUE
Type that represents a Ruby object.