diff options
Diffstat (limited to 'include/t1ha_bits.h')
| -rw-r--r-- | include/t1ha_bits.h | 1466 |
1 files changed, 817 insertions, 649 deletions
diff --git a/include/t1ha_bits.h b/include/t1ha_bits.h index 539369aa..e7a8d53c 100644 --- a/include/t1ha_bits.h +++ b/include/t1ha_bits.h @@ -44,30 +44,30 @@ #pragma once #if defined(_MSC_VER) -#pragma warning(disable : 4201) /* nameless struct/union */ -#if _MSC_VER > 1800 -#pragma warning(disable : 4464) /* relative include path contains '..' */ -#endif /* 1800 */ -#endif /* MSVC */ + #pragma warning(disable : 4201) /* nameless struct/union */ + #if _MSC_VER > 1800 + #pragma warning(disable : 4464) /* relative include path contains '..' */ + #endif /* 1800 */ +#endif /* MSVC */ #include "t1ha.h" #ifndef T1HA_USE_FAST_ONESHOT_READ -/* Define it to 1 for little bit faster code. - * Unfortunately this may triggering a false-positive alarms from Valgrind, - * AddressSanitizer and other similar tool. - * So, define it to 0 for calmness if doubt. */ -#define T1HA_USE_FAST_ONESHOT_READ 1 -#endif /* T1HA_USE_FAST_ONESHOT_READ */ + /* Define it to 1 for little bit faster code. + * Unfortunately this may triggering a false-positive alarms from Valgrind, + * AddressSanitizer and other similar tool. + * So, define it to 0 for calmness if doubt. */ + #define T1HA_USE_FAST_ONESHOT_READ 1 +#endif /* T1HA_USE_FAST_ONESHOT_READ */ /*****************************************************************************/ -#include <assert.h> /* for assert() */ -#include <stdbool.h> /* for bool */ -#include <string.h> /* for memcpy() */ +#include <assert.h> /* for assert() */ +#include <stdbool.h> /* for bool */ +#include <string.h> /* for memcpy() */ -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \ +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \ __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ -#error Unsupported byte order. + #error Unsupported byte order. #endif #define T1HA_UNALIGNED_ACCESS__UNABLE 0 @@ -75,534 +75,600 @@ #define T1HA_UNALIGNED_ACCESS__EFFICIENT 2 #ifndef T1HA_SYS_UNALIGNED_ACCESS -#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) -#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT -#elif defined(__ia32__) -#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT -#elif defined(__e2k__) -#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__SLOW -#elif defined(__ARM_FEATURE_UNALIGNED) -#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT -#else -#define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__UNABLE -#endif -#endif /* T1HA_SYS_UNALIGNED_ACCESS */ + #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT + #elif defined(__ia32__) + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT + #elif defined(__e2k__) + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__SLOW + #elif defined(__ARM_FEATURE_UNALIGNED) + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__EFFICIENT + #else + #define T1HA_SYS_UNALIGNED_ACCESS T1HA_UNALIGNED_ACCESS__UNABLE + #endif +#endif /* T1HA_SYS_UNALIGNED_ACCESS */ #define ALIGNMENT_16 2 #define ALIGNMENT_32 4 #if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul -#define ALIGNMENT_64 8 + #define ALIGNMENT_64 8 #else -#define ALIGNMENT_64 4 + #define ALIGNMENT_64 4 #endif #ifndef PAGESIZE -#define PAGESIZE 4096 -#endif /* PAGESIZE */ + #define PAGESIZE 4096 +#endif /* PAGESIZE */ /***************************************************************************/ #ifndef __has_builtin -#define __has_builtin(x) (0) + #define __has_builtin(x) (0) #endif #ifndef __has_warning -#define __has_warning(x) (0) + #define __has_warning(x) (0) #endif #ifndef __has_feature -#define __has_feature(x) (0) + #define __has_feature(x) (0) #endif #ifndef __has_extension -#define __has_extension(x) (0) + #define __has_extension(x) (0) #endif #if __has_feature(address_sanitizer) -#define __SANITIZE_ADDRESS__ 1 + #define __SANITIZE_ADDRESS__ 1 #endif #ifndef __optimize -#if defined(__clang__) && !__has_attribute(__optimize__) -#define __optimize(ops) -#elif defined(__GNUC__) || __has_attribute(__optimize__) -#define __optimize(ops) __attribute__((__optimize__(ops))) -#else -#define __optimize(ops) -#endif -#endif /* __optimize */ + #if defined(__clang__) && !__has_attribute(__optimize__) + #define __optimize(ops) + #elif defined(__GNUC__) || __has_attribute(__optimize__) + #define __optimize(ops) __attribute__((__optimize__(ops))) + #else + #define __optimize(ops) + #endif +#endif /* __optimize */ #ifndef __cold -#if defined(__OPTIMIZE__) -#if defined(__e2k__) -#define __cold __optimize(1) __attribute__((__cold__)) -#elif defined(__clang__) && !__has_attribute(__cold__) && \ - __has_attribute(__section__) -/* just put infrequently used functions in separate section */ -#define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") -#elif defined(__GNUC__) || __has_attribute(__cold__) -#define __cold __attribute__((__cold__)) __optimize("Os") -#else -#define __cold __optimize("Os") -#endif -#else -#define __cold -#endif -#endif /* __cold */ + #if defined(__OPTIMIZE__) + #if defined(__e2k__) + #define __cold __optimize(1) __attribute__((__cold__)) + #elif defined(__clang__) && !__has_attribute(__cold__) && \ + __has_attribute(__section__) + /* just put infrequently used functions in separate section */ + #define __cold \ + __attribute__((__section__("text.unlikely"))) __optimize("Os") + #elif defined(__GNUC__) || __has_attribute(__cold__) + #define __cold __attribute__((__cold__)) __optimize("Os") + #else + #define __cold __optimize("Os") + #endif + #else + #define __cold + #endif +#endif /* __cold */ #if __GNUC_PREREQ(4, 4) || defined(__clang__) -#if defined(__ia32__) || defined(__e2k__) -#include <x86intrin.h> -#endif + #if defined(__ia32__) || defined(__e2k__) + #include <x86intrin.h> + #endif -#if defined(__ia32__) && !defined(__cpuid_count) -#include <cpuid.h> -#endif + #if defined(__ia32__) && !defined(__cpuid_count) + #include <cpuid.h> + #endif -#if defined(__e2k__) -#include <e2kbuiltin.h> -#endif + #if defined(__e2k__) + #include <e2kbuiltin.h> + #endif -#ifndef likely -#define likely(cond) __builtin_expect(!!(cond), 1) -#endif + #ifndef likely + #define likely(cond) __builtin_expect(!!(cond), 1) + #endif -#ifndef unlikely -#define unlikely(cond) __builtin_expect(!!(cond), 0) -#endif + #ifndef unlikely + #define unlikely(cond) __builtin_expect(!!(cond), 0) + #endif -#if __GNUC_PREREQ(4, 5) || __has_builtin(__builtin_unreachable) -#define unreachable() __builtin_unreachable() -#endif + #if __GNUC_PREREQ(4, 5) || __has_builtin(__builtin_unreachable) + #define unreachable() __builtin_unreachable() + #endif -#define bswap64(v) __builtin_bswap64(v) -#define bswap32(v) __builtin_bswap32(v) -#if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16) -#define bswap16(v) __builtin_bswap16(v) -#endif + #define bswap64(v) __builtin_bswap64(v) + #define bswap32(v) __builtin_bswap32(v) + #if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16) + #define bswap16(v) __builtin_bswap16(v) + #endif -#if !defined(__maybe_unused) && \ - (__GNUC_PREREQ(4, 3) || __has_attribute(__unused__)) -#define __maybe_unused __attribute__((__unused__)) -#endif + #if !defined(__maybe_unused) && \ + (__GNUC_PREREQ(4, 3) || __has_attribute(__unused__)) + #define __maybe_unused __attribute__((__unused__)) + #endif -#if !defined(__always_inline) && \ - (__GNUC_PREREQ(3, 2) || __has_attribute(__always_inline__)) -#define __always_inline __inline __attribute__((__always_inline__)) -#endif + #if !defined(__always_inline) && \ + (__GNUC_PREREQ(3, 2) || __has_attribute(__always_inline__)) + #define __always_inline __inline __attribute__((__always_inline__)) + #endif -#if defined(__e2k__) + #if defined(__e2k__) -#if __iset__ >= 3 -#define mul_64x64_high(a, b) __builtin_e2k_umulhd(a, b) -#endif /* __iset__ >= 3 */ + #if __iset__ >= 3 + #define mul_64x64_high(a, b) __builtin_e2k_umulhd(a, b) + #endif /* __iset__ >= 3 */ + + #if __iset__ >= 5 +static __maybe_unused __always_inline unsigned e2k_add64carry_first( + uint64_t base, uint64_t addend, uint64_t *sum) { -#if __iset__ >= 5 -static __maybe_unused __always_inline unsigned -e2k_add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) { *sum = base + addend; return (unsigned)__builtin_e2k_addcd_c(base, addend, 0); + } -#define add64carry_first(base, addend, sum) \ - e2k_add64carry_first(base, addend, sum) +\ + #define add64carry_first(base, addend, sum) \ + e2k_add64carry_first(base, addend, sum) + +static __maybe_unused __always_inline unsigned e2k_add64carry_next( + unsigned carry, uint64_t base, uint64_t addend, uint64_t *sum) { -static __maybe_unused __always_inline unsigned -e2k_add64carry_next(unsigned carry, uint64_t base, uint64_t addend, - uint64_t *sum) { *sum = __builtin_e2k_addcd(base, addend, carry); return (unsigned)__builtin_e2k_addcd_c(base, addend, carry); + } -#define add64carry_next(carry, base, addend, sum) \ - e2k_add64carry_next(carry, base, addend, sum) +\ + #define add64carry_next(carry, base, addend, sum) \ + e2k_add64carry_next(carry, base, addend, sum) -static __maybe_unused __always_inline void e2k_add64carry_last(unsigned carry, - uint64_t base, - uint64_t addend, +static __maybe_unused __always_inline void e2k_add64carry_last(unsigned carry, + uint64_t base, + uint64_t addend, uint64_t *sum) { + *sum = __builtin_e2k_addcd(base, addend, carry); + } -#define add64carry_last(carry, base, addend, sum) \ - e2k_add64carry_last(carry, base, addend, sum) -#endif /* __iset__ >= 5 */ +\ + #define add64carry_last(carry, base, addend, sum) \ + e2k_add64carry_last(carry, base, addend, sum) + #endif /* __iset__ >= 5 */ -#define fetch64_be_aligned(ptr) ((uint64_t)__builtin_e2k_ld_64s_be(ptr)) -#define fetch32_be_aligned(ptr) ((uint32_t)__builtin_e2k_ld_32u_be(ptr)) + #define fetch64_be_aligned(ptr) ((uint64_t)__builtin_e2k_ld_64s_be(ptr)) + #define fetch32_be_aligned(ptr) ((uint32_t)__builtin_e2k_ld_32u_be(ptr)) -#endif /* __e2k__ Elbrus */ + #endif /* __e2k__ Elbrus */ #elif defined(_MSC_VER) -#if _MSC_FULL_VER < 190024234 && defined(_M_IX86) -#pragma message( \ - "For AES-NI at least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required.") -#endif -#if _MSC_FULL_VER < 191526730 -#pragma message( \ - "It is recommended to use \"Microsoft C/C++ Compiler\" version 19.15.26730 (Visual Studio 2017 15.8) or newer.") -#endif -#if _MSC_FULL_VER < 180040629 -#error At least "Microsoft C/C++ Compiler" version 18.00.40629 (Visual Studio 2013 Update 5) is required. -#endif - -#pragma warning(push, 1) - -#include <intrin.h> -#include <stdlib.h> -#define likely(cond) (cond) -#define unlikely(cond) (cond) -#define unreachable() __assume(0) -#define bswap64(v) _byteswap_uint64(v) -#define bswap32(v) _byteswap_ulong(v) -#define bswap16(v) _byteswap_ushort(v) -#define rot64(v, s) _rotr64(v, s) -#define rot32(v, s) _rotr(v, s) -#define __always_inline __forceinline - -#if defined(_M_X64) || defined(_M_IA64) -#pragma intrinsic(_umul128) -#define mul_64x64_128(a, b, ph) _umul128(a, b, ph) -#pragma intrinsic(_addcarry_u64) -#define add64carry_first(base, addend, sum) _addcarry_u64(0, base, addend, sum) -#define add64carry_next(carry, base, addend, sum) \ - _addcarry_u64(carry, base, addend, sum) -#define add64carry_last(carry, base, addend, sum) \ - (void)_addcarry_u64(carry, base, addend, sum) -#endif - -#if defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) -#pragma intrinsic(__umulh) -#define mul_64x64_high(a, b) __umulh(a, b) -#endif - -#if defined(_M_IX86) -#pragma intrinsic(__emulu) -#define mul_32x32_64(a, b) __emulu(a, b) + #if _MSC_FULL_VER < 190024234 && defined(_M_IX86) + #pragma message( \ + "For AES-NI at least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required.") + #endif + #if _MSC_FULL_VER < 191526730 + #pragma message( \ + "It is recommended to use \"Microsoft C/C++ Compiler\" version 19.15.26730 (Visual Studio 2017 15.8) or newer.") + #endif + #if _MSC_FULL_VER < 180040629 + #error At least "Microsoft C/C++ Compiler" version 18.00.40629 (Visual Studio 2013 Update 5) is required. + #endif + + #pragma warning(push, 1) + + #include <intrin.h> + #include <stdlib.h> + #define likely(cond) (cond) + #define unlikely(cond) (cond) + #define unreachable() __assume(0) + #define bswap64(v) _byteswap_uint64(v) + #define bswap32(v) _byteswap_ulong(v) + #define bswap16(v) _byteswap_ushort(v) + #define rot64(v, s) _rotr64(v, s) + #define rot32(v, s) _rotr(v, s) + #define __always_inline __forceinline + + #if defined(_M_X64) || defined(_M_IA64) + #pragma intrinsic(_umul128) + #define mul_64x64_128(a, b, ph) _umul128(a, b, ph) + #pragma intrinsic(_addcarry_u64) + #define add64carry_first(base, addend, sum) \ + _addcarry_u64(0, base, addend, sum) + #define add64carry_next(carry, base, addend, sum) \ + _addcarry_u64(carry, base, addend, sum) + #define add64carry_last(carry, base, addend, sum) \ + (void)_addcarry_u64(carry, base, addend, sum) + #endif + + #if defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) + #pragma intrinsic(__umulh) + #define mul_64x64_high(a, b) __umulh(a, b) + #endif + + #if defined(_M_IX86) + #pragma intrinsic(__emulu) + #define mul_32x32_64(a, b) __emulu(a, b) + + #if _MSC_VER >= 1915 /* LY: workaround for SSA-optimizer bug */ + #pragma intrinsic(_addcarry_u32) + #define add32carry_first(base, addend, sum) \ + _addcarry_u32(0, base, addend, sum) + #define add32carry_next(carry, base, addend, sum) \ + _addcarry_u32(carry, base, addend, sum) + #define add32carry_last(carry, base, addend, sum) \ + (void)_addcarry_u32(carry, base, addend, sum) + +static __forceinline char msvc32_add64carry_first(uint64_t base, + uint64_t addend, + uint64_t *sum) { -#if _MSC_VER >= 1915 /* LY: workaround for SSA-optimizer bug */ -#pragma intrinsic(_addcarry_u32) -#define add32carry_first(base, addend, sum) _addcarry_u32(0, base, addend, sum) -#define add32carry_next(carry, base, addend, sum) \ - _addcarry_u32(carry, base, addend, sum) -#define add32carry_last(carry, base, addend, sum) \ - (void)_addcarry_u32(carry, base, addend, sum) - -static __forceinline char -msvc32_add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) { uint32_t *const sum32 = (uint32_t *)sum; - const uint32_t base_32l = (uint32_t)base; - const uint32_t base_32h = (uint32_t)(base >> 32); - const uint32_t addend_32l = (uint32_t)addend; - const uint32_t addend_32h = (uint32_t)(addend >> 32); + const uint32_t base_32l = (uint32_t)base; + const uint32_t base_32h = (uint32_t)(base >> 32); + const uint32_t addend_32l = (uint32_t)addend; + const uint32_t addend_32h = (uint32_t)(addend >> 32); return add32carry_next(add32carry_first(base_32l, addend_32l, sum32), base_32h, addend_32h, sum32 + 1); + } -#define add64carry_first(base, addend, sum) \ - msvc32_add64carry_first(base, addend, sum) +\ + #define add64carry_first(base, addend, sum) \ + msvc32_add64carry_first(base, addend, sum) static __forceinline char msvc32_add64carry_next(char carry, uint64_t base, - uint64_t addend, + uint64_t addend, uint64_t *sum) { + uint32_t *const sum32 = (uint32_t *)sum; - const uint32_t base_32l = (uint32_t)base; - const uint32_t base_32h = (uint32_t)(base >> 32); - const uint32_t addend_32l = (uint32_t)addend; - const uint32_t addend_32h = (uint32_t)(addend >> 32); + const uint32_t base_32l = (uint32_t)base; + const uint32_t base_32h = (uint32_t)(base >> 32); + const uint32_t addend_32l = (uint32_t)addend; + const uint32_t addend_32h = (uint32_t)(addend >> 32); return add32carry_next(add32carry_next(carry, base_32l, addend_32l, sum32), base_32h, addend_32h, sum32 + 1); + } -#define add64carry_next(carry, base, addend, sum) \ - msvc32_add64carry_next(carry, base, addend, sum) +\ + #define add64carry_next(carry, base, addend, sum) \ + msvc32_add64carry_next(carry, base, addend, sum) static __forceinline void msvc32_add64carry_last(char carry, uint64_t base, - uint64_t addend, + uint64_t addend, uint64_t *sum) { + uint32_t *const sum32 = (uint32_t *)sum; - const uint32_t base_32l = (uint32_t)base; - const uint32_t base_32h = (uint32_t)(base >> 32); - const uint32_t addend_32l = (uint32_t)addend; - const uint32_t addend_32h = (uint32_t)(addend >> 32); + const uint32_t base_32l = (uint32_t)base; + const uint32_t base_32h = (uint32_t)(base >> 32); + const uint32_t addend_32l = (uint32_t)addend; + const uint32_t addend_32h = (uint32_t)(addend >> 32); add32carry_last(add32carry_next(carry, base_32l, addend_32l, sum32), base_32h, addend_32h, sum32 + 1); -} -#define add64carry_last(carry, base, addend, sum) \ - msvc32_add64carry_last(carry, base, addend, sum) -#endif /* _MSC_FULL_VER >= 190024231 */ - -#elif defined(_M_ARM) -#define mul_32x32_64(a, b) _arm_umull(a, b) -#endif -#pragma warning(pop) -#pragma warning(disable : 4514) /* 'xyz': unreferenced inline function \ - has been removed */ -#pragma warning(disable : 4710) /* 'xyz': function not inlined */ -#pragma warning(disable : 4711) /* function 'xyz' selected for \ - automatic inline expansion */ -#pragma warning(disable : 4127) /* conditional expression is constant */ -#pragma warning(disable : 4702) /* unreachable code */ -#endif /* Compiler */ +} +\ + #define add64carry_last(carry, base, addend, sum) \ + msvc32_add64carry_last(carry, base, addend, sum) + #endif /* _MSC_FULL_VER >= 190024231 */ + + #elif defined(_M_ARM) + #define mul_32x32_64(a, b) _arm_umull(a, b) + #endif + + #pragma warning(pop) + #pragma warning(disable : 4514) /* 'xyz': unreferenced inline function \ + has been removed */ + #pragma warning(disable : 4710) /* 'xyz': function not inlined */ + #pragma warning(disable : 4711) /* function 'xyz' selected for \ + automatic inline expansion */ + #pragma warning(disable : 4127) /* conditional expression is constant */ + #pragma warning(disable : 4702) /* unreachable code */ +#endif /* Compiler */ #ifndef likely -#define likely(cond) (cond) + #define likely(cond) (cond) #endif #ifndef unlikely -#define unlikely(cond) (cond) + #define unlikely(cond) (cond) #endif #ifndef __maybe_unused -#define __maybe_unused + #define __maybe_unused #endif #ifndef __always_inline -#define __always_inline __inline + #define __always_inline __inline #endif #ifndef unreachable -#define unreachable() \ - do { \ - } while (1) + #define unreachable() \ + do { \ + \ + } while (1) #endif #ifndef bswap64 -#if defined(bswap_64) -#define bswap64 bswap_64 -#elif defined(__bswap_64) -#define bswap64 __bswap_64 -#else + #if defined(bswap_64) + #define bswap64 bswap_64 + #elif defined(__bswap_64) + #define bswap64 __bswap_64 + #else static __always_inline uint64_t bswap64(uint64_t v) { + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | ((v << 24) & UINT64_C(0x0000ff0000000000)) | ((v << 8) & UINT64_C(0x000000ff00000000)) | ((v >> 8) & UINT64_C(0x00000000ff000000)) | ((v >> 24) & UINT64_C(0x0000000000ff0000)) | ((v >> 40) & UINT64_C(0x000000000000ff00)); + } -#endif -#endif /* bswap64 */ + + #endif +#endif /* bswap64 */ #ifndef bswap32 -#if defined(bswap_32) -#define bswap32 bswap_32 -#elif defined(__bswap_32) -#define bswap32 __bswap_32 -#else + #if defined(bswap_32) + #define bswap32 bswap_32 + #elif defined(__bswap_32) + #define bswap32 __bswap_32 + #else static __always_inline uint32_t bswap32(uint32_t v) { + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | ((v >> 8) & UINT32_C(0x0000ff00)); + } -#endif -#endif /* bswap32 */ + + #endif +#endif /* bswap32 */ #ifndef bswap16 -#if defined(bswap_16) -#define bswap16 bswap_16 -#elif defined(__bswap_16) -#define bswap16 __bswap_16 -#else -static __always_inline uint16_t bswap16(uint16_t v) { return v << 8 | v >> 8; } -#endif -#endif /* bswap16 */ + #if defined(bswap_16) + #define bswap16 bswap_16 + #elif defined(__bswap_16) + #define bswap16 __bswap_16 + #else +static __always_inline uint16_t bswap16(uint16_t v) { + + return v << 8 | v >> 8; + +} + + #endif +#endif /* bswap16 */ -#if defined(__ia32__) || \ +#if defined(__ia32__) || \ T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT -/* The __builtin_assume_aligned() leads gcc/clang to load values into the - * registers, even when it is possible to directly use an operand from memory. - * This can lead to a shortage of registers and a significant slowdown. - * Therefore avoid unnecessary use of __builtin_assume_aligned() for x86. */ -#define read_unaligned(ptr, bits) (*(const uint##bits##_t *__restrict)(ptr)) -#define read_aligned(ptr, bits) (*(const uint##bits##_t *__restrict)(ptr)) -#endif /* __ia32__ */ + /* The __builtin_assume_aligned() leads gcc/clang to load values into the + * registers, even when it is possible to directly use an operand from memory. + * This can lead to a shortage of registers and a significant slowdown. + * Therefore avoid unnecessary use of __builtin_assume_aligned() for x86. */ + #define read_unaligned(ptr, bits) (*(const uint##bits##_t *__restrict)(ptr)) + #define read_aligned(ptr, bits) (*(const uint##bits##_t *__restrict)(ptr)) +#endif /* __ia32__ */ #ifndef read_unaligned -#if defined(__GNUC__) || __has_attribute(__packed__) + #if defined(__GNUC__) || __has_attribute(__packed__) typedef struct { - uint8_t unaligned_8; + + uint8_t unaligned_8; uint16_t unaligned_16; uint32_t unaligned_32; uint64_t unaligned_64; + } __attribute__((__packed__)) t1ha_unaligned_proxy; -#define read_unaligned(ptr, bits) \ - (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \ - t1ha_unaligned_proxy, unaligned_##bits))) \ - ->unaligned_##bits) -#elif defined(_MSC_VER) -#pragma warning( \ - disable : 4235) /* nonstandard extension used: '__unaligned' \ - * keyword not supported on this architecture */ -#define read_unaligned(ptr, bits) (*(const __unaligned uint##bits##_t *)(ptr)) -#else -#pragma pack(push, 1) +\ + #define read_unaligned(ptr, bits) \ + (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \ + t1ha_unaligned_proxy, unaligned_##bits))) \ + ->unaligned_##bits) + #elif defined(_MSC_VER) + #pragma warning( \ + disable : 4235) /* nonstandard extension used: '__unaligned' \ + * keyword not supported on this architecture */ + #define read_unaligned(ptr, bits) \ + (*(const __unaligned uint##bits##_t *)(ptr)) + #else + #pragma pack(push, 1) typedef struct { - uint8_t unaligned_8; + + uint8_t unaligned_8; uint16_t unaligned_16; uint32_t unaligned_32; uint64_t unaligned_64; + } t1ha_unaligned_proxy; -#pragma pack(pop) -#define read_unaligned(ptr, bits) \ - (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \ - t1ha_unaligned_proxy, unaligned_##bits))) \ - ->unaligned_##bits) -#endif -#endif /* read_unaligned */ + + #pragma pack(pop) + #define read_unaligned(ptr, bits) \ + (((const t1ha_unaligned_proxy *)((const uint8_t *)(ptr)-offsetof( \ + t1ha_unaligned_proxy, unaligned_##bits))) \ + ->unaligned_##bits) + #endif +#endif /* read_unaligned */ #ifndef read_aligned -#if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_assume_aligned) -#define read_aligned(ptr, bits) \ - (*(const uint##bits##_t *)__builtin_assume_aligned(ptr, ALIGNMENT_##bits)) -#elif (__GNUC_PREREQ(3, 3) || __has_attribute(__aligned__)) && \ - !defined(__clang__) -#define read_aligned(ptr, bits) \ - (*(const uint##bits##_t \ - __attribute__((__aligned__(ALIGNMENT_##bits))) *)(ptr)) -#elif __has_attribute(__assume_aligned__) - -static __always_inline const - uint16_t *__attribute__((__assume_aligned__(ALIGNMENT_16))) - cast_aligned_16(const void *ptr) { + #if __GNUC_PREREQ(4, 8) || __has_builtin(__builtin_assume_aligned) + #define read_aligned(ptr, bits) \ + (*(const uint##bits##_t *)__builtin_assume_aligned(ptr, ALIGNMENT_##bits)) + #elif (__GNUC_PREREQ(3, 3) || __has_attribute(__aligned__)) && \ + !defined(__clang__) + #define read_aligned(ptr, bits) \ + (*(const uint##bits##_t \ + __attribute__((__aligned__(ALIGNMENT_##bits))) *)(ptr)) + #elif __has_attribute(__assume_aligned__) + +static __always_inline const uint16_t *__attribute__(( + __assume_aligned__(ALIGNMENT_16))) cast_aligned_16(const void *ptr) { + return (const uint16_t *)ptr; + } -static __always_inline const - uint32_t *__attribute__((__assume_aligned__(ALIGNMENT_32))) - cast_aligned_32(const void *ptr) { + +static __always_inline const uint32_t *__attribute__(( + __assume_aligned__(ALIGNMENT_32))) cast_aligned_32(const void *ptr) { + return (const uint32_t *)ptr; + } -static __always_inline const - uint64_t *__attribute__((__assume_aligned__(ALIGNMENT_64))) - cast_aligned_64(const void *ptr) { + +static __always_inline const uint64_t *__attribute__(( + __assume_aligned__(ALIGNMENT_64))) cast_aligned_64(const void *ptr) { + return (const uint64_t *)ptr; + } -#define read_aligned(ptr, bits) (*cast_aligned_##bits(ptr)) + #define read_aligned(ptr, bits) (*cast_aligned_##bits(ptr)) -#elif defined(_MSC_VER) -#define read_aligned(ptr, bits) \ - (*(const __declspec(align(ALIGNMENT_##bits)) uint##bits##_t *)(ptr)) -#else -#define read_aligned(ptr, bits) (*(const uint##bits##_t *)(ptr)) -#endif -#endif /* read_aligned */ + #elif defined(_MSC_VER) + #define read_aligned(ptr, bits) \ + (*(const __declspec(align(ALIGNMENT_##bits)) uint##bits##_t *)(ptr)) + #else + #define read_aligned(ptr, bits) (*(const uint##bits##_t *)(ptr)) + #endif +#endif /* read_aligned */ #ifndef prefetch -#if (__GNUC_PREREQ(4, 0) || __has_builtin(__builtin_prefetch)) && \ - !defined(__ia32__) -#define prefetch(ptr) __builtin_prefetch(ptr) -#elif defined(_M_ARM64) || defined(_M_ARM) -#define prefetch(ptr) __prefetch(ptr) -#else -#define prefetch(ptr) \ - do { \ - (void)(ptr); \ - } while (0) -#endif -#endif /* prefetch */ + #if (__GNUC_PREREQ(4, 0) || __has_builtin(__builtin_prefetch)) && \ + !defined(__ia32__) + #define prefetch(ptr) __builtin_prefetch(ptr) + #elif defined(_M_ARM64) || defined(_M_ARM) + #define prefetch(ptr) __prefetch(ptr) + #else + #define prefetch(ptr) \ + do { \ + \ + (void)(ptr); \ + \ + } while (0) + #endif +#endif /* prefetch */ #if __has_warning("-Wconstant-logical-operand") -#if defined(__clang__) -#pragma clang diagnostic ignored "-Wconstant-logical-operand" -#elif defined(__GNUC__) -#pragma GCC diagnostic ignored "-Wconstant-logical-operand" -#else -#pragma warning disable "constant-logical-operand" -#endif -#endif /* -Wconstant-logical-operand */ + #if defined(__clang__) + #pragma clang diagnostic ignored "-Wconstant-logical-operand" + #elif defined(__GNUC__) + #pragma GCC diagnostic ignored "-Wconstant-logical-operand" + #else + #pragma warning disable "constant-logical-operand" + #endif +#endif /* -Wconstant-logical-operand */ #if __has_warning("-Wtautological-pointer-compare") -#if defined(__clang__) -#pragma clang diagnostic ignored "-Wtautological-pointer-compare" -#elif defined(__GNUC__) -#pragma GCC diagnostic ignored "-Wtautological-pointer-compare" -#else -#pragma warning disable "tautological-pointer-compare" -#endif -#endif /* -Wtautological-pointer-compare */ + #if defined(__clang__) + #pragma clang diagnostic ignored "-Wtautological-pointer-compare" + #elif defined(__GNUC__) + #pragma GCC diagnostic ignored "-Wtautological-pointer-compare" + #else + #pragma warning disable "tautological-pointer-compare" + #endif +#endif /* -Wtautological-pointer-compare */ /***************************************************************************/ #if __GNUC_PREREQ(4, 0) -#pragma GCC visibility push(hidden) -#endif /* __GNUC_PREREQ(4,0) */ + #pragma GCC visibility push(hidden) +#endif /* __GNUC_PREREQ(4,0) */ /*---------------------------------------------------------- Little Endian */ #ifndef fetch16_le_aligned static __maybe_unused __always_inline uint16_t fetch16_le_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_16 == 0); -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return read_aligned(v, 16); -#else + #else return bswap16(read_aligned(v, 16)); -#endif + #endif + } -#endif /* fetch16_le_aligned */ + +#endif /* fetch16_le_aligned */ #ifndef fetch16_le_unaligned static __maybe_unused __always_inline uint16_t fetch16_le_unaligned(const void *v) { -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE const uint8_t *p = (const uint8_t *)v; return p[0] | (uint16_t)p[1] << 8; -#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return read_unaligned(v, 16); -#else + #else return bswap16(read_unaligned(v, 16)); -#endif + #endif + } -#endif /* fetch16_le_unaligned */ + +#endif /* fetch16_le_unaligned */ #ifndef fetch32_le_aligned static __maybe_unused __always_inline uint32_t fetch32_le_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_32 == 0); -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return read_aligned(v, 32); -#else + #else return bswap32(read_aligned(v, 32)); -#endif + #endif + } -#endif /* fetch32_le_aligned */ + +#endif /* fetch32_le_aligned */ #ifndef fetch32_le_unaligned static __maybe_unused __always_inline uint32_t fetch32_le_unaligned(const void *v) { -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE return fetch16_le_unaligned(v) | (uint32_t)fetch16_le_unaligned((const uint8_t *)v + 2) << 16; -#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return read_unaligned(v, 32); -#else + #else return bswap32(read_unaligned(v, 32)); -#endif + #endif + } -#endif /* fetch32_le_unaligned */ + +#endif /* fetch32_le_unaligned */ #ifndef fetch64_le_aligned static __maybe_unused __always_inline uint64_t fetch64_le_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_64 == 0); -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return read_aligned(v, 64); -#else + #else return bswap64(read_aligned(v, 64)); -#endif + #endif + } -#endif /* fetch64_le_aligned */ + +#endif /* fetch64_le_aligned */ #ifndef fetch64_le_unaligned static __maybe_unused __always_inline uint64_t fetch64_le_unaligned(const void *v) { -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE return fetch32_le_unaligned(v) | (uint64_t)fetch32_le_unaligned((const uint8_t *)v + 4) << 32; -#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ return read_unaligned(v, 64); -#else + #else return bswap64(read_unaligned(v, 64)); -#endif + #endif + } -#endif /* fetch64_le_unaligned */ + +#endif /* fetch64_le_unaligned */ static __maybe_unused __always_inline uint64_t tail64_le_aligned(const void *v, size_t tail) { + const uint8_t *const p = (const uint8_t *)v; #if T1HA_USE_FAST_ONESHOT_READ && !defined(__SANITIZE_ADDRESS__) /* We can perform a 'oneshot' read, which is little bit faster. */ @@ -611,79 +677,84 @@ static __maybe_unused __always_inline uint64_t tail64_le_aligned(const void *v, #else uint64_t r = 0; switch (tail & 7) { - default: - unreachable(); -/* fall through */ -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - /* For most CPUs this code is better when not needed byte reordering. */ - case 0: - return fetch64_le_aligned(p); - case 7: - r = (uint64_t)p[6] << 8; - /* fall through */ - case 6: - r += p[5]; - r <<= 8; - /* fall through */ - case 5: - r += p[4]; - r <<= 32; - /* fall through */ - case 4: - return r + fetch32_le_aligned(p); - case 3: - r = (uint64_t)p[2] << 16; - /* fall through */ - case 2: - return r + fetch16_le_aligned(p); - case 1: - return p[0]; -#else - case 0: - r = p[7] << 8; - /* fall through */ - case 7: - r += p[6]; - r <<= 8; - /* fall through */ - case 6: - r += p[5]; - r <<= 8; - /* fall through */ - case 5: - r += p[4]; - r <<= 8; - /* fall through */ - case 4: - r += p[3]; - r <<= 8; - /* fall through */ - case 3: - r += p[2]; - r <<= 8; - /* fall through */ - case 2: - r += p[1]; - r <<= 8; + + default: + unreachable(); /* fall through */ - case 1: - return r + p[0]; -#endif + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + /* For most CPUs this code is better when not needed byte reordering. */ + case 0: + return fetch64_le_aligned(p); + case 7: + r = (uint64_t)p[6] << 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 32; + /* fall through */ + case 4: + return r + fetch32_le_aligned(p); + case 3: + r = (uint64_t)p[2] << 16; + /* fall through */ + case 2: + return r + fetch16_le_aligned(p); + case 1: + return p[0]; + #else + case 0: + r = p[7] << 8; + /* fall through */ + case 7: + r += p[6]; + r <<= 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 8; + /* fall through */ + case 4: + r += p[3]; + r <<= 8; + /* fall through */ + case 3: + r += p[2]; + r <<= 8; + /* fall through */ + case 2: + r += p[1]; + r <<= 8; + /* fall through */ + case 1: + return r + p[0]; + #endif + } -#endif /* T1HA_USE_FAST_ONESHOT_READ */ + +#endif /* T1HA_USE_FAST_ONESHOT_READ */ + } -#if T1HA_USE_FAST_ONESHOT_READ && \ - T1HA_SYS_UNALIGNED_ACCESS != T1HA_UNALIGNED_ACCESS__UNABLE && \ +#if T1HA_USE_FAST_ONESHOT_READ && \ + T1HA_SYS_UNALIGNED_ACCESS != T1HA_UNALIGNED_ACCESS__UNABLE && \ defined(PAGESIZE) && PAGESIZE > 42 && !defined(__SANITIZE_ADDRESS__) -#define can_read_underside(ptr, size) \ - (((PAGESIZE - (size)) & (uintptr_t)(ptr)) != 0) -#endif /* T1HA_USE_FAST_ONESHOT_READ */ + #define can_read_underside(ptr, size) \ + (((PAGESIZE - (size)) & (uintptr_t)(ptr)) != 0) +#endif /* T1HA_USE_FAST_ONESHOT_READ */ static __maybe_unused __always_inline uint64_t tail64_le_unaligned(const void *v, size_t tail) { + const uint8_t *p = (const uint8_t *)v; -#if defined(can_read_underside) && \ +#if defined(can_read_underside) && \ (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) /* On some systems (e.g. x86_64) we can perform a 'oneshot' read, which * is little bit faster. Thanks Marcin Żukowski <marcin.zukowski@gmail.com> @@ -691,77 +762,84 @@ tail64_le_unaligned(const void *v, size_t tail) { const unsigned offset = (8 - tail) & 7; const unsigned shift = offset << 3; if (likely(can_read_underside(p, 8))) { + p -= offset; return fetch64_le_unaligned(p) >> shift; + } + return fetch64_le_unaligned(p) & ((~UINT64_C(0)) >> shift); #else uint64_t r = 0; switch (tail & 7) { - default: - unreachable(); -/* fall through */ -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT && \ - __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - /* For most CPUs this code is better when not needed - * copying for alignment or byte reordering. */ - case 0: - return fetch64_le_unaligned(p); - case 7: - r = (uint64_t)p[6] << 8; - /* fall through */ - case 6: - r += p[5]; - r <<= 8; - /* fall through */ - case 5: - r += p[4]; - r <<= 32; - /* fall through */ - case 4: - return r + fetch32_le_unaligned(p); - case 3: - r = (uint64_t)p[2] << 16; - /* fall through */ - case 2: - return r + fetch16_le_unaligned(p); - case 1: - return p[0]; -#else - /* For most CPUs this code is better than a - * copying for alignment and/or byte reordering. */ - case 0: - r = p[7] << 8; - /* fall through */ - case 7: - r += p[6]; - r <<= 8; - /* fall through */ - case 6: - r += p[5]; - r <<= 8; - /* fall through */ - case 5: - r += p[4]; - r <<= 8; - /* fall through */ - case 4: - r += p[3]; - r <<= 8; - /* fall through */ - case 3: - r += p[2]; - r <<= 8; - /* fall through */ - case 2: - r += p[1]; - r <<= 8; + + default: + unreachable(); /* fall through */ - case 1: - return r + p[0]; -#endif + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT && \ + __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + /* For most CPUs this code is better when not needed + * copying for alignment or byte reordering. */ + case 0: + return fetch64_le_unaligned(p); + case 7: + r = (uint64_t)p[6] << 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 32; + /* fall through */ + case 4: + return r + fetch32_le_unaligned(p); + case 3: + r = (uint64_t)p[2] << 16; + /* fall through */ + case 2: + return r + fetch16_le_unaligned(p); + case 1: + return p[0]; + #else + /* For most CPUs this code is better than a + * copying for alignment and/or byte reordering. */ + case 0: + r = p[7] << 8; + /* fall through */ + case 7: + r += p[6]; + r <<= 8; + /* fall through */ + case 6: + r += p[5]; + r <<= 8; + /* fall through */ + case 5: + r += p[4]; + r <<= 8; + /* fall through */ + case 4: + r += p[3]; + r <<= 8; + /* fall through */ + case 3: + r += p[2]; + r <<= 8; + /* fall through */ + case 2: + r += p[1]; + r <<= 8; + /* fall through */ + case 1: + return r + p[0]; + #endif + } -#endif /* can_read_underside */ + +#endif /* can_read_underside */ + } /*------------------------------------------------------------- Big Endian */ @@ -769,83 +847,102 @@ tail64_le_unaligned(const void *v, size_t tail) { #ifndef fetch16_be_aligned static __maybe_unused __always_inline uint16_t fetch16_be_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_16 == 0); -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return read_aligned(v, 16); -#else + #else return bswap16(read_aligned(v, 16)); -#endif + #endif + } -#endif /* fetch16_be_aligned */ + +#endif /* fetch16_be_aligned */ #ifndef fetch16_be_unaligned static __maybe_unused __always_inline uint16_t fetch16_be_unaligned(const void *v) { -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE const uint8_t *p = (const uint8_t *)v; return (uint16_t)p[0] << 8 | p[1]; -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return read_unaligned(v, 16); -#else + #else return bswap16(read_unaligned(v, 16)); -#endif + #endif + } -#endif /* fetch16_be_unaligned */ + +#endif /* fetch16_be_unaligned */ #ifndef fetch32_be_aligned static __maybe_unused __always_inline uint32_t fetch32_be_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_32 == 0); -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return read_aligned(v, 32); -#else + #else return bswap32(read_aligned(v, 32)); -#endif + #endif + } -#endif /* fetch32_be_aligned */ + +#endif /* fetch32_be_aligned */ #ifndef fetch32_be_unaligned static __maybe_unused __always_inline uint32_t fetch32_be_unaligned(const void *v) { -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE return (uint32_t)fetch16_be_unaligned(v) << 16 | fetch16_be_unaligned((const uint8_t *)v + 2); -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return read_unaligned(v, 32); -#else + #else return bswap32(read_unaligned(v, 32)); -#endif + #endif + } -#endif /* fetch32_be_unaligned */ + +#endif /* fetch32_be_unaligned */ #ifndef fetch64_be_aligned static __maybe_unused __always_inline uint64_t fetch64_be_aligned(const void *v) { + assert(((uintptr_t)v) % ALIGNMENT_64 == 0); -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return read_aligned(v, 64); -#else + #else return bswap64(read_aligned(v, 64)); -#endif + #endif + } -#endif /* fetch64_be_aligned */ + +#endif /* fetch64_be_aligned */ #ifndef fetch64_be_unaligned static __maybe_unused __always_inline uint64_t fetch64_be_unaligned(const void *v) { -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE + + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__UNABLE return (uint64_t)fetch32_be_unaligned(v) << 32 | fetch32_be_unaligned((const uint8_t *)v + 4); -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ return read_unaligned(v, 64); -#else + #else return bswap64(read_unaligned(v, 64)); -#endif + #endif + } -#endif /* fetch64_be_unaligned */ + +#endif /* fetch64_be_unaligned */ static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v, size_t tail) { + const uint8_t *const p = (const uint8_t *)v; #if T1HA_USE_FAST_ONESHOT_READ && !defined(__SANITIZE_ADDRESS__) /* We can perform a 'oneshot' read, which is little bit faster. */ @@ -853,61 +950,66 @@ static __maybe_unused __always_inline uint64_t tail64_be_aligned(const void *v, return fetch64_be_aligned(p) >> shift; #else switch (tail & 7) { - default: - unreachable(); -/* fall through */ -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - /* For most CPUs this code is better when not byte reordering. */ - case 1: - return p[0]; - case 2: - return fetch16_be_aligned(p); - case 3: - return (uint32_t)fetch16_be_aligned(p) << 8 | p[2]; - case 4: - return fetch32_be_aligned(p); - case 5: - return (uint64_t)fetch32_be_aligned(p) << 8 | p[4]; - case 6: - return (uint64_t)fetch32_be_aligned(p) << 16 | fetch16_be_aligned(p + 4); - case 7: - return (uint64_t)fetch32_be_aligned(p) << 24 | - (uint32_t)fetch16_be_aligned(p + 4) << 8 | p[6]; - case 0: - return fetch64_be_aligned(p); -#else - case 1: - return p[0]; - case 2: - return p[1] | (uint32_t)p[0] << 8; - case 3: - return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16; - case 4: - return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 | - (uint32_t)p[0] << 24; - case 5: - return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 | - (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32; - case 6: - return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 | - (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40; - case 7: - return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 | - (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | (uint64_t)p[1] << 40 | - (uint64_t)p[0] << 48; - case 0: - return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 | - (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | (uint64_t)p[2] << 40 | - (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56; -#endif + + default: + unreachable(); + /* fall through */ + #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + /* For most CPUs this code is better when not byte reordering. */ + case 1: + return p[0]; + case 2: + return fetch16_be_aligned(p); + case 3: + return (uint32_t)fetch16_be_aligned(p) << 8 | p[2]; + case 4: + return fetch32_be_aligned(p); + case 5: + return (uint64_t)fetch32_be_aligned(p) << 8 | p[4]; + case 6: + return (uint64_t)fetch32_be_aligned(p) << 16 | fetch16_be_aligned(p + 4); + case 7: + return (uint64_t)fetch32_be_aligned(p) << 24 | + (uint32_t)fetch16_be_aligned(p + 4) << 8 | p[6]; + case 0: + return fetch64_be_aligned(p); + #else + case 1: + return p[0]; + case 2: + return p[1] | (uint32_t)p[0] << 8; + case 3: + return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16; + case 4: + return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 | + (uint32_t)p[0] << 24; + case 5: + return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 | + (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32; + case 6: + return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 | + (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40; + case 7: + return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 | + (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | + (uint64_t)p[1] << 40 | (uint64_t)p[0] << 48; + case 0: + return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 | + (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | + (uint64_t)p[2] << 40 | (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56; + #endif + } -#endif /* T1HA_USE_FAST_ONESHOT_READ */ + +#endif /* T1HA_USE_FAST_ONESHOT_READ */ + } static __maybe_unused __always_inline uint64_t tail64_be_unaligned(const void *v, size_t tail) { + const uint8_t *p = (const uint8_t *)v; -#if defined(can_read_underside) && \ +#if defined(can_read_underside) && \ (UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul) /* On some systems (e.g. x86_64) we can perform a 'oneshot' read, which * is little bit faster. Thanks Marcin Żukowski <marcin.zukowski@gmail.com> @@ -915,139 +1017,167 @@ tail64_be_unaligned(const void *v, size_t tail) { const unsigned offset = (8 - tail) & 7; const unsigned shift = offset << 3; if (likely(can_read_underside(p, 8))) { + p -= offset; return fetch64_be_unaligned(p) & ((~UINT64_C(0)) >> shift); + } + return fetch64_be_unaligned(p) >> shift; #else switch (tail & 7) { - default: - unreachable(); -/* fall through */ -#if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT && \ - __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - /* For most CPUs this code is better when not needed - * copying for alignment or byte reordering. */ - case 1: - return p[0]; - case 2: - return fetch16_be_unaligned(p); - case 3: - return (uint32_t)fetch16_be_unaligned(p) << 8 | p[2]; - case 4: - return fetch32_be(p); - case 5: - return (uint64_t)fetch32_be_unaligned(p) << 8 | p[4]; - case 6: - return (uint64_t)fetch32_be_unaligned(p) << 16 | - fetch16_be_unaligned(p + 4); - case 7: - return (uint64_t)fetch32_be_unaligned(p) << 24 | - (uint32_t)fetch16_be_unaligned(p + 4) << 8 | p[6]; - case 0: - return fetch64_be_unaligned(p); -#else - /* For most CPUs this code is better than a - * copying for alignment and/or byte reordering. */ - case 1: - return p[0]; - case 2: - return p[1] | (uint32_t)p[0] << 8; - case 3: - return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16; - case 4: - return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 | - (uint32_t)p[0] << 24; - case 5: - return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 | - (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32; - case 6: - return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 | - (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40; - case 7: - return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 | - (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | (uint64_t)p[1] << 40 | - (uint64_t)p[0] << 48; - case 0: - return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 | - (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | (uint64_t)p[2] << 40 | - (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56; -#endif + + default: + unreachable(); + /* fall through */ + #if T1HA_SYS_UNALIGNED_ACCESS == T1HA_UNALIGNED_ACCESS__EFFICIENT && \ + __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + /* For most CPUs this code is better when not needed + * copying for alignment or byte reordering. */ + case 1: + return p[0]; + case 2: + return fetch16_be_unaligned(p); + case 3: + return (uint32_t)fetch16_be_unaligned(p) << 8 | p[2]; + case 4: + return fetch32_be(p); + case 5: + return (uint64_t)fetch32_be_unaligned(p) << 8 | p[4]; + case 6: + return (uint64_t)fetch32_be_unaligned(p) << 16 | + fetch16_be_unaligned(p + 4); + case 7: + return (uint64_t)fetch32_be_unaligned(p) << 24 | + (uint32_t)fetch16_be_unaligned(p + 4) << 8 | p[6]; + case 0: + return fetch64_be_unaligned(p); + #else + /* For most CPUs this code is better than a + * copying for alignment and/or byte reordering. */ + case 1: + return p[0]; + case 2: + return p[1] | (uint32_t)p[0] << 8; + case 3: + return p[2] | (uint32_t)p[1] << 8 | (uint32_t)p[0] << 16; + case 4: + return p[3] | (uint32_t)p[2] << 8 | (uint32_t)p[1] << 16 | + (uint32_t)p[0] << 24; + case 5: + return p[4] | (uint32_t)p[3] << 8 | (uint32_t)p[2] << 16 | + (uint32_t)p[1] << 24 | (uint64_t)p[0] << 32; + case 6: + return p[5] | (uint32_t)p[4] << 8 | (uint32_t)p[3] << 16 | + (uint32_t)p[2] << 24 | (uint64_t)p[1] << 32 | (uint64_t)p[0] << 40; + case 7: + return p[6] | (uint32_t)p[5] << 8 | (uint32_t)p[4] << 16 | + (uint32_t)p[3] << 24 | (uint64_t)p[2] << 32 | + (uint64_t)p[1] << 40 | (uint64_t)p[0] << 48; + case 0: + return p[7] | (uint32_t)p[6] << 8 | (uint32_t)p[5] << 16 | + (uint32_t)p[4] << 24 | (uint64_t)p[3] << 32 | + (uint64_t)p[2] << 40 | (uint64_t)p[1] << 48 | (uint64_t)p[0] << 56; + #endif + } -#endif /* can_read_underside */ + +#endif /* can_read_underside */ + } /***************************************************************************/ #ifndef rot64 static __maybe_unused __always_inline uint64_t rot64(uint64_t v, unsigned s) { + return (v >> s) | (v << (64 - s)); + } -#endif /* rot64 */ + +#endif /* rot64 */ #ifndef mul_32x32_64 static __maybe_unused __always_inline uint64_t mul_32x32_64(uint32_t a, uint32_t b) { + return a * (uint64_t)b; + } -#endif /* mul_32x32_64 */ + +#endif /* mul_32x32_64 */ #ifndef add64carry_first -static __maybe_unused __always_inline unsigned -add64carry_first(uint64_t base, uint64_t addend, uint64_t *sum) { -#if __has_builtin(__builtin_addcll) +static __maybe_unused __always_inline unsigned add64carry_first(uint64_t base, + uint64_t addend, + uint64_t *sum) { + + #if __has_builtin(__builtin_addcll) unsigned long long carryout; *sum = __builtin_addcll(base, addend, 0, &carryout); return (unsigned)carryout; -#else + #else *sum = base + addend; return *sum < addend; -#endif /* __has_builtin(__builtin_addcll) */ + #endif /* __has_builtin(__builtin_addcll) */ + } -#endif /* add64carry_fist */ + +#endif /* add64carry_fist */ #ifndef add64carry_next -static __maybe_unused __always_inline unsigned -add64carry_next(unsigned carry, uint64_t base, uint64_t addend, uint64_t *sum) { -#if __has_builtin(__builtin_addcll) +static __maybe_unused __always_inline unsigned add64carry_next(unsigned carry, + uint64_t base, + uint64_t addend, + uint64_t *sum) { + + #if __has_builtin(__builtin_addcll) unsigned long long carryout; *sum = __builtin_addcll(base, addend, carry, &carryout); return (unsigned)carryout; -#else + #else *sum = base + addend + carry; return *sum < addend || (carry && *sum == addend); -#endif /* __has_builtin(__builtin_addcll) */ + #endif /* __has_builtin(__builtin_addcll) */ + } -#endif /* add64carry_next */ + +#endif /* add64carry_next */ #ifndef add64carry_last -static __maybe_unused __always_inline void -add64carry_last(unsigned carry, uint64_t base, uint64_t addend, uint64_t *sum) { -#if __has_builtin(__builtin_addcll) +static __maybe_unused __always_inline void add64carry_last(unsigned carry, + uint64_t base, + uint64_t addend, + uint64_t *sum) { + + #if __has_builtin(__builtin_addcll) unsigned long long carryout; *sum = __builtin_addcll(base, addend, carry, &carryout); (void)carryout; -#else + #else *sum = base + addend + carry; -#endif /* __has_builtin(__builtin_addcll) */ + #endif /* __has_builtin(__builtin_addcll) */ + } -#endif /* add64carry_last */ + +#endif /* add64carry_last */ #ifndef mul_64x64_128 -static __maybe_unused __always_inline uint64_t mul_64x64_128(uint64_t a, - uint64_t b, +static __maybe_unused __always_inline uint64_t mul_64x64_128(uint64_t a, + uint64_t b, uint64_t *h) { -#if (defined(__SIZEOF_INT128__) || \ - (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)) && \ - (!defined(__LCC__) || __LCC__ != 124) + + #if (defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)) && \ + (!defined(__LCC__) || __LCC__ != 124) __uint128_t r = (__uint128_t)a * (__uint128_t)b; /* modern GCC could nicely optimize this */ *h = (uint64_t)(r >> 64); return (uint64_t)r; -#elif defined(mul_64x64_high) + #elif defined(mul_64x64_high) *h = mul_64x64_high(a, b); return a * b; -#else + #else /* performs 64x64 to 128 bit multiplication */ const uint64_t ll = mul_32x32_64((uint32_t)a, (uint32_t)b); const uint64_t lh = mul_32x32_64(a >> 32, (uint32_t)b); @@ -1062,18 +1192,23 @@ static __maybe_unused __always_inline uint64_t mul_64x64_128(uint64_t a, add64carry_last(add64carry_first(ll, lh << 32, &l), hh, lh >> 32, h); add64carry_last(add64carry_first(l, hl << 32, &l), *h, hl >> 32, h); return l; -#endif + #endif + } -#endif /* mul_64x64_128() */ + +#endif /* mul_64x64_128() */ #ifndef mul_64x64_high static __maybe_unused __always_inline uint64_t mul_64x64_high(uint64_t a, uint64_t b) { + uint64_t h; mul_64x64_128(a, b, &h); return h; + } -#endif /* mul_64x64_high */ + +#endif /* mul_64x64_high */ /***************************************************************************/ @@ -1089,45 +1224,56 @@ static const uint64_t prime_6 = UINT64_C(0xCB5AF53AE3AAAC31); /* xor high and low parts of full 128-bit product */ static __maybe_unused __always_inline uint64_t mux64(uint64_t v, uint64_t prime) { + uint64_t l, h; l = mul_64x64_128(v, prime, &h); return l ^ h; + } static __maybe_unused __always_inline uint64_t final64(uint64_t a, uint64_t b) { + uint64_t x = (a + rot64(b, 41)) * prime_0; uint64_t y = (rot64(a, 23) + b) * prime_6; return mux64(x ^ y, prime_5); + } static __maybe_unused __always_inline void mixup64(uint64_t *__restrict a, uint64_t *__restrict b, uint64_t v, uint64_t prime) { + uint64_t h; *a ^= mul_64x64_128(*b + v, prime, &h); *b += h; + } /***************************************************************************/ typedef union t1ha_uint128 { -#if defined(__SIZEOF_INT128__) || \ + +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) __uint128_t v; #endif struct { + #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ uint64_t l, h; #else uint64_t h, l; #endif + }; + } t1ha_uint128_t; static __maybe_unused __always_inline t1ha_uint128_t not128(const t1ha_uint128_t v) { + t1ha_uint128_t r; -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) r.v = ~v.v; #else @@ -1135,13 +1281,15 @@ not128(const t1ha_uint128_t v) { r.h = ~v.h; #endif return r; + } static __maybe_unused __always_inline t1ha_uint128_t left128(const t1ha_uint128_t v, unsigned s) { + t1ha_uint128_t r; assert(s < 128); -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) r.v = v.v << s; #else @@ -1149,13 +1297,15 @@ left128(const t1ha_uint128_t v, unsigned s) { r.h = (s < 64) ? (v.h << s) | (s ? v.l >> (64 - s) : 0) : v.l << (s - 64); #endif return r; + } static __maybe_unused __always_inline t1ha_uint128_t right128(const t1ha_uint128_t v, unsigned s) { + t1ha_uint128_t r; assert(s < 128); -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) r.v = v.v >> s; #else @@ -1163,12 +1313,14 @@ right128(const t1ha_uint128_t v, unsigned s) { r.h = (s < 64) ? v.h >> s : 0; #endif return r; + } static __maybe_unused __always_inline t1ha_uint128_t or128(t1ha_uint128_t x, t1ha_uint128_t y) { + t1ha_uint128_t r; -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) r.v = x.v | y.v; #else @@ -1176,12 +1328,14 @@ static __maybe_unused __always_inline t1ha_uint128_t or128(t1ha_uint128_t x, r.h = x.h | y.h; #endif return r; + } static __maybe_unused __always_inline t1ha_uint128_t xor128(t1ha_uint128_t x, t1ha_uint128_t y) { + t1ha_uint128_t r; -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) r.v = x.v ^ y.v; #else @@ -1189,36 +1343,42 @@ static __maybe_unused __always_inline t1ha_uint128_t xor128(t1ha_uint128_t x, r.h = x.h ^ y.h; #endif return r; + } static __maybe_unused __always_inline t1ha_uint128_t rot128(t1ha_uint128_t v, - unsigned s) { + unsigned s) { + s &= 127; -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) v.v = (v.v << (128 - s)) | (v.v >> s); return v; #else return s ? or128(left128(v, 128 - s), right128(v, s)) : v; #endif + } static __maybe_unused __always_inline t1ha_uint128_t add128(t1ha_uint128_t x, t1ha_uint128_t y) { + t1ha_uint128_t r; -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) r.v = x.v + y.v; #else add64carry_last(add64carry_first(x.l, y.l, &r.l), x.h, y.h, &r.h); #endif return r; + } static __maybe_unused __always_inline t1ha_uint128_t mul128(t1ha_uint128_t x, t1ha_uint128_t y) { + t1ha_uint128_t r; -#if defined(__SIZEOF_INT128__) || \ +#if defined(__SIZEOF_INT128__) || \ (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) r.v = x.v * y.v; #else @@ -1226,6 +1386,7 @@ static __maybe_unused __always_inline t1ha_uint128_t mul128(t1ha_uint128_t x, r.h += x.l * y.h + y.l * x.h; #endif return r; + } /***************************************************************************/ @@ -1233,22 +1394,29 @@ static __maybe_unused __always_inline t1ha_uint128_t mul128(t1ha_uint128_t x, #if T1HA0_AESNI_AVAILABLE && defined(__ia32__) uint64_t t1ha_ia32cpu_features(void); -static __maybe_unused __always_inline bool -t1ha_ia32_AESNI_avail(uint64_t ia32cpu_features) { +static __maybe_unused __always_inline bool t1ha_ia32_AESNI_avail( + uint64_t ia32cpu_features) { + /* check for AES-NI */ return (ia32cpu_features & UINT32_C(0x02000000)) != 0; + } -static __maybe_unused __always_inline bool -t1ha_ia32_AVX_avail(uint64_t ia32cpu_features) { +static __maybe_unused __always_inline bool t1ha_ia32_AVX_avail( + uint64_t ia32cpu_features) { + /* check for any AVX */ return (ia32cpu_features & UINT32_C(0x1A000000)) == UINT32_C(0x1A000000); + } -static __maybe_unused __always_inline bool -t1ha_ia32_AVX2_avail(uint64_t ia32cpu_features) { +static __maybe_unused __always_inline bool t1ha_ia32_AVX2_avail( + uint64_t ia32cpu_features) { + /* check for 'Advanced Vector Extensions 2' */ return ((ia32cpu_features >> 32) & 32) != 0; + } -#endif /* T1HA0_AESNI_AVAILABLE && __ia32__ */ +#endif /* T1HA0_AESNI_AVAILABLE && __ia32__ */ + |