#ifndef __Vec_Intrin_H #define __Vec_Intrin_H #include <sys/types.h> /* We include config.h here to ensure that the various feature-flags are * properly brought into scope. Users can either run the configure script, or * write a config.h themselves and put it under version control. */ #if defined(__has_include) #if __has_include("config.h") #include "config.h" #endif #endif /* # DEBUGGING: * ============ * It is possible to debug the current definitions by using libintvector_debug.h * See the include at the bottom of the file. */ #define Lib_IntVector_Intrinsics_bit_mask64(x) … #if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) #if defined(HACL_CAN_COMPILE_VEC128) #include <emmintrin.h> #include <tmmintrin.h> #include <smmintrin.h> typedef __m128i Lib_IntVector_Intrinsics_vec128; #define Lib_IntVector_Intrinsics_ni_aes_enc … #define Lib_IntVector_Intrinsics_ni_aes_enc_last … #define Lib_IntVector_Intrinsics_ni_aes_keygen_assist … #define Lib_IntVector_Intrinsics_ni_clmul … #define Lib_IntVector_Intrinsics_vec128_xor … #define Lib_IntVector_Intrinsics_vec128_eq64 … #define Lib_IntVector_Intrinsics_vec128_eq32 … #define Lib_IntVector_Intrinsics_vec128_gt64 … #define Lib_IntVector_Intrinsics_vec128_gt32 … #define Lib_IntVector_Intrinsics_vec128_or … #define Lib_IntVector_Intrinsics_vec128_and … #define Lib_IntVector_Intrinsics_vec128_lognot … #define Lib_IntVector_Intrinsics_vec128_shift_left … #define Lib_IntVector_Intrinsics_vec128_shift_right … #define Lib_IntVector_Intrinsics_vec128_shift_left64 … #define Lib_IntVector_Intrinsics_vec128_shift_right64 … #define Lib_IntVector_Intrinsics_vec128_shift_left32 … #define Lib_IntVector_Intrinsics_vec128_shift_right32 … #define Lib_IntVector_Intrinsics_vec128_rotate_left32_8 … #define Lib_IntVector_Intrinsics_vec128_rotate_left32_16 … #define Lib_IntVector_Intrinsics_vec128_rotate_left32_24 … #define Lib_IntVector_Intrinsics_vec128_rotate_left32 … #define Lib_IntVector_Intrinsics_vec128_rotate_right32 … #define Lib_IntVector_Intrinsics_vec128_shuffle32 … #define Lib_IntVector_Intrinsics_vec128_shuffle64 … #define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32 … #define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes64 … #define Lib_IntVector_Intrinsics_vec128_load32_le … #define Lib_IntVector_Intrinsics_vec128_load64_le … #define Lib_IntVector_Intrinsics_vec128_store32_le … #define Lib_IntVector_Intrinsics_vec128_store64_le … #define Lib_IntVector_Intrinsics_vec128_load_be … #define Lib_IntVector_Intrinsics_vec128_load32_be … #define Lib_IntVector_Intrinsics_vec128_load64_be … #define Lib_IntVector_Intrinsics_vec128_store_be … #define Lib_IntVector_Intrinsics_vec128_store32_be … #define Lib_IntVector_Intrinsics_vec128_store64_be … #define Lib_IntVector_Intrinsics_vec128_insert8 … #define Lib_IntVector_Intrinsics_vec128_insert32 … #define Lib_IntVector_Intrinsics_vec128_insert64 … #define Lib_IntVector_Intrinsics_vec128_extract8 … #define Lib_IntVector_Intrinsics_vec128_extract32 … #define Lib_IntVector_Intrinsics_vec128_extract64 … #define Lib_IntVector_Intrinsics_vec128_zero … #define Lib_IntVector_Intrinsics_vec128_add64 … #define Lib_IntVector_Intrinsics_vec128_sub64 … #define Lib_IntVector_Intrinsics_vec128_mul64 … #define Lib_IntVector_Intrinsics_vec128_smul64 … #define Lib_IntVector_Intrinsics_vec128_add32 … #define Lib_IntVector_Intrinsics_vec128_sub32 … #define Lib_IntVector_Intrinsics_vec128_mul32 … #define Lib_IntVector_Intrinsics_vec128_smul32 … #define Lib_IntVector_Intrinsics_vec128_load128 … #define Lib_IntVector_Intrinsics_vec128_load64 … #define Lib_IntVector_Intrinsics_vec128_load64s … #define Lib_IntVector_Intrinsics_vec128_load32 … #define Lib_IntVector_Intrinsics_vec128_load32s … #define Lib_IntVector_Intrinsics_vec128_interleave_low32 … #define Lib_IntVector_Intrinsics_vec128_interleave_high32 … #define Lib_IntVector_Intrinsics_vec128_interleave_low64 … #define Lib_IntVector_Intrinsics_vec128_interleave_high64 … #endif /* HACL_CAN_COMPILE_VEC128 */ #if defined(HACL_CAN_COMPILE_VEC256) #include <immintrin.h> Lib_IntVector_Intrinsics_vec256; #define Lib_IntVector_Intrinsics_vec256_eq64(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_eq32(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_gt64(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_gt32(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_xor(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_or(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_and(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_lognot(x0) … #define Lib_IntVector_Intrinsics_vec256_shift_left(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_shift_right(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_shift_left64(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_shift_right64(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_shift_left32(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_shift_right32(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_rotate_left32_8(x0) … #define Lib_IntVector_Intrinsics_vec256_rotate_left32_16(x0) … #define Lib_IntVector_Intrinsics_vec256_rotate_left32_24(x0) … #define Lib_IntVector_Intrinsics_vec256_rotate_left32(x0,x1) … #define Lib_IntVector_Intrinsics_vec256_rotate_right32(x0,x1) … #define Lib_IntVector_Intrinsics_vec256_rotate_right64_8(x0) … #define Lib_IntVector_Intrinsics_vec256_rotate_right64_16(x0) … #define Lib_IntVector_Intrinsics_vec256_rotate_right64_24(x0) … #define Lib_IntVector_Intrinsics_vec256_rotate_right64_32(x0) … #define Lib_IntVector_Intrinsics_vec256_rotate_right64_40(x0) … #define Lib_IntVector_Intrinsics_vec256_rotate_right64_48(x0) … #define Lib_IntVector_Intrinsics_vec256_rotate_right64_56(x0) … #define Lib_IntVector_Intrinsics_vec256_rotate_right64(x0,x1) … #define Lib_IntVector_Intrinsics_vec256_rotate_left64(x0,x1) … #define Lib_IntVector_Intrinsics_vec256_shuffle64(x0, x1, x2, x3, x4) … #define Lib_IntVector_Intrinsics_vec256_shuffle32(x0, x1, x2, x3, x4, x5, x6, x7, x8) … #define Lib_IntVector_Intrinsics_vec256_rotate_right_lanes32(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_load32_le(x0) … #define Lib_IntVector_Intrinsics_vec256_load64_le(x0) … #define Lib_IntVector_Intrinsics_vec256_load32_be(x0) … #define Lib_IntVector_Intrinsics_vec256_load64_be(x0) … #define Lib_IntVector_Intrinsics_vec256_store32_le(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_store64_le(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_store32_be(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_store64_be(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_insert8(x0, x1, x2) … #define Lib_IntVector_Intrinsics_vec256_insert32(x0, x1, x2) … #define Lib_IntVector_Intrinsics_vec256_insert64(x0, x1, x2) … #define Lib_IntVector_Intrinsics_vec256_extract8(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_extract32(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_extract64(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_zero … #define Lib_IntVector_Intrinsics_vec256_add64(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_sub64(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_mul64(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_smul64(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_add32(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_sub32(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_mul32(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_smul32(x0, x1) … #define Lib_IntVector_Intrinsics_vec256_load64(x1) … #define Lib_IntVector_Intrinsics_vec256_load64s(x0, x1, x2, x3) … #define Lib_IntVector_Intrinsics_vec256_load32(x) … #define Lib_IntVector_Intrinsics_vec256_load32s(x0,x1,x2,x3,x4, x5, x6, x7) … #define Lib_IntVector_Intrinsics_vec256_load128(x) … #define Lib_IntVector_Intrinsics_vec256_load128s(x0,x1) … #define Lib_IntVector_Intrinsics_vec256_interleave_low32(x1, x2) … #define Lib_IntVector_Intrinsics_vec256_interleave_high32(x1, x2) … #define Lib_IntVector_Intrinsics_vec256_interleave_low64(x1, x2) … #define Lib_IntVector_Intrinsics_vec256_interleave_high64(x1, x2) … #define Lib_IntVector_Intrinsics_vec256_interleave_low128(x1, x2) … #define Lib_IntVector_Intrinsics_vec256_interleave_high128(x1, x2) … #endif /* HACL_CAN_COMPILE_VEC256 */ #elif (defined(__aarch64__) || defined(_M_ARM64) || defined(__arm__) || defined(_M_ARM)) \ && !defined(__ARM_32BIT_STATE) #if defined(HACL_CAN_COMPILE_VEC128) #include <arm_neon.h> typedef uint32x4_t Lib_IntVector_Intrinsics_vec128; #define Lib_IntVector_Intrinsics_vec128_xor … #define Lib_IntVector_Intrinsics_vec128_eq64 … #define Lib_IntVector_Intrinsics_vec128_eq32 … #define Lib_IntVector_Intrinsics_vec128_gt32 … #define high32 … #define low32 … #define Lib_IntVector_Intrinsics_vec128_gt64 … #define Lib_IntVector_Intrinsics_vec128_or … #define Lib_IntVector_Intrinsics_vec128_and … #define Lib_IntVector_Intrinsics_vec128_lognot … #define Lib_IntVector_Intrinsics_vec128_shift_left … #define Lib_IntVector_Intrinsics_vec128_shift_right … #define Lib_IntVector_Intrinsics_vec128_shift_left64 … #define Lib_IntVector_Intrinsics_vec128_shift_right64 … #define Lib_IntVector_Intrinsics_vec128_shift_left32 … #define Lib_IntVector_Intrinsics_vec128_shift_right32 … #define Lib_IntVector_Intrinsics_vec128_rotate_left32_16 … #define Lib_IntVector_Intrinsics_vec128_rotate_left32 … #define Lib_IntVector_Intrinsics_vec128_rotate_right32_16 … #define Lib_IntVector_Intrinsics_vec128_rotate_right32 … #define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32 … #define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes64 … /* #define Lib_IntVector_Intrinsics_vec128_shuffle32(x0, x1, x2, x3, x4) \ (_mm_shuffle_epi32(x0, _MM_SHUFFLE(x1,x2,x3,x4))) #define Lib_IntVector_Intrinsics_vec128_shuffle64(x0, x1, x2) \ (_mm_shuffle_epi32(x0, _MM_SHUFFLE(2*x1+1,2*x1,2*x2+1,2*x2))) */ #define Lib_IntVector_Intrinsics_vec128_load32_le … #define Lib_IntVector_Intrinsics_vec128_load64_le … #define Lib_IntVector_Intrinsics_vec128_store32_le … #define Lib_IntVector_Intrinsics_vec128_store64_le … /* #define Lib_IntVector_Intrinsics_vec128_load_be(x0) \ ( Lib_IntVector_Intrinsics_vec128 l = vrev64q_u8(vld1q_u32((uint32_t*)(x0))); */ #define Lib_IntVector_Intrinsics_vec128_load32_be … #define Lib_IntVector_Intrinsics_vec128_load64_be … /* #define Lib_IntVector_Intrinsics_vec128_store_be(x0, x1) \ (_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)))) */ #define Lib_IntVector_Intrinsics_vec128_store32_be … #define Lib_IntVector_Intrinsics_vec128_store64_be … #define Lib_IntVector_Intrinsics_vec128_insert8 … #define Lib_IntVector_Intrinsics_vec128_insert32 … #define Lib_IntVector_Intrinsics_vec128_insert64 … #define Lib_IntVector_Intrinsics_vec128_extract8 … #define Lib_IntVector_Intrinsics_vec128_extract32 … #define Lib_IntVector_Intrinsics_vec128_extract64 … #define Lib_IntVector_Intrinsics_vec128_zero … #define Lib_IntVector_Intrinsics_vec128_add64 … #define Lib_IntVector_Intrinsics_vec128_sub64 … #define Lib_IntVector_Intrinsics_vec128_mul64 … #define Lib_IntVector_Intrinsics_vec128_smul64 … #define Lib_IntVector_Intrinsics_vec128_add32 … #define Lib_IntVector_Intrinsics_vec128_sub32 … #define Lib_IntVector_Intrinsics_vec128_mul32 … #define Lib_IntVector_Intrinsics_vec128_smul32 … #define Lib_IntVector_Intrinsics_vec128_load128 … #define Lib_IntVector_Intrinsics_vec128_load64 … #define Lib_IntVector_Intrinsics_vec128_load32 … static inline Lib_IntVector_Intrinsics_vec128 Lib_IntVector_Intrinsics_vec128_load64s(uint64_t x1, uint64_t x2){ const uint64_t a[2] = {x1,x2}; return vreinterpretq_u32_u64(vld1q_u64(a)); } static inline Lib_IntVector_Intrinsics_vec128 Lib_IntVector_Intrinsics_vec128_load32s(uint32_t x1, uint32_t x2, uint32_t x3, uint32_t x4){ const uint32_t a[4] = {x1,x2,x3,x4}; return vld1q_u32(a); } #define Lib_IntVector_Intrinsics_vec128_interleave_low32 … #define Lib_IntVector_Intrinsics_vec128_interleave_high32 … #define Lib_IntVector_Intrinsics_vec128_interleave_low64 … #define Lib_IntVector_Intrinsics_vec128_interleave_high64 … #endif /* HACL_CAN_COMPILE_VEC128 */ /* IBM z architecture */ #elif defined(__s390x__) /* this flag is for GCC only */ #if defined(HACL_CAN_COMPILE_VEC128) #include <stdint.h> #include <vecintrin.h> /* The main vector 128 type * We can't use uint8_t, uint32_t, uint64_t... instead of unsigned char, * unsigned int, unsigned long long: the compiler complains that the parameter * combination is invalid. */ typedef unsigned char vector128_8 __attribute__ ((vector_size(16))); typedef unsigned int vector128_32 __attribute__ ((vector_size(16))); typedef unsigned long long vector128_64 __attribute__ ((vector_size(16))); typedef vector128_8 Lib_IntVector_Intrinsics_vec128; typedef vector128_8 vector128; #define Lib_IntVector_Intrinsics_vec128_load32_le … #define Lib_IntVector_Intrinsics_vec128_load32_be … #define Lib_IntVector_Intrinsics_vec128_load64_le … static inline void Lib_IntVector_Intrinsics_vec128_store32_le(const uint8_t *x0, vector128 x1) { *((vector128_32*)x0) = vec_revb((vector128_32) x1); } static inline void Lib_IntVector_Intrinsics_vec128_store32_be(const uint8_t *x0, vector128 x1) { *((vector128_32*)x0) = (vector128_32) x1; } static inline void Lib_IntVector_Intrinsics_vec128_store64_le(const uint8_t *x0, vector128 x1) { *((vector128_64*)x0) = vec_revb((vector128_64) x1); } #define Lib_IntVector_Intrinsics_vec128_add32 … #define Lib_IntVector_Intrinsics_vec128_add64 … #define Lib_IntVector_Intrinsics_vec128_and … #define Lib_IntVector_Intrinsics_vec128_eq32 … #define Lib_IntVector_Intrinsics_vec128_eq64 … #define Lib_IntVector_Intrinsics_vec128_extract32 … #define Lib_IntVector_Intrinsics_vec128_extract64 … #define Lib_IntVector_Intrinsics_vec128_gt32 … #define Lib_IntVector_Intrinsics_vec128_gt64 … #define Lib_IntVector_Intrinsics_vec128_insert32 … #define Lib_IntVector_Intrinsics_vec128_insert64 … #define Lib_IntVector_Intrinsics_vec128_interleave_high32 … #define Lib_IntVector_Intrinsics_vec128_interleave_high64 … #define Lib_IntVector_Intrinsics_vec128_interleave_low32 … #define Lib_IntVector_Intrinsics_vec128_interleave_low64 … #define Lib_IntVector_Intrinsics_vec128_load32 … #define Lib_IntVector_Intrinsics_vec128_load32s … #define Lib_IntVector_Intrinsics_vec128_load64 … #define Lib_IntVector_Intrinsics_vec128_lognot … #define Lib_IntVector_Intrinsics_vec128_mul64 … #define Lib_IntVector_Intrinsics_vec128_or … #define Lib_IntVector_Intrinsics_vec128_rotate_left32 … #define Lib_IntVector_Intrinsics_vec128_rotate_right32 … #define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32 … #define Lib_IntVector_Intrinsics_vec128_shift_left64 … #define Lib_IntVector_Intrinsics_vec128_shift_right64 … #define Lib_IntVector_Intrinsics_vec128_shift_right32 … /* Doesn't work with vec_splat_u64 */ #define Lib_IntVector_Intrinsics_vec128_smul64 … #define Lib_IntVector_Intrinsics_vec128_sub64 … static inline vector128 Lib_IntVector_Intrinsics_vec128_xor(vector128 x0, vector128 x1) { return ((vector128)(vec_xor((vector128)(x0), (vector128)(x1)))); } #define Lib_IntVector_Intrinsics_vec128_zero … #endif /* HACL_CAN_COMPILE_VEC128 */ #elif defined(__powerpc64__) // PowerPC 64 - this flag is for GCC only #if defined(HACL_CAN_COMPILE_VEC128) #include <altivec.h> #include <string.h> // for memcpy #include <stdint.h> // The main vector 128 type // We can't use uint8_t, uint32_t, uint64_t... instead of unsigned char, // unsigned int, unsigned long long: the compiler complains that the parameter // combination is invalid. typedef vector unsigned char vector128_8; typedef vector unsigned int vector128_32; typedef vector unsigned long long vector128_64; typedef vector128_8 Lib_IntVector_Intrinsics_vec128; typedef vector128_8 vector128; #define Lib_IntVector_Intrinsics_vec128_load32_le … #define Lib_IntVector_Intrinsics_vec128_load64_le … #define Lib_IntVector_Intrinsics_vec128_store32_le … #define Lib_IntVector_Intrinsics_vec128_store64_le … #define Lib_IntVector_Intrinsics_vec128_add32 … #define Lib_IntVector_Intrinsics_vec128_add64 … #define Lib_IntVector_Intrinsics_vec128_and … #define Lib_IntVector_Intrinsics_vec128_eq32 … #define Lib_IntVector_Intrinsics_vec128_eq64 … #define Lib_IntVector_Intrinsics_vec128_extract32 … #define Lib_IntVector_Intrinsics_vec128_extract64 … #define Lib_IntVector_Intrinsics_vec128_gt32 … #define Lib_IntVector_Intrinsics_vec128_gt64 … #define Lib_IntVector_Intrinsics_vec128_insert32 … #define Lib_IntVector_Intrinsics_vec128_insert64 … #define Lib_IntVector_Intrinsics_vec128_interleave_high32 … #define Lib_IntVector_Intrinsics_vec128_interleave_high64 … #define Lib_IntVector_Intrinsics_vec128_interleave_low32 … #define Lib_IntVector_Intrinsics_vec128_interleave_low64 … #define Lib_IntVector_Intrinsics_vec128_load32 … #define Lib_IntVector_Intrinsics_vec128_load32s … #define Lib_IntVector_Intrinsics_vec128_load64 … #define Lib_IntVector_Intrinsics_vec128_lognot … #define Lib_IntVector_Intrinsics_vec128_mul64 … #define Lib_IntVector_Intrinsics_vec128_or … #define Lib_IntVector_Intrinsics_vec128_rotate_left32 … #define Lib_IntVector_Intrinsics_vec128_rotate_right32 … #define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32 … #define Lib_IntVector_Intrinsics_vec128_shift_left64 … #define Lib_IntVector_Intrinsics_vec128_shift_right64 … // Doesn't work with vec_splat_u64 #define Lib_IntVector_Intrinsics_vec128_smul64 … #define Lib_IntVector_Intrinsics_vec128_sub64 … #define Lib_IntVector_Intrinsics_vec128_xor … #define Lib_IntVector_Intrinsics_vec128_zero … #endif /* HACL_CAN_COMPILE_VEC128 */ #endif // PowerPC64 // DEBUGGING: // If libintvector_debug.h exists, use it to debug the current implementations. // Note that some flags must be enabled for the debugging to be effective: // see libintvector_debug.h for more details. #if defined(__has_include) #if __has_include("libintvector_debug.h") #include "libintvector_debug.h" #endif #endif #endif // __Vec_Intrin_H
Codebase Browser
cpython