simdx86_p.h (405051B)
1 // This file is part of Blend2D project <https://blend2d.com> 2 // 3 // See blend2d.h or LICENSE.md for license and copyright information 4 // SPDX-License-Identifier: Zlib 5 6 #ifndef BLEND2D_SIMD_SIMDX86_P_H_INCLUDED 7 #define BLEND2D_SIMD_SIMDX86_P_H_INCLUDED 8 9 #include "../simd/simdbase_p.h" 10 #include "../support/bitops_p.h" 11 #include "../support/intops_p.h" 12 #include "../support/memops_p.h" 13 #include "../tables/tables_p.h" 14 15 #if defined(_MSC_VER) 16 #include <intrin.h> 17 #endif 18 19 #if defined(BL_TARGET_OPT_SSE) 20 #include <xmmintrin.h> 21 #endif 22 23 #if defined(BL_TARGET_OPT_SSE2) 24 #include <emmintrin.h> 25 #endif 26 27 #if defined(BL_TARGET_OPT_SSE3) && !defined(_MSC_VER) 28 #include <pmmintrin.h> 29 #endif 30 31 #if defined(BL_TARGET_OPT_SSSE3) 32 #include <tmmintrin.h> 33 #endif 34 35 #if defined(BL_TARGET_OPT_SSE4_1) 36 #include <smmintrin.h> 37 #endif 38 39 #if defined(BL_TARGET_OPT_SSE4_2) 40 #include <nmmintrin.h> 41 #include <wmmintrin.h> 42 #endif 43 44 #if defined(BL_TARGET_OPT_AVX) || defined(BL_TARGET_OPT_AVX2) || defined(BL_TARGET_OPT_AVX512) 45 #include <immintrin.h> 46 #endif 47 48 #ifdef min 49 #undef min 50 #endif 51 52 #ifdef max 53 #undef max 54 #endif 55 56 #if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 57 #define BL_SIMD_MISSING_INTRINSICS 58 #endif 59 60 //! \cond INTERNAL 61 //! \addtogroup blend2d_internal 62 //! \{ 63 64 // The reason for doing so is to make the public interface using SIMD::Internal easier to read. 65 #define I Internal 66 67 // SIMD - Register Widths 68 // ====================== 69 70 #if defined(BL_TARGET_OPT_AVX512) 71 #define BL_SIMD_WIDTH_I 512 72 #define BL_SIMD_WIDTH_F 512 73 #define BL_SIMD_WIDTH_D 512 74 #elif defined(BL_TARGET_OPT_AVX2) 75 #define BL_SIMD_WIDTH_I 256 76 #define BL_SIMD_WIDTH_F 256 77 #define BL_SIMD_WIDTH_D 256 78 #elif defined(BL_TARGET_OPT_AVX) 79 #define BL_SIMD_WIDTH_I 128 80 #define BL_SIMD_WIDTH_F 256 81 #define BL_SIMD_WIDTH_D 256 82 #else 83 #define BL_SIMD_WIDTH_I 128 84 #define BL_SIMD_WIDTH_F 128 85 #define BL_SIMD_WIDTH_D 128 86 #endif 87 88 // SIMD - Features 89 // =============== 90 91 // Features describe the availability of some SIMD instructions that are not emulated if not available. 92 93 #if defined(BL_TARGET_OPT_AVX512) 94 #define BL_SIMD_FEATURE_TERNLOG 95 #endif // BL_TARGET_OPT_AVX512 96 97 #if defined(BL_TARGET_OPT_AVX2) 98 #define BL_SIMD_FEATURE_MOVW 99 #endif // BL_TARGET_OPT_AVX2 100 101 #if defined(BL_TARGET_OPT_SSE4_1) 102 #define BL_SIMD_FEATURE_BLEND_IMM 103 #endif // BL_TARGET_OPT_SSE4_1 104 105 #if defined(BL_TARGET_OPT_SSSE3) 106 #define BL_SIMD_FEATURE_SWIZZLEV_U8 107 #endif // BL_TARGET_OPT_SSSE3 108 109 // SIMD - Cost Tables 110 // ================== 111 112 // NOTE: Cost in general tells us how complex is to emulate the given instruction in terms of other instructions. 113 // In general it doesn't tell us the latency as that depends on a micro-architecture, which we don't specialize 114 // for here. So, if the cost is 1 it means a single instruction can do the operation, if it's 3 it means that 3 115 // instructions are needed. The instruction cost doesn't include moves, which would only be present when targeting 116 // pre-AVX hardware. 117 118 #if defined(BL_TARGET_OPT_SSE2) 119 #define BL_SIMD_COST_MIN_MAX_U8 1 // native 120 #define BL_SIMD_COST_MIN_MAX_I16 1 // native 121 #define BL_SIMD_COST_MUL_I16 1 // native 122 #endif 123 124 #if defined(BL_TARGET_OPT_SSSE3) 125 #define BL_SIMD_COST_ABS_I8 1 // native 126 #define BL_SIMD_COST_ABS_I16 1 // native 127 #define BL_SIMD_COST_ABS_I32 1 // native 128 #define BL_SIMD_COST_ALIGNR_U8 1 // native 129 #else 130 #define BL_SIMD_COST_ABS_I8 2 // emulated ('sub_i8' + 'min_u8') 131 #define BL_SIMD_COST_ABS_I16 2 // emulated ('sub_i16' + 'max_i16') 132 #define BL_SIMD_COST_ABS_I32 3 // emulated ('srai_i32' + 'sub_i32' + 'xor') 133 #define BL_SIMD_COST_ALIGNR_U8 3 // emulated ('srlb' + 'sllb' + 'or') 134 #endif 135 136 #if defined(BL_TARGET_OPT_AVX512) 137 #define BL_SIMD_COST_ABS_I64 1 // native 138 #define BL_SIMD_COST_CMP_EQ_I64 1 // native 139 #define BL_SIMD_COST_CMP_LT_GT_I64 1 // native 140 #define BL_SIMD_COST_CMP_LE_GE_I64 1 // native 141 #define BL_SIMD_COST_CMP_LT_GT_U64 1 // native 142 #define BL_SIMD_COST_CMP_LE_GE_U64 1 // native 143 #define BL_SIMD_COST_MIN_MAX_I8 1 // native 144 #define BL_SIMD_COST_MIN_MAX_U16 1 // native 145 #define BL_SIMD_COST_MIN_MAX_I32 1 // native 146 #define BL_SIMD_COST_MIN_MAX_U32 1 // native 147 #define BL_SIMD_COST_MIN_MAX_I64 1 // native 148 #define BL_SIMD_COST_MIN_MAX_U64 1 // native 149 #define BL_SIMD_COST_MUL_I32 1 // native 150 #define BL_SIMD_COST_MUL_I64 1 // native 151 #elif defined(BL_TARGET_OPT_SSE4_2) 152 #define BL_SIMD_COST_ABS_I64 4 // emulated (complex) 153 #define BL_SIMD_COST_CMP_EQ_I64 1 // native 154 #define BL_SIMD_COST_CMP_LT_GT_I64 1 // native 155 #define BL_SIMD_COST_CMP_LE_GE_I64 2 // emulated ('cmp_lt_gt_i64' + 'not') 156 #define BL_SIMD_COST_CMP_LT_GT_U64 3 // emulated ('cmp_lt_gt_i64' + 2x'xor') 157 #define BL_SIMD_COST_CMP_LE_GE_U64 4 // emulated ('cmp_lt_gt_u64' + 'not') 158 #define BL_SIMD_COST_MIN_MAX_I8 1 // native 159 #define BL_SIMD_COST_MIN_MAX_U16 1 // native 160 #define BL_SIMD_COST_MIN_MAX_I32 1 // native 161 #define BL_SIMD_COST_MIN_MAX_U32 1 // native 162 #define BL_SIMD_COST_MIN_MAX_I64 2 // emulated ('cmp_lt_gt_i64' + 'blend') 163 #define BL_SIMD_COST_MIN_MAX_U64 4 // emulated ('cmp_lt_gt_u64' + 'blend') 164 #define BL_SIMD_COST_MUL_I32 1 // native 165 #define BL_SIMD_COST_MUL_I64 7 // emulated (complex) 166 #elif defined(BL_TARGET_OPT_SSE4_1) 167 #define BL_SIMD_COST_ABS_I64 4 // emulated (complex) 168 #define BL_SIMD_COST_CMP_EQ_I64 1 // native 169 #define BL_SIMD_COST_CMP_LT_GT_I64 6 // emulated (complex) 170 #define BL_SIMD_COST_CMP_LE_GE_I64 7 // emulated ('cmp_lt_gt_i64' + 'not') 171 #define BL_SIMD_COST_CMP_LT_GT_U64 8 // emulated ('cmp_lt_gt_i64' + 2x'xor') 172 #define BL_SIMD_COST_CMP_LE_GE_U64 9 // emulated ('cmp_lt_gt_u64' + 'not') 173 #define BL_SIMD_COST_MIN_MAX_I8 1 // native 174 #define BL_SIMD_COST_MIN_MAX_U16 1 // native 175 #define BL_SIMD_COST_MIN_MAX_I32 1 // native 176 #define BL_SIMD_COST_MIN_MAX_U32 1 // native 177 #define BL_SIMD_COST_MIN_MAX_I64 7 // emulated ('cmp_lt_gt_i64' + 'blend') 178 #define BL_SIMD_COST_MIN_MAX_U64 9 // emulated ('cmp_lt_gt_u64' + 'blend') 179 #define BL_SIMD_COST_MUL_I32 1 // native 180 #define BL_SIMD_COST_MUL_I64 7 // emulated (complex) 181 #else 182 #define BL_SIMD_COST_ABS_I64 4 // emulated (complex) 183 #define BL_SIMD_COST_CMP_EQ_I64 3 // emulated ('cmp_eq_i32' + 'shuffle_i32' + 'and') 184 #define BL_SIMD_COST_CMP_LT_GT_I64 6 // emulated (complex) 185 #define BL_SIMD_COST_CMP_LE_GE_I64 7 // emulated ('cmp_lt_gt_i64' + 'not') 186 #define BL_SIMD_COST_CMP_LT_GT_U64 8 // emulated ('cmp_lt_gt_i64' + 2x'xor') 187 #define BL_SIMD_COST_CMP_LE_GE_U64 9 // emulated ('cmp_lt_gt_u64' + 'not') 188 #define BL_SIMD_COST_MIN_MAX_I8 4 // emulated ('cmp_lt_gt_i8' + 'blend') 189 #define BL_SIMD_COST_MIN_MAX_U16 2 // emulated ('sub_u16' and 'subs_u16') 190 #define BL_SIMD_COST_MIN_MAX_I32 4 // emulated ('cmp_lt_gt_i8' + 'blend') 191 #define BL_SIMD_COST_MIN_MAX_U32 6 // emulated ('cmp_lt_gt_u32' + 'blend') 192 #define BL_SIMD_COST_MIN_MAX_I64 9 // emulated ('cmp_lt_gt_i64' + 'blend') 193 #define BL_SIMD_COST_MIN_MAX_U64 11 // emulated ('cmp_lt_gt_u64' + 'blend') 194 #define BL_SIMD_COST_MUL_I32 6 // emulated (complex) 195 #define BL_SIMD_COST_MUL_I64 7 // emulated (complex) 196 #endif 197 198 // SIMD - Features 199 // =============== 200 201 #define BL_SIMD_FEATURE_ARRAY_LOOKUP 202 #define BL_SIMD_FEATURE_EXTRACT_SIGN_BITS 203 204 namespace SIMD { 205 206 // SIMD - Vector Registers 207 // ======================= 208 209 //! \cond NEVER 210 211 template<> struct Vec<16, int8_t>; 212 template<> struct Vec<16, uint8_t>; 213 template<> struct Vec<16, int16_t>; 214 template<> struct Vec<16, uint16_t>; 215 template<> struct Vec<16, int32_t>; 216 template<> struct Vec<16, uint32_t>; 217 template<> struct Vec<16, int64_t>; 218 template<> struct Vec<16, uint64_t>; 219 template<> struct Vec<16, float>; 220 template<> struct Vec<16, double>; 221 222 template<> struct Vec<32, int8_t>; 223 template<> struct Vec<32, uint8_t>; 224 template<> struct Vec<32, int16_t>; 225 template<> struct Vec<32, uint16_t>; 226 template<> struct Vec<32, int32_t>; 227 template<> struct Vec<32, uint32_t>; 228 template<> struct Vec<32, int64_t>; 229 template<> struct Vec<32, uint64_t>; 230 template<> struct Vec<32, float>; 231 template<> struct Vec<32, double>; 232 233 template<> struct Vec<64, int8_t>; 234 template<> struct Vec<64, uint8_t>; 235 template<> struct Vec<64, int16_t>; 236 template<> struct Vec<64, uint16_t>; 237 template<> struct Vec<64, int32_t>; 238 template<> struct Vec<64, uint32_t>; 239 template<> struct Vec<64, int64_t>; 240 template<> struct Vec<64, uint64_t>; 241 template<> struct Vec<64, float>; 242 template<> struct Vec<64, double>; 243 244 #define BL_DECLARE_SIMD_TYPE(type_name, width, simd_type, element_type) \ 245 template<> \ 246 struct Vec<width, element_type> { \ 247 static inline constexpr uint32_t kW = width; \ 248 static inline constexpr uint32_t kHalfVectorWidth = width > 16 ? width / 2u : 16; \ 249 \ 250 static inline constexpr uint32_t kElementWidth = uint32_t(sizeof(element_type)); \ 251 static inline constexpr uint32_t kElementCount = width / kElementWidth; \ 252 \ 253 typedef Vec<width, element_type> VectorType; \ 254 typedef Vec<kHalfVectorWidth, element_type> VectorHalfType; \ 255 typedef Vec<16, element_type> Vector128Type; \ 256 typedef Vec<32, element_type> Vector256Type; \ 257 typedef Vec<64, element_type> Vector512Type; \ 258 \ 259 typedef simd_type SimdType; \ 260 typedef typename VectorHalfType::SimdType HalfSimdType; \ 261 \ 262 typedef element_type ElementType; \ 263 \ 264 SimdType v; \ 265 }; \ 266 \ 267 typedef Vec<width, element_type> type_name 268 269 typedef BL_UNALIGNED_TYPE(__m128i, 1) unaligned_m128i; 270 typedef BL_UNALIGNED_TYPE(__m128 , 1) unaligned_m128; 271 typedef BL_UNALIGNED_TYPE(__m128d, 1) unaligned_m128d; 272 273 BL_DECLARE_SIMD_TYPE(Vec16xI8 , 16, __m128i, int8_t); 274 BL_DECLARE_SIMD_TYPE(Vec16xU8 , 16, __m128i, uint8_t); 275 BL_DECLARE_SIMD_TYPE(Vec8xI16 , 16, __m128i, int16_t); 276 BL_DECLARE_SIMD_TYPE(Vec8xU16 , 16, __m128i, uint16_t); 277 BL_DECLARE_SIMD_TYPE(Vec4xI32 , 16, __m128i, int32_t); 278 BL_DECLARE_SIMD_TYPE(Vec4xU32 , 16, __m128i, uint32_t); 279 BL_DECLARE_SIMD_TYPE(Vec2xI64 , 16, __m128i, int64_t); 280 BL_DECLARE_SIMD_TYPE(Vec2xU64 , 16, __m128i, uint64_t); 281 BL_DECLARE_SIMD_TYPE(Vec4xF32 , 16, __m128 , float); 282 BL_DECLARE_SIMD_TYPE(Vec2xF64 , 16, __m128d, double); 283 284 // 256-bit types (including integers) are accessible through AVX as AVX also includes conversion instructions 285 // between integer types and FP types. So, we don't need AVX2 check to provide 256-bit integer vectors. 286 #if defined(BL_TARGET_OPT_AVX) 287 typedef BL_UNALIGNED_TYPE(__m256i, 1) unaligned_m256i; 288 typedef BL_UNALIGNED_TYPE(__m256 , 1) unaligned_m256; 289 typedef BL_UNALIGNED_TYPE(__m256d, 1) unaligned_m256d; 290 291 BL_DECLARE_SIMD_TYPE(Vec32xI8 , 32, __m256i, int8_t); 292 BL_DECLARE_SIMD_TYPE(Vec32xU8 , 32, __m256i, uint8_t); 293 BL_DECLARE_SIMD_TYPE(Vec16xI16, 32, __m256i, int16_t); 294 BL_DECLARE_SIMD_TYPE(Vec16xU16, 32, __m256i, uint16_t); 295 BL_DECLARE_SIMD_TYPE(Vec8xI32 , 32, __m256i, int32_t); 296 BL_DECLARE_SIMD_TYPE(Vec8xU32 , 32, __m256i, uint32_t); 297 BL_DECLARE_SIMD_TYPE(Vec4xI64 , 32, __m256i, int64_t); 298 BL_DECLARE_SIMD_TYPE(Vec4xU64 , 32, __m256i, uint64_t); 299 BL_DECLARE_SIMD_TYPE(Vec8xF32 , 32, __m256 , float); 300 BL_DECLARE_SIMD_TYPE(Vec4xF64 , 32, __m256d, double); 301 #endif // BL_TARGET_OPT_AVX 302 303 #if defined(BL_TARGET_OPT_AVX512) 304 typedef BL_UNALIGNED_TYPE(__m512i, 1) unaligned_m512i; 305 typedef BL_UNALIGNED_TYPE(__m512 , 1) unaligned_m512; 306 typedef BL_UNALIGNED_TYPE(__m512d, 1) unaligned_m512d; 307 308 BL_DECLARE_SIMD_TYPE(Vec64xI8 , 64, __m512i, int8_t); 309 BL_DECLARE_SIMD_TYPE(Vec64xU8 , 64, __m512i, uint8_t); 310 BL_DECLARE_SIMD_TYPE(Vec32xI16, 64, __m512i, int16_t); 311 BL_DECLARE_SIMD_TYPE(Vec32xU16, 64, __m512i, uint16_t); 312 BL_DECLARE_SIMD_TYPE(Vec16xI32, 64, __m512i, int32_t); 313 BL_DECLARE_SIMD_TYPE(Vec16xU32, 64, __m512i, uint32_t); 314 BL_DECLARE_SIMD_TYPE(Vec8xI64 , 64, __m512i, int64_t); 315 BL_DECLARE_SIMD_TYPE(Vec8xU64 , 64, __m512i, uint64_t); 316 BL_DECLARE_SIMD_TYPE(Vec16xF32, 64, __m512 , float); 317 BL_DECLARE_SIMD_TYPE(Vec8xF64 , 64, __m512d, double); 318 #endif // BL_TARGET_OPT_AVX512 319 320 #undef BL_DECLARE_SIMD_TYPE 321 322 //! \endcond 323 324 // Everything must be in anonymous namespace to avoid ODR violation. 325 namespace { 326 327 // SIMD - Internal - Info 328 // ====================== 329 330 namespace Internal { 331 332 template<size_t kW> 333 struct SimdInfo; 334 335 //! \cond NEVER 336 337 template<> 338 struct SimdInfo<16> { 339 typedef __m128i RegI; 340 typedef __m128 RegF; 341 typedef __m128d RegD; 342 }; 343 344 #if defined(BL_TARGET_OPT_AVX) 345 template<> 346 struct SimdInfo<32> { 347 typedef __m256i RegI; 348 typedef __m256 RegF; 349 typedef __m256d RegD; 350 }; 351 #endif // BL_TARGET_OPT_AVX 352 353 #if defined(BL_TARGET_OPT_AVX512) 354 template<> 355 struct SimdInfo<64> { 356 typedef __m512i RegI; 357 typedef __m512 RegF; 358 typedef __m512d RegD; 359 }; 360 #endif // BL_TARGET_OPT_AVX512 361 362 //! \endcond 363 364 } // {Internal} 365 366 // SIMD - Internal - Cast 367 // ====================== 368 369 // Low-level cast operation that casts a SIMD register type (used by high-level wrappers). 370 template<typename DstSimdT, typename SrcSimdT> 371 BL_INLINE_NODEBUG DstSimdT simd_cast(const SrcSimdT& src) noexcept; 372 373 BL_INLINE_NODEBUG __m128i simd_as_i(const __m128i& src) noexcept { return src; } 374 BL_INLINE_NODEBUG __m128i simd_as_i(const __m128& src) noexcept { return _mm_castps_si128(src); } 375 BL_INLINE_NODEBUG __m128i simd_as_i(const __m128d& src) noexcept { return _mm_castpd_si128(src); } 376 377 BL_INLINE_NODEBUG __m128 simd_as_f(const __m128i& src) noexcept { return _mm_castsi128_ps(src); } 378 BL_INLINE_NODEBUG __m128 simd_as_f(const __m128& src) noexcept { return src; } 379 BL_INLINE_NODEBUG __m128 simd_as_f(const __m128d& src) noexcept { return _mm_castpd_ps(src); } 380 381 BL_INLINE_NODEBUG __m128d simd_as_d(const __m128i& src) noexcept { return _mm_castsi128_pd(src); } 382 BL_INLINE_NODEBUG __m128d simd_as_d(const __m128& src) noexcept { return _mm_castps_pd(src); } 383 BL_INLINE_NODEBUG __m128d simd_as_d(const __m128d& src) noexcept { return src; } 384 385 template<> BL_INLINE_NODEBUG __m128i simd_cast(const __m128i& src) noexcept { return src; } 386 template<> BL_INLINE_NODEBUG __m128i simd_cast(const __m128& src) noexcept { return _mm_castps_si128(src); } 387 template<> BL_INLINE_NODEBUG __m128i simd_cast(const __m128d& src) noexcept { return _mm_castpd_si128(src); } 388 389 template<> BL_INLINE_NODEBUG __m128 simd_cast(const __m128i& src) noexcept { return _mm_castsi128_ps(src); } 390 template<> BL_INLINE_NODEBUG __m128 simd_cast(const __m128& src) noexcept { return src; } 391 template<> BL_INLINE_NODEBUG __m128 simd_cast(const __m128d& src) noexcept { return _mm_castpd_ps(src); } 392 393 template<> BL_INLINE_NODEBUG __m128d simd_cast(const __m128i& src) noexcept { return _mm_castsi128_pd(src); } 394 template<> BL_INLINE_NODEBUG __m128d simd_cast(const __m128& src) noexcept { return _mm_castps_pd(src); } 395 template<> BL_INLINE_NODEBUG __m128d simd_cast(const __m128d& src) noexcept { return src; } 396 397 #if defined(BL_TARGET_OPT_AVX) 398 BL_INLINE_NODEBUG __m256i simd_as_i(const __m256i& src) noexcept { return src; } 399 BL_INLINE_NODEBUG __m256i simd_as_i(const __m256& src) noexcept { return _mm256_castps_si256(src); } 400 BL_INLINE_NODEBUG __m256i simd_as_i(const __m256d& src) noexcept { return _mm256_castpd_si256(src); } 401 BL_INLINE_NODEBUG __m256 simd_as_f(const __m256i& src) noexcept { return _mm256_castsi256_ps(src); } 402 BL_INLINE_NODEBUG __m256 simd_as_f(const __m256& src) noexcept { return src; } 403 BL_INLINE_NODEBUG __m256 simd_as_f(const __m256d& src) noexcept { return _mm256_castpd_ps(src); } 404 BL_INLINE_NODEBUG __m256d simd_as_d(const __m256i& src) noexcept { return _mm256_castsi256_pd(src); } 405 BL_INLINE_NODEBUG __m256d simd_as_d(const __m256& src) noexcept { return _mm256_castps_pd(src); } 406 BL_INLINE_NODEBUG __m256d simd_as_d(const __m256d& src) noexcept { return src; } 407 408 template<> BL_INLINE_NODEBUG __m128i simd_cast(const __m256i& src) noexcept { return _mm256_castsi256_si128(src); } 409 template<> BL_INLINE_NODEBUG __m128i simd_cast(const __m256& src) noexcept { return _mm256_castsi256_si128(simd_as_i(src)); } 410 template<> BL_INLINE_NODEBUG __m128i simd_cast(const __m256d& src) noexcept { return _mm256_castsi256_si128(simd_as_i(src)); } 411 template<> BL_INLINE_NODEBUG __m256i simd_cast(const __m128i& src) noexcept { return _mm256_castsi128_si256(src); } 412 template<> BL_INLINE_NODEBUG __m256i simd_cast(const __m128& src) noexcept { return _mm256_castsi128_si256(simd_as_i(src)); } 413 template<> BL_INLINE_NODEBUG __m256i simd_cast(const __m128d& src) noexcept { return _mm256_castsi128_si256(simd_as_i(src)); } 414 template<> BL_INLINE_NODEBUG __m256i simd_cast(const __m256i& src) noexcept { return src; } 415 template<> BL_INLINE_NODEBUG __m256i simd_cast(const __m256& src) noexcept { return _mm256_castps_si256(src); } 416 template<> BL_INLINE_NODEBUG __m256i simd_cast(const __m256d& src) noexcept { return _mm256_castpd_si256(src); } 417 418 template<> BL_INLINE_NODEBUG __m128 simd_cast(const __m256i& src) noexcept { return _mm256_castps256_ps128(simd_as_f(src)); } 419 template<> BL_INLINE_NODEBUG __m128 simd_cast(const __m256& src) noexcept { return _mm256_castps256_ps128(src); } 420 template<> BL_INLINE_NODEBUG __m128 simd_cast(const __m256d& src) noexcept { return _mm256_castps256_ps128(simd_as_f(src)); } 421 template<> BL_INLINE_NODEBUG __m256 simd_cast(const __m128i& src) noexcept { return _mm256_castps128_ps256(simd_as_f(src)); } 422 template<> BL_INLINE_NODEBUG __m256 simd_cast(const __m128& src) noexcept { return _mm256_castps128_ps256(src); } 423 template<> BL_INLINE_NODEBUG __m256 simd_cast(const __m128d& src) noexcept { return _mm256_castps128_ps256(simd_as_f(src)); } 424 template<> BL_INLINE_NODEBUG __m256 simd_cast(const __m256i& src) noexcept { return _mm256_castsi256_ps(src); } 425 template<> BL_INLINE_NODEBUG __m256 simd_cast(const __m256& src) noexcept { return src; } 426 template<> BL_INLINE_NODEBUG __m256 simd_cast(const __m256d& src) noexcept { return _mm256_castpd_ps(src); } 427 428 template<> BL_INLINE_NODEBUG __m128d simd_cast(const __m256i& src) noexcept { return _mm256_castpd256_pd128(simd_as_d(src)); } 429 template<> BL_INLINE_NODEBUG __m128d simd_cast(const __m256& src) noexcept { return _mm256_castpd256_pd128(simd_as_d(src)); } 430 template<> BL_INLINE_NODEBUG __m128d simd_cast(const __m256d& src) noexcept { return _mm256_castpd256_pd128(src); } 431 template<> BL_INLINE_NODEBUG __m256d simd_cast(const __m128i& src) noexcept { return _mm256_castpd128_pd256(simd_as_d(src)); } 432 template<> BL_INLINE_NODEBUG __m256d simd_cast(const __m128& src) noexcept { return _mm256_castpd128_pd256(simd_as_d(src)); } 433 template<> BL_INLINE_NODEBUG __m256d simd_cast(const __m128d& src) noexcept { return _mm256_castpd128_pd256(src); } 434 template<> BL_INLINE_NODEBUG __m256d simd_cast(const __m256i& src) noexcept { return _mm256_castsi256_pd(src); } 435 template<> BL_INLINE_NODEBUG __m256d simd_cast(const __m256& src) noexcept { return _mm256_castps_pd(src); } 436 template<> BL_INLINE_NODEBUG __m256d simd_cast(const __m256d& src) noexcept { return src; } 437 #endif // BL_TARGET_OPT_AVX 438 439 #if defined(BL_TARGET_OPT_AVX512) 440 BL_INLINE_NODEBUG __m512i simd_as_i(const __m512i& src) noexcept { return src; } 441 BL_INLINE_NODEBUG __m512i simd_as_i(const __m512& src) noexcept { return _mm512_castps_si512(src); } 442 BL_INLINE_NODEBUG __m512i simd_as_i(const __m512d& src) noexcept { return _mm512_castpd_si512(src); } 443 444 BL_INLINE_NODEBUG __m512 simd_as_f(const __m512i& src) noexcept { return _mm512_castsi512_ps(src); } 445 BL_INLINE_NODEBUG __m512 simd_as_f(const __m512& src) noexcept { return src; } 446 BL_INLINE_NODEBUG __m512 simd_as_f(const __m512d& src) noexcept { return _mm512_castpd_ps(src); } 447 448 BL_INLINE_NODEBUG __m512d simd_as_d(const __m512i& src) noexcept { return _mm512_castsi512_pd(src); } 449 BL_INLINE_NODEBUG __m512d simd_as_d(const __m512& src) noexcept { return _mm512_castps_pd(src); } 450 BL_INLINE_NODEBUG __m512d simd_as_d(const __m512d& src) noexcept { return src; } 451 452 template<> BL_INLINE_NODEBUG __m128i simd_cast(const __m512i& src) noexcept { return _mm512_castsi512_si128(src); } 453 template<> BL_INLINE_NODEBUG __m128i simd_cast(const __m512& src) noexcept { return _mm512_castsi512_si128(simd_as_i(src)); } 454 template<> BL_INLINE_NODEBUG __m128i simd_cast(const __m512d& src) noexcept { return _mm512_castsi512_si128(simd_as_i(src)); } 455 template<> BL_INLINE_NODEBUG __m256i simd_cast(const __m512i& src) noexcept { return _mm512_castsi512_si256(src); } 456 template<> BL_INLINE_NODEBUG __m256i simd_cast(const __m512& src) noexcept { return _mm512_castsi512_si256(simd_as_i(src)); } 457 template<> BL_INLINE_NODEBUG __m256i simd_cast(const __m512d& src) noexcept { return _mm512_castsi512_si256(simd_as_i(src)); } 458 template<> BL_INLINE_NODEBUG __m512i simd_cast(const __m128i& src) noexcept { return _mm512_castsi128_si512(src); } 459 template<> BL_INLINE_NODEBUG __m512i simd_cast(const __m128& src) noexcept { return _mm512_castsi128_si512(simd_as_i(src)); } 460 template<> BL_INLINE_NODEBUG __m512i simd_cast(const __m128d& src) noexcept { return _mm512_castsi128_si512(simd_as_i(src)); } 461 template<> BL_INLINE_NODEBUG __m512i simd_cast(const __m256i& src) noexcept { return _mm512_castsi256_si512(src); } 462 template<> BL_INLINE_NODEBUG __m512i simd_cast(const __m256& src) noexcept { return _mm512_castsi256_si512(simd_as_i(src)); } 463 template<> BL_INLINE_NODEBUG __m512i simd_cast(const __m256d& src) noexcept { return _mm512_castsi256_si512(simd_as_i(src)); } 464 template<> BL_INLINE_NODEBUG __m512i simd_cast(const __m512i& src) noexcept { return src; } 465 template<> BL_INLINE_NODEBUG __m512i simd_cast(const __m512& src) noexcept { return simd_as_i(src); } 466 template<> BL_INLINE_NODEBUG __m512i simd_cast(const __m512d& src) noexcept { return simd_as_i(src); } 467 468 template<> BL_INLINE_NODEBUG __m128 simd_cast(const __m512i& src) noexcept { return _mm512_castps512_ps128(simd_as_f(src)); } 469 template<> BL_INLINE_NODEBUG __m128 simd_cast(const __m512& src) noexcept { return _mm512_castps512_ps128(src); } 470 template<> BL_INLINE_NODEBUG __m128 simd_cast(const __m512d& src) noexcept { return _mm512_castps512_ps128(simd_as_f(src)); } 471 template<> BL_INLINE_NODEBUG __m256 simd_cast(const __m512i& src) noexcept { return _mm512_castps512_ps256(simd_as_f(src)); } 472 template<> BL_INLINE_NODEBUG __m256 simd_cast(const __m512& src) noexcept { return _mm512_castps512_ps256(src); } 473 template<> BL_INLINE_NODEBUG __m256 simd_cast(const __m512d& src) noexcept { return _mm512_castps512_ps256(simd_as_f(src)); } 474 template<> BL_INLINE_NODEBUG __m512 simd_cast(const __m128i& src) noexcept { return _mm512_castps128_ps512(simd_as_f(src)); } 475 template<> BL_INLINE_NODEBUG __m512 simd_cast(const __m128& src) noexcept { return _mm512_castps128_ps512(src); } 476 template<> BL_INLINE_NODEBUG __m512 simd_cast(const __m128d& src) noexcept { return _mm512_castps128_ps512(simd_as_f(src)); } 477 template<> BL_INLINE_NODEBUG __m512 simd_cast(const __m256i& src) noexcept { return _mm512_castps256_ps512(simd_as_f(src)); } 478 template<> BL_INLINE_NODEBUG __m512 simd_cast(const __m256& src) noexcept { return _mm512_castps256_ps512(src); } 479 template<> BL_INLINE_NODEBUG __m512 simd_cast(const __m256d& src) noexcept { return _mm512_castps256_ps512(simd_as_f(src)); } 480 template<> BL_INLINE_NODEBUG __m512 simd_cast(const __m512i& src) noexcept { return _mm512_castsi512_ps(src); } 481 template<> BL_INLINE_NODEBUG __m512 simd_cast(const __m512& src) noexcept { return src; } 482 template<> BL_INLINE_NODEBUG __m512 simd_cast(const __m512d& src) noexcept { return _mm512_castpd_ps(src); } 483 484 template<> BL_INLINE_NODEBUG __m128d simd_cast(const __m512i& src) noexcept { return _mm512_castpd512_pd128(simd_as_d(src)); } 485 template<> BL_INLINE_NODEBUG __m128d simd_cast(const __m512& src) noexcept { return _mm512_castpd512_pd128(simd_as_d(src)); } 486 template<> BL_INLINE_NODEBUG __m128d simd_cast(const __m512d& src) noexcept { return _mm512_castpd512_pd128(src); } 487 template<> BL_INLINE_NODEBUG __m256d simd_cast(const __m512i& src) noexcept { return _mm512_castpd512_pd256(simd_as_d(src)); } 488 template<> BL_INLINE_NODEBUG __m256d simd_cast(const __m512& src) noexcept { return _mm512_castpd512_pd256(simd_as_d(src)); } 489 template<> BL_INLINE_NODEBUG __m256d simd_cast(const __m512d& src) noexcept { return _mm512_castpd512_pd256(src); } 490 template<> BL_INLINE_NODEBUG __m512d simd_cast(const __m128i& src) noexcept { return _mm512_castpd128_pd512(simd_as_d(src)); } 491 template<> BL_INLINE_NODEBUG __m512d simd_cast(const __m128& src) noexcept { return _mm512_castpd128_pd512(simd_as_d(src)); } 492 template<> BL_INLINE_NODEBUG __m512d simd_cast(const __m128d& src) noexcept { return _mm512_castpd128_pd512(src); } 493 template<> BL_INLINE_NODEBUG __m512d simd_cast(const __m256i& src) noexcept { return _mm512_castpd256_pd512(simd_as_d(src)); } 494 template<> BL_INLINE_NODEBUG __m512d simd_cast(const __m256& src) noexcept { return _mm512_castpd256_pd512(simd_as_d(src)); } 495 template<> BL_INLINE_NODEBUG __m512d simd_cast(const __m256d& src) noexcept { return _mm512_castpd256_pd512(src); } 496 template<> BL_INLINE_NODEBUG __m512d simd_cast(const __m512i& src) noexcept { return _mm512_castsi512_pd(src); } 497 template<> BL_INLINE_NODEBUG __m512d simd_cast(const __m512& src) noexcept { return _mm512_castps_pd(src); } 498 template<> BL_INLINE_NODEBUG __m512d simd_cast(const __m512d& src) noexcept { return src; } 499 #endif // BL_TARGET_OPT_AVX512 500 501 template<typename DstVectorT, typename SrcVectorT> 502 BL_INLINE_NODEBUG DstVectorT vec_cast(const SrcVectorT& x) noexcept { 503 return DstVectorT{simd_cast<typename DstVectorT::SimdType>(x.v)}; 504 } 505 506 template<typename DstElementT, typename SrcVectorT> 507 BL_INLINE_NODEBUG DstElementT vec_of(const SrcVectorT& x) noexcept { 508 return Vec<SrcVectorT::kW, DstElementT>{simd_cast<Vec<SrcVectorT::kW, DstElementT>>(x.v)}; 509 } 510 511 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, int8_t> vec_i8(const Vec<W, T>& src) noexcept { return vec_cast<Vec<W, int8_t>>(src); } 512 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, uint8_t> vec_u8(const Vec<W, T>& src) noexcept { return vec_cast<Vec<W, uint8_t>>(src); } 513 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, int16_t> vec_i16(const Vec<W, T>& src) noexcept { return vec_cast<Vec<W, int16_t>>(src); } 514 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, uint16_t> vec_u16(const Vec<W, T>& src) noexcept { return vec_cast<Vec<W, uint16_t>>(src); } 515 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, int32_t> vec_i32(const Vec<W, T>& src) noexcept { return vec_cast<Vec<W, int32_t>>(src); } 516 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, uint32_t> vec_u32(const Vec<W, T>& src) noexcept { return vec_cast<Vec<W, uint32_t>>(src); } 517 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, int64_t> vec_i64(const Vec<W, T>& src) noexcept { return vec_cast<Vec<W, int64_t>>(src); } 518 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, uint64_t> vec_u64(const Vec<W, T>& src) noexcept { return vec_cast<Vec<W, uint64_t>>(src); } 519 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, float> vec_f32(const Vec<W, T>& src) noexcept { return vec_cast<Vec<W, float>>(src); } 520 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, double> vec_f64(const Vec<W, T>& src) noexcept { return vec_cast<Vec<W, double>>(src); } 521 522 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<16, T> vec_128(const Vec<W, T>& src) noexcept { return vec_cast<Vec<16, T>>(src); } 523 524 #if defined(BL_TARGET_OPT_AVX) 525 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<32, T> vec_256(const Vec<W, T>& src) noexcept { return vec_cast<Vec<32, T>>(src); } 526 #endif // BL_TARGET_OPT_AVX 527 528 #if defined(BL_TARGET_OPT_AVX512) 529 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<64, T> vec_512(const Vec<W, T>& src) noexcept { return vec_cast<Vec<64, T>>(src); } 530 #endif // BL_TARGET_OPT_AVX512 531 532 template<typename DstVectorT, typename SrcT> 533 BL_INLINE_NODEBUG const DstVectorT& vec_const(const SrcT* src) noexcept { return *static_cast<const DstVectorT*>(static_cast<const void*>(src)); } 534 535 // Converts a native SimdT register type to a wrapped V. 536 template<typename V, typename SimdT> 537 BL_INLINE_NODEBUG V from_simd(const SimdT& reg) noexcept { return V{simd_cast<typename V::SimdType>(reg)}; } 538 539 // Converts a wrapped V to a native SimdT register type. 540 template<typename SimdT, typename V> 541 BL_INLINE_NODEBUG SimdT to_simd(const V& vec) noexcept { return simd_cast<SimdT>(vec.v); } 542 543 // SIMD - Internal - Make Zero & Ones & Undefined 544 // ============================================== 545 546 namespace Internal { 547 548 template<typename SimdT> 549 BL_INLINE_NODEBUG SimdT simd_make_zero() noexcept; 550 551 template<typename SimdT> 552 BL_INLINE_NODEBUG SimdT simd_make_ones() noexcept; 553 554 template<typename SimdT> 555 BL_INLINE_NODEBUG SimdT simd_make_undefined() noexcept; 556 557 template<> BL_INLINE_NODEBUG __m128i simd_make_zero() noexcept { return _mm_setzero_si128(); } 558 template<> BL_INLINE_NODEBUG __m128 simd_make_zero() noexcept { return _mm_setzero_ps(); } 559 template<> BL_INLINE_NODEBUG __m128d simd_make_zero() noexcept { return _mm_setzero_pd(); } 560 561 template<> BL_INLINE_NODEBUG __m128i simd_make_ones() noexcept { return _mm_set1_epi32(-1); } 562 template<> BL_INLINE_NODEBUG __m128 simd_make_ones() noexcept { return simd_cast<__m128>(simd_make_ones<__m128i>()); } 563 template<> BL_INLINE_NODEBUG __m128d simd_make_ones() noexcept { return simd_cast<__m128d>(simd_make_ones<__m128i>()); } 564 565 template<> BL_INLINE_NODEBUG __m128i simd_make_undefined() noexcept { return _mm_undefined_si128(); } 566 template<> BL_INLINE_NODEBUG __m128 simd_make_undefined() noexcept { return _mm_undefined_ps(); } 567 template<> BL_INLINE_NODEBUG __m128d simd_make_undefined() noexcept { return _mm_undefined_pd(); } 568 569 #if defined(BL_TARGET_OPT_AVX) 570 template<> BL_INLINE_NODEBUG __m256i simd_make_zero() noexcept { return _mm256_setzero_si256(); } 571 template<> BL_INLINE_NODEBUG __m256 simd_make_zero() noexcept { return _mm256_setzero_ps(); } 572 template<> BL_INLINE_NODEBUG __m256d simd_make_zero() noexcept { return _mm256_setzero_pd(); } 573 574 template<> BL_INLINE_NODEBUG __m256i simd_make_ones() noexcept { return _mm256_set1_epi32(-1); } 575 template<> BL_INLINE_NODEBUG __m256 simd_make_ones() noexcept { return simd_cast<__m256>(simd_make_ones<__m256i>()); } 576 template<> BL_INLINE_NODEBUG __m256d simd_make_ones() noexcept { return simd_cast<__m256d>(simd_make_ones<__m256i>()); } 577 578 template<> BL_INLINE_NODEBUG __m256i simd_make_undefined() noexcept { return _mm256_undefined_si256(); } 579 template<> BL_INLINE_NODEBUG __m256 simd_make_undefined() noexcept { return _mm256_undefined_ps(); } 580 template<> BL_INLINE_NODEBUG __m256d simd_make_undefined() noexcept { return _mm256_undefined_pd(); } 581 #endif // BL_TARGET_OPT_AVX 582 583 #if defined(BL_TARGET_OPT_AVX512) 584 template<> BL_INLINE_NODEBUG __m512i simd_make_zero() noexcept { return _mm512_setzero_si512(); } 585 template<> BL_INLINE_NODEBUG __m512 simd_make_zero() noexcept { return _mm512_setzero_ps(); } 586 template<> BL_INLINE_NODEBUG __m512d simd_make_zero() noexcept { return _mm512_setzero_pd(); } 587 588 template<> BL_INLINE_NODEBUG __m512i simd_make_ones() noexcept { return _mm512_set1_epi32(-1); } 589 template<> BL_INLINE_NODEBUG __m512 simd_make_ones() noexcept { return simd_cast<__m512>(simd_make_ones<__m512i>()); } 590 template<> BL_INLINE_NODEBUG __m512d simd_make_ones() noexcept { return simd_cast<__m512d>(simd_make_ones<__m512i>()); } 591 592 template<> BL_INLINE_NODEBUG __m512i simd_make_undefined() noexcept { return _mm512_undefined_epi32(); } 593 template<> BL_INLINE_NODEBUG __m512 simd_make_undefined() noexcept { return _mm512_undefined_ps(); } 594 template<> BL_INLINE_NODEBUG __m512d simd_make_undefined() noexcept { return _mm512_undefined_pd(); } 595 #endif // BL_TARGET_OPT_AVX512 596 597 } // {Internal} 598 599 // SIMD - Internal - Make Vector 600 // ============================= 601 602 namespace Internal { 603 604 template<size_t kW> 605 struct SimdMake; 606 607 BL_INLINE_NODEBUG __m128i simd_make128_u64(uint64_t x0) noexcept { 608 #if BL_TARGET_ARCH_BITS >= 64 609 return _mm_set1_epi64x(int64_t(x0)); 610 #else 611 return _mm_set_epi32(int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF), 612 int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF)); 613 #endif 614 } 615 616 BL_INLINE_NODEBUG __m128i simd_make128_u64(uint64_t x1, uint64_t x0) noexcept { 617 #if BL_TARGET_ARCH_BITS >= 64 618 return _mm_set_epi64x(int64_t(x1), int64_t(x0)); 619 #else 620 return _mm_set_epi32(int32_t(x1 >> 32), int32_t(x1 & 0xFFFFFFFF), 621 int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF)); 622 #endif 623 } 624 625 BL_INLINE_NODEBUG __m128i simd_make128_u32(uint32_t x0) noexcept { 626 return _mm_set1_epi32(int32_t(x0)); 627 } 628 629 BL_INLINE_NODEBUG __m128i simd_make128_u32(uint32_t x1, uint32_t x0) noexcept { 630 #if BL_TARGET_ARCH_BITS >= 64 631 return _mm_set1_epi64x(int64_t((uint64_t(x1) << 32) | x0)); 632 #else 633 return _mm_set_epi32(int32_t(x1), int32_t(x0), int32_t(x1), int32_t(x0)); 634 #endif 635 } 636 637 BL_INLINE_NODEBUG __m128i simd_make128_u32(uint32_t x3, uint32_t x2, uint32_t x1, uint32_t x0) noexcept { 638 return _mm_set_epi32(int32_t(x3), int32_t(x2), int32_t(x1), int32_t(x0)); 639 } 640 641 BL_INLINE_NODEBUG __m128i simd_make128_u16(uint16_t x0) noexcept { 642 return _mm_set1_epi16(int16_t(x0)); 643 } 644 645 BL_INLINE_NODEBUG __m128i simd_make128_u16(uint16_t x1, uint16_t x0) noexcept { 646 uint32_t v = (uint32_t(x1) << 16) | x0; 647 return _mm_set1_epi32(int32_t(v)); 648 } 649 650 BL_INLINE_NODEBUG __m128i simd_make128_u16(uint16_t x3, uint16_t x2, uint16_t x1, uint16_t x0) noexcept { 651 #if BL_TARGET_ARCH_BITS >= 64 652 uint64_t v = (uint64_t(x3) << 48) | (uint64_t(x2) << 32) | (uint64_t(x1) << 16) | x0; 653 return _mm_set1_epi64x(int64_t(v)); 654 #else 655 return _mm_set_epi16(int16_t(x3), int16_t(x2), int16_t(x1), int16_t(x0), 656 int16_t(x3), int16_t(x2), int16_t(x1), int16_t(x0)); 657 #endif 658 } 659 660 BL_INLINE_NODEBUG __m128i simd_make128_u16(uint16_t x7, uint16_t x6, uint16_t x5, uint16_t x4, 661 uint16_t x3, uint16_t x2, uint16_t x1, uint16_t x0) noexcept { 662 return _mm_set_epi16(int16_t(x7), int16_t(x6), int16_t(x5), int16_t(x4), 663 int16_t(x3), int16_t(x2), int16_t(x1), int16_t(x0)); 664 } 665 666 BL_INLINE_NODEBUG __m128i simd_make128_u8(uint8_t x0) noexcept { 667 return _mm_set1_epi8(int8_t(x0)); 668 } 669 670 BL_INLINE_NODEBUG __m128i simd_make128_u8(uint8_t x1, uint8_t x0) noexcept { 671 uint16_t v = uint16_t((uint16_t(x1) << 8) | (uint16_t(x0) << 0)); 672 return _mm_set1_epi16(int16_t(v)); 673 } 674 675 BL_INLINE_NODEBUG __m128i simd_make128_u8(uint8_t x3, uint8_t x2, uint8_t x1, uint8_t x0) noexcept { 676 uint32_t v = (uint32_t(x3) << 24) | (uint32_t(x2) << 16) | (uint32_t(x1) << 8) | (uint32_t(x0) << 0); 677 return _mm_set1_epi32(int32_t(v)); 678 } 679 680 BL_INLINE_NODEBUG __m128i simd_make128_u8(uint8_t x7, uint8_t x6, uint8_t x5, uint8_t x4, 681 uint8_t x3, uint8_t x2, uint8_t x1, uint8_t x0) noexcept { 682 #if BL_TARGET_ARCH_BITS >= 64 683 uint64_t v = (uint64_t(x7) << 56) | (uint64_t(x6) << 48) | (uint64_t(x5) << 40) | (uint64_t(x4) << 32) 684 | (uint64_t(x3) << 24) | (uint64_t(x2) << 16) | (uint64_t(x1) << 8) | (uint64_t(x0) << 0); 685 return _mm_set1_epi64x(int64_t(v)); 686 #else 687 return _mm_set_epi8(int8_t(x7), int8_t(x6), int8_t(x5), int8_t(x4), 688 int8_t(x3), int8_t(x2), int8_t(x1), int8_t(x0), 689 int8_t(x7), int8_t(x6), int8_t(x5), int8_t(x4), 690 int8_t(x3), int8_t(x2), int8_t(x1), int8_t(x0)); 691 #endif 692 } 693 694 BL_INLINE_NODEBUG __m128i simd_make128_u8(uint8_t x15, uint8_t x14, uint8_t x13, uint8_t x12, 695 uint8_t x11, uint8_t x10, uint8_t x09, uint8_t x08, 696 uint8_t x07, uint8_t x06, uint8_t x05, uint8_t x04, 697 uint8_t x03, uint8_t x02, uint8_t x01, uint8_t x00) noexcept { 698 return _mm_set_epi8(int8_t(x15), int8_t(x14), int8_t(x13), int8_t(x12), 699 int8_t(x11), int8_t(x10), int8_t(x09), int8_t(x08), 700 int8_t(x07), int8_t(x06), int8_t(x05), int8_t(x04), 701 int8_t(x03), int8_t(x02), int8_t(x01), int8_t(x00)); 702 } 703 704 BL_INLINE_NODEBUG __m128 simd_make128_f32(float x0) noexcept { 705 return _mm_set1_ps(x0); 706 } 707 708 BL_INLINE_NODEBUG __m128 simd_make128_f32(float x1, float x0) noexcept { 709 return _mm_set_ps(x1, x0, x1, x0); 710 } 711 712 BL_INLINE_NODEBUG __m128 simd_make128_f32(float x3, float x2, float x1, float x0) noexcept { 713 return _mm_set_ps(x3, x2, x1, x0); 714 } 715 716 BL_INLINE_NODEBUG __m128d simd_make128_f64(double x0) noexcept { 717 return _mm_set1_pd(x0); 718 } 719 720 BL_INLINE_NODEBUG __m128d simd_make128_f64(double x1, double x0) noexcept { 721 return _mm_set_pd(x1, x0); 722 } 723 724 template<> 725 struct SimdMake<16> { 726 template<typename... Args> static BL_INLINE_NODEBUG __m128i make_u8(Args&&... args) noexcept { return simd_make128_u8(BLInternal::forward<Args>(args)...); } 727 template<typename... Args> static BL_INLINE_NODEBUG __m128i make_u16(Args&&... args) noexcept { return simd_make128_u16(BLInternal::forward<Args>(args)...); } 728 template<typename... Args> static BL_INLINE_NODEBUG __m128i make_u32(Args&&... args) noexcept { return simd_make128_u32(BLInternal::forward<Args>(args)...); } 729 template<typename... Args> static BL_INLINE_NODEBUG __m128i make_u64(Args&&... args) noexcept { return simd_make128_u64(BLInternal::forward<Args>(args)...); } 730 template<typename... Args> static BL_INLINE_NODEBUG __m128 make_f32(Args&&... args) noexcept { return simd_make128_f32(BLInternal::forward<Args>(args)...); } 731 template<typename... Args> static BL_INLINE_NODEBUG __m128d make_f64(Args&&... args) noexcept { return simd_make128_f64(BLInternal::forward<Args>(args)...); } 732 }; 733 734 #ifdef BL_TARGET_OPT_AVX 735 BL_INLINE_NODEBUG __m256i simd_make256_u64(uint64_t x0) noexcept { 736 #if BL_TARGET_ARCH_BITS >= 64 737 return _mm256_set1_epi64x(int64_t(x0)); 738 #else 739 return _mm256_set_epi32(int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF), 740 int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF), 741 int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF), 742 int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF)); 743 #endif 744 } 745 746 BL_INLINE_NODEBUG __m256i simd_make256_u64(uint64_t x1, uint64_t x0) noexcept { 747 #if BL_TARGET_ARCH_BITS >= 64 748 return _mm256_set_epi64x(int64_t(x1), int64_t(x0), int64_t(x1), int64_t(x0)); 749 #else 750 return _mm256_set_epi32(int32_t(x1 >> 32), int32_t(x1 & 0xFFFFFFFF), 751 int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF), 752 int32_t(x1 >> 32), int32_t(x1 & 0xFFFFFFFF), 753 int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF)); 754 #endif 755 } 756 757 BL_INLINE_NODEBUG __m256i simd_make256_u64(uint64_t x3, uint64_t x2, uint64_t x1, uint64_t x0) noexcept { 758 #if BL_TARGET_ARCH_BITS >= 64 759 return _mm256_set_epi64x(int64_t(x3), int64_t(x2), int64_t(x1), int64_t(x0)); 760 #else 761 return _mm256_set_epi32(int32_t(x3 >> 32), int32_t(x3 & 0xFFFFFFFF), 762 int32_t(x2 >> 32), int32_t(x2 & 0xFFFFFFFF), 763 int32_t(x1 >> 32), int32_t(x1 & 0xFFFFFFFF), 764 int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF)); 765 #endif 766 } 767 768 BL_INLINE_NODEBUG __m256i simd_make256_u32(uint32_t x0) noexcept { 769 return _mm256_set1_epi32(int32_t(x0)); 770 } 771 772 BL_INLINE_NODEBUG __m256i simd_make256_u32(uint32_t x1, uint32_t x0) noexcept { 773 #if BL_TARGET_ARCH_BITS >= 64 774 return _mm256_set1_epi64x(int64_t((uint64_t(x1) << 32) | x0)); 775 #else 776 return _mm256_set_epi32(int32_t(x1), int32_t(x0), int32_t(x1), int32_t(x0), 777 int32_t(x1), int32_t(x0), int32_t(x1), int32_t(x0)); 778 #endif 779 } 780 781 BL_INLINE_NODEBUG __m256i simd_make256_u32(uint32_t x3, uint32_t x2, uint32_t x1, uint32_t x0) noexcept { 782 return _mm256_set_epi32(int32_t(x3), int32_t(x2), int32_t(x1), int32_t(x0), 783 int32_t(x3), int32_t(x2), int32_t(x1), int32_t(x0)); 784 } 785 786 BL_INLINE_NODEBUG __m256i simd_make256_u32(uint32_t x7, uint32_t x6, uint32_t x5, uint32_t x4, 787 uint32_t x3, uint32_t x2, uint32_t x1, uint32_t x0) noexcept { 788 return _mm256_set_epi32(int32_t(x7), int32_t(x6), int32_t(x5), int32_t(x4), 789 int32_t(x3), int32_t(x2), int32_t(x1), int32_t(x0)); 790 } 791 792 BL_INLINE_NODEBUG __m256i simd_make256_u16(uint16_t x0) noexcept { 793 return _mm256_set1_epi16(int16_t(x0)); 794 } 795 796 BL_INLINE_NODEBUG __m256i simd_make256_u16(uint16_t x1, uint16_t x0) noexcept { 797 uint32_t v = (uint32_t(x1) << 16) | x0; 798 return _mm256_set1_epi32(int32_t(v)); 799 } 800 801 BL_INLINE_NODEBUG __m256i simd_make256_u16(uint16_t x3, uint16_t x2, uint16_t x1, uint16_t x0) noexcept { 802 #if BL_TARGET_ARCH_BITS >= 64 803 uint64_t v = (uint64_t(x3) << 48) | (uint64_t(x2) << 32) | (uint64_t(x1) << 16) | x0; 804 return _mm256_set1_epi64x(int64_t(v)); 805 #else 806 return _mm256_set_epi16(int16_t(x3), int16_t(x2), int16_t(x1), int16_t(x0), 807 int16_t(x3), int16_t(x2), int16_t(x1), int16_t(x0), 808 int16_t(x3), int16_t(x2), int16_t(x1), int16_t(x0), 809 int16_t(x3), int16_t(x2), int16_t(x1), int16_t(x0)); 810 #endif 811 } 812 813 BL_INLINE_NODEBUG __m256i simd_make256_u16(uint16_t x7, uint16_t x6, uint16_t x5, uint16_t x4, 814 uint16_t x3, uint16_t x2, uint16_t x1, uint16_t x0) noexcept { 815 return _mm256_set_epi16(int16_t(x7), int16_t(x6), int16_t(x5), int16_t(x4), 816 int16_t(x3), int16_t(x2), int16_t(x1), int16_t(x0), 817 int16_t(x7), int16_t(x6), int16_t(x5), int16_t(x4), 818 int16_t(x3), int16_t(x2), int16_t(x1), int16_t(x0)); 819 } 820 821 BL_INLINE_NODEBUG __m256i simd_make256_u16(uint16_t x15, uint16_t x14, uint16_t x13, uint16_t x12, 822 uint16_t x11, uint16_t x10, uint16_t x09, uint16_t x08, 823 uint16_t x07, uint16_t x06, uint16_t x05, uint16_t x04, 824 uint16_t x03, uint16_t x02, uint16_t x01, uint16_t x00) noexcept { 825 return _mm256_set_epi16(int16_t(x15), int16_t(x14), int16_t(x13), int16_t(x12), 826 int16_t(x11), int16_t(x10), int16_t(x09), int16_t(x08), 827 int16_t(x07), int16_t(x06), int16_t(x05), int16_t(x04), 828 int16_t(x03), int16_t(x02), int16_t(x01), int16_t(x00)); 829 } 830 831 BL_INLINE_NODEBUG __m256i simd_make256_u8(uint8_t x0) noexcept { 832 return _mm256_set1_epi8(int8_t(x0)); 833 } 834 835 BL_INLINE_NODEBUG __m256i simd_make256_u8(uint8_t x1, uint8_t x0) noexcept { 836 uint16_t v = uint16_t((uint16_t(x1) << 8) | (uint16_t(x0) << 0)); 837 return _mm256_set1_epi16(int16_t(v)); 838 } 839 840 BL_INLINE_NODEBUG __m256i simd_make256_u8(uint8_t x3, uint8_t x2, uint8_t x1, uint8_t x0) noexcept { 841 uint32_t v = (uint32_t(x3) << 24) | (uint32_t(x2) << 16) | (uint32_t(x1) << 8) | (uint32_t(x0) << 0); 842 return _mm256_set1_epi32(int32_t(v)); 843 } 844 845 BL_INLINE_NODEBUG __m256i simd_make256_u8(uint8_t x7, uint8_t x6, uint8_t x5, uint8_t x4, 846 uint8_t x3, uint8_t x2, uint8_t x1, uint8_t x0) noexcept { 847 #if BL_TARGET_ARCH_BITS >= 64 848 uint64_t v = (uint64_t(x7) << 56) | (uint64_t(x6) << 48) | (uint64_t(x5) << 40) | (uint64_t(x4) << 32) 849 | (uint64_t(x3) << 24) | (uint64_t(x2) << 16) | (uint64_t(x1) << 8) | (uint64_t(x0) << 0); 850 return _mm256_set1_epi64x(int64_t(v)); 851 #else 852 int32_t hi = int32_t((uint32_t(x7) << 24) | (uint32_t(x6) << 16) | (uint32_t(x5) << 8) | uint32_t(x4)); 853 int32_t lo = int32_t((uint32_t(x3) << 24) | (uint32_t(x2) << 16) | (uint32_t(x1) << 8) | uint32_t(x0)); 854 return _mm256_set_epi32(hi, lo, hi, lo, hi, lo, hi, lo); 855 #endif 856 } 857 858 BL_INLINE_NODEBUG __m256i simd_make256_u8(uint8_t x15, uint8_t x14, uint8_t x13, uint8_t x12, 859 uint8_t x11, uint8_t x10, uint8_t x09, uint8_t x08, 860 uint8_t x07, uint8_t x06, uint8_t x05, uint8_t x04, 861 uint8_t x03, uint8_t x02, uint8_t x01, uint8_t x00) noexcept { 862 int32_t v3 = int32_t((uint32_t(x15) << 24) | (uint32_t(x14) << 16) | (uint32_t(x13) << 8) | uint32_t(x12)); 863 int32_t v2 = int32_t((uint32_t(x11) << 24) | (uint32_t(x10) << 16) | (uint32_t(x09) << 8) | uint32_t(x08)); 864 int32_t v1 = int32_t((uint32_t(x07) << 24) | (uint32_t(x06) << 16) | (uint32_t(x05) << 8) | uint32_t(x04)); 865 int32_t v0 = int32_t((uint32_t(x03) << 24) | (uint32_t(x02) << 16) | (uint32_t(x01) << 8) | uint32_t(x00)); 866 return _mm256_set_epi32(v3, v2, v1, v0, v3, v2, v1, v0); 867 } 868 869 BL_INLINE_NODEBUG __m256i simd_make256_u8(uint8_t x31, uint8_t x30, uint8_t x29, uint8_t x28, 870 uint8_t x27, uint8_t x26, uint8_t x25, uint8_t x24, 871 uint8_t x23, uint8_t x22, uint8_t x21, uint8_t x20, 872 uint8_t x19, uint8_t x18, uint8_t x17, uint8_t x16, 873 uint8_t x15, uint8_t x14, uint8_t x13, uint8_t x12, 874 uint8_t x11, uint8_t x10, uint8_t x09, uint8_t x08, 875 uint8_t x07, uint8_t x06, uint8_t x05, uint8_t x04, 876 uint8_t x03, uint8_t x02, uint8_t x01, uint8_t x00) noexcept { 877 return _mm256_set_epi8(int8_t(x31), int8_t(x30), int8_t(x29), int8_t(x28), 878 int8_t(x27), int8_t(x26), int8_t(x25), int8_t(x24), 879 int8_t(x23), int8_t(x22), int8_t(x21), int8_t(x20), 880 int8_t(x19), int8_t(x18), int8_t(x17), int8_t(x16), 881 int8_t(x15), int8_t(x14), int8_t(x13), int8_t(x12), 882 int8_t(x11), int8_t(x10), int8_t(x09), int8_t(x08), 883 int8_t(x07), int8_t(x06), int8_t(x05), int8_t(x04), 884 int8_t(x03), int8_t(x02), int8_t(x01), int8_t(x00)); 885 } 886 887 BL_INLINE_NODEBUG __m256 simd_make256_f32(float x0) noexcept { 888 return _mm256_set1_ps(x0); 889 } 890 891 BL_INLINE_NODEBUG __m256 simd_make256_f32(float x1, float x0) noexcept { 892 return _mm256_set_ps(x1, x0, x1, x0, 893 x1, x0, x1, x0); 894 } 895 896 BL_INLINE_NODEBUG __m256 simd_make256_f32(float x3, float x2, float x1, float x0) noexcept { 897 return _mm256_set_ps(x3, x2, x1, x0, 898 x3, x2, x1, x0); 899 } 900 901 BL_INLINE_NODEBUG __m256 simd_make256_f32(float x7, float x6, float x5, float x4, 902 float x3, float x2, float x1, float x0) noexcept { 903 return _mm256_set_ps(x7, x6, x5, x4, 904 x3, x2, x1, x0); 905 } 906 907 BL_INLINE_NODEBUG __m256d simd_make256_f64(double x0) noexcept { 908 return _mm256_set1_pd(x0); 909 } 910 911 BL_INLINE_NODEBUG __m256d simd_make256_f64(double x1, double x0) noexcept { 912 return _mm256_set_pd(x1, x0, x1, x0); 913 } 914 915 BL_INLINE_NODEBUG __m256d simd_make256_f64(double x3, double x2, double x1, double x0) noexcept { 916 return _mm256_set_pd(x3, x2, x1, x0); 917 } 918 919 template<> 920 struct SimdMake<32> { 921 template<typename... Args> static BL_INLINE_NODEBUG __m256i make_u8(Args&&... args) noexcept { return simd_make256_u8(BLInternal::forward<Args>(args)...); } 922 template<typename... Args> static BL_INLINE_NODEBUG __m256i make_u16(Args&&... args) noexcept { return simd_make256_u16(BLInternal::forward<Args>(args)...); } 923 template<typename... Args> static BL_INLINE_NODEBUG __m256i make_u32(Args&&... args) noexcept { return simd_make256_u32(BLInternal::forward<Args>(args)...); } 924 template<typename... Args> static BL_INLINE_NODEBUG __m256i make_u64(Args&&... args) noexcept { return simd_make256_u64(BLInternal::forward<Args>(args)...); } 925 template<typename... Args> static BL_INLINE_NODEBUG __m256 make_f32(Args&&... args) noexcept { return simd_make256_f32(BLInternal::forward<Args>(args)...); } 926 template<typename... Args> static BL_INLINE_NODEBUG __m256d make_f64(Args&&... args) noexcept { return simd_make256_f64(BLInternal::forward<Args>(args)...); } 927 }; 928 #endif // BL_TARGET_OPT_AVX 929 930 #ifdef BL_TARGET_OPT_AVX512 931 BL_INLINE_NODEBUG __m512i simd_make512_u64(uint64_t x0) noexcept { 932 #if BL_TARGET_ARCH_BITS >= 64 933 return _mm512_set1_epi64(int64_t(x0)); 934 #else 935 return _mm512_broadcast_i32x4( 936 _mm_set_epi32( 937 int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF), 938 int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF))); 939 #endif 940 } 941 942 BL_INLINE_NODEBUG __m512i simd_make512_u64(uint64_t x1, uint64_t x0) noexcept { 943 #if BL_TARGET_ARCH_BITS >= 64 944 return _mm512_set_epi64(int64_t(x1), int64_t(x0), int64_t(x1), int64_t(x0), 945 int64_t(x1), int64_t(x0), int64_t(x1), int64_t(x0)); 946 #else 947 return _mm512_broadcast_i32x4( 948 _mm_set_epi32( 949 int32_t(x1 >> 32), int32_t(x1 & 0xFFFFFFFF), 950 int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF))); 951 #endif 952 } 953 954 BL_INLINE_NODEBUG __m512i simd_make512_u64(uint64_t x3, uint64_t x2, uint64_t x1, uint64_t x0) noexcept { 955 #if BL_TARGET_ARCH_BITS >= 64 956 return _mm512_set_epi64(int64_t(x3), int64_t(x2), int64_t(x1), int64_t(x0), 957 int64_t(x3), int64_t(x2), int64_t(x1), int64_t(x0)); 958 #else 959 return _mm512_broadcast_i32x8( 960 _mm256_set_epi32( 961 int32_t(x3 >> 32), int32_t(x3 & 0xFFFFFFFF), 962 int32_t(x2 >> 32), int32_t(x2 & 0xFFFFFFFF), 963 int32_t(x1 >> 32), int32_t(x1 & 0xFFFFFFFF), 964 int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF))); 965 #endif 966 } 967 968 BL_INLINE_NODEBUG __m512i simd_make512_u64(uint64_t x7, uint64_t x6, uint64_t x5, uint64_t x4, 969 uint64_t x3, uint64_t x2, uint64_t x1, uint64_t x0) noexcept { 970 #if BL_TARGET_ARCH_BITS >= 64 971 return _mm512_set_epi64(int64_t(x7), int64_t(x6), int64_t(x5), int64_t(x4), 972 int64_t(x3), int64_t(x2), int64_t(x1), int64_t(x0)); 973 #else 974 return _mm512_set_epi32(int32_t(x7 >> 32), int32_t(x7 & 0xFFFFFFFF), 975 int32_t(x6 >> 32), int32_t(x6 & 0xFFFFFFFF), 976 int32_t(x5 >> 32), int32_t(x5 & 0xFFFFFFFF), 977 int32_t(x4 >> 32), int32_t(x4 & 0xFFFFFFFF), 978 int32_t(x3 >> 32), int32_t(x3 & 0xFFFFFFFF), 979 int32_t(x2 >> 32), int32_t(x2 & 0xFFFFFFFF), 980 int32_t(x1 >> 32), int32_t(x1 & 0xFFFFFFFF), 981 int32_t(x0 >> 32), int32_t(x0 & 0xFFFFFFFF)); 982 #endif 983 } 984 985 BL_INLINE_NODEBUG __m512i simd_make512_u32(uint32_t x0) noexcept { 986 return _mm512_set1_epi32(int32_t(x0)); 987 } 988 989 BL_INLINE_NODEBUG __m512i simd_make512_u32(uint32_t x1, uint32_t x0) noexcept { 990 #if BL_TARGET_ARCH_BITS >= 64 991 return _mm512_set1_epi64(int64_t((uint64_t(x1) << 32) | x0)); 992 #else 993 return _mm512_broadcast_i32x4( 994 _mm_set_epi32( 995 int32_t(x1), int32_t(x0), int32_t(x1), int32_t(x0))); 996 #endif 997 } 998 999 BL_INLINE_NODEBUG __m512i simd_make512_u32(uint32_t x3, uint32_t x2, uint32_t x1, uint32_t x0) noexcept { 1000 return _mm512_broadcast_i32x4( 1001 _mm_set_epi32( 1002 int32_t(x3), int32_t(x2), int32_t(x1), int32_t(x0))); 1003 } 1004 1005 BL_INLINE_NODEBUG __m512i simd_make512_u32(uint32_t x7, uint32_t x6, uint32_t x5, uint32_t x4, 1006 uint32_t x3, uint32_t x2, uint32_t x1, uint32_t x0) noexcept { 1007 return _mm512_broadcast_i32x8( 1008 _mm256_set_epi32( 1009 int32_t(x7), int32_t(x6), int32_t(x5), int32_t(x4), 1010 int32_t(x3), int32_t(x2), int32_t(x1), int32_t(x0))); 1011 } 1012 1013 BL_INLINE_NODEBUG __m512i simd_make512_u32(uint32_t x15, uint32_t x14, uint32_t x13, uint32_t x12, 1014 uint32_t x11, uint32_t x10, uint32_t x09, uint32_t x08, 1015 uint32_t x07, uint32_t x06, uint32_t x05, uint32_t x04, 1016 uint32_t x03, uint32_t x02, uint32_t x01, uint32_t x00) noexcept { 1017 return _mm512_set_epi32(int32_t(x15), int32_t(x14), int32_t(x13), int32_t(x12), 1018 int32_t(x11), int32_t(x10), int32_t(x09), int32_t(x08), 1019 int32_t(x07), int32_t(x06), int32_t(x05), int32_t(x04), 1020 int32_t(x03), int32_t(x02), int32_t(x01), int32_t(x00)); 1021 } 1022 1023 BL_INLINE_NODEBUG __m512i simd_make512_u16(uint16_t x0) noexcept { 1024 return _mm512_set1_epi16(int16_t(x0)); 1025 } 1026 1027 BL_INLINE_NODEBUG __m512i simd_make512_u16(uint16_t x1, uint16_t x0) noexcept { 1028 uint32_t v = (uint32_t(x1) << 16) | x0; 1029 return _mm512_set1_epi32(int32_t(v)); 1030 } 1031 1032 BL_INLINE_NODEBUG __m512i simd_make512_u16(uint16_t x3, uint16_t x2, uint16_t x1, uint16_t x0) noexcept { 1033 #if BL_TARGET_ARCH_BITS >= 64 1034 uint64_t v = (uint64_t(x3) << 48) | (uint64_t(x2) << 32) | (uint64_t(x1) << 16) | x0; 1035 return _mm512_set1_epi64(int64_t(v)); 1036 #else 1037 return _mm512_broadcast_i32x4( 1038 _mm_set_epi16( 1039 int16_t(x3), int16_t(x2), int16_t(x1), int16_t(x0), 1040 int16_t(x3), int16_t(x2), int16_t(x1), int16_t(x0))); 1041 #endif 1042 } 1043 1044 BL_INLINE_NODEBUG __m512i simd_make512_u16(uint16_t x7, uint16_t x6, uint16_t x5, uint16_t x4, 1045 uint16_t x3, uint16_t x2, uint16_t x1, uint16_t x0) noexcept { 1046 return _mm512_broadcast_i32x4( 1047 _mm_set_epi16( 1048 int16_t(x7), int16_t(x6), int16_t(x5), int16_t(x4), 1049 int16_t(x3), int16_t(x2), int16_t(x1), int16_t(x0))); 1050 } 1051 1052 BL_INLINE_NODEBUG __m512i simd_make512_u16(uint16_t x15, uint16_t x14, uint16_t x13, uint16_t x12, 1053 uint16_t x11, uint16_t x10, uint16_t x09, uint16_t x08, 1054 uint16_t x07, uint16_t x06, uint16_t x05, uint16_t x04, 1055 uint16_t x03, uint16_t x02, uint16_t x01, uint16_t x00) noexcept { 1056 return _mm512_broadcast_i32x8( 1057 _mm256_set_epi16( 1058 int16_t(x15), int16_t(x14), int16_t(x13), int16_t(x12), 1059 int16_t(x11), int16_t(x10), int16_t(x09), int16_t(x08), 1060 int16_t(x07), int16_t(x06), int16_t(x05), int16_t(x04), 1061 int16_t(x03), int16_t(x02), int16_t(x01), int16_t(x00))); 1062 } 1063 1064 BL_INLINE_NODEBUG __m512i simd_make512_u16(uint16_t x31, uint16_t x30, uint16_t x29, uint16_t x28, 1065 uint16_t x27, uint16_t x26, uint16_t x25, uint16_t x24, 1066 uint16_t x23, uint16_t x22, uint16_t x21, uint16_t x20, 1067 uint16_t x19, uint16_t x18, uint16_t x17, uint16_t x16, 1068 uint16_t x15, uint16_t x14, uint16_t x13, uint16_t x12, 1069 uint16_t x11, uint16_t x10, uint16_t x09, uint16_t x08, 1070 uint16_t x07, uint16_t x06, uint16_t x05, uint16_t x04, 1071 uint16_t x03, uint16_t x02, uint16_t x01, uint16_t x00) noexcept { 1072 #if defined(__GNUC__) && __GNUC__ < 10 && !defined(__clang__) 1073 // GCC doesn't provide this intrinsic up to GCC 9.2. 1074 uint32_t u15 = scalar_u32_from_2x_u16(x31, x30); 1075 uint32_t u14 = scalar_u32_from_2x_u16(x29, x28); 1076 uint32_t u13 = scalar_u32_from_2x_u16(x27, x26); 1077 uint32_t u12 = scalar_u32_from_2x_u16(x25, x24); 1078 uint32_t u11 = scalar_u32_from_2x_u16(x23, x22); 1079 uint32_t u10 = scalar_u32_from_2x_u16(x21, x20); 1080 uint32_t u09 = scalar_u32_from_2x_u16(x19, x18); 1081 uint32_t u08 = scalar_u32_from_2x_u16(x17, x16); 1082 uint32_t u07 = scalar_u32_from_2x_u16(x15, x14); 1083 uint32_t u06 = scalar_u32_from_2x_u16(x13, x12); 1084 uint32_t u05 = scalar_u32_from_2x_u16(x11, x10); 1085 uint32_t u04 = scalar_u32_from_2x_u16(x09, x08); 1086 uint32_t u03 = scalar_u32_from_2x_u16(x07, x06); 1087 uint32_t u02 = scalar_u32_from_2x_u16(x05, x04); 1088 uint32_t u01 = scalar_u32_from_2x_u16(x03, x02); 1089 uint32_t u00 = scalar_u32_from_2x_u16(x01, x00); 1090 return _mm512_set_epi32(u15, u14, u13, u12, u11, u10, u09, u08, 1091 u07, u06, u05, u04, u03, u02, u01, u00); 1092 #else 1093 return _mm512_set_epi16(int16_t(x31), int16_t(x30), int16_t(x29), int16_t(x28), 1094 int16_t(x27), int16_t(x26), int16_t(x25), int16_t(x24), 1095 int16_t(x23), int16_t(x22), int16_t(x21), int16_t(x20), 1096 int16_t(x19), int16_t(x18), int16_t(x17), int16_t(x16), 1097 int16_t(x15), int16_t(x14), int16_t(x13), int16_t(x12), 1098 int16_t(x11), int16_t(x10), int16_t(x09), int16_t(x08), 1099 int16_t(x07), int16_t(x06), int16_t(x05), int16_t(x04), 1100 int16_t(x03), int16_t(x02), int16_t(x01), int16_t(x00)); 1101 #endif 1102 } 1103 1104 BL_INLINE_NODEBUG __m512i simd_make512_u8(uint8_t x0) noexcept { 1105 return _mm512_set1_epi8(int8_t(x0)); 1106 } 1107 1108 BL_INLINE_NODEBUG __m512i simd_make512_u8(uint8_t x1, uint8_t x0) noexcept { 1109 uint16_t v = uint16_t((uint16_t(x1) << 8) | (uint16_t(x0) << 0)); 1110 return _mm512_set1_epi16(int16_t(v)); 1111 } 1112 1113 BL_INLINE_NODEBUG __m512i simd_make512_u8(uint8_t x3, uint8_t x2, uint8_t x1, uint8_t x0) noexcept { 1114 uint32_t v = (uint32_t(x3) << 24) | (uint32_t(x2) << 16) | (uint32_t(x1) << 8) | (uint32_t(x0) << 0); 1115 return _mm512_set1_epi32(int32_t(v)); 1116 } 1117 1118 BL_INLINE_NODEBUG __m512i simd_make512_u8(uint8_t x7, uint8_t x6, uint8_t x5, uint8_t x4, 1119 uint8_t x3, uint8_t x2, uint8_t x1, uint8_t x0) noexcept { 1120 #if BL_TARGET_ARCH_BITS >= 64 1121 uint64_t v = (uint64_t(x7) << 56) | (uint64_t(x6) << 48) | (uint64_t(x5) << 40) | (uint64_t(x4) << 32) 1122 | (uint64_t(x3) << 24) | (uint64_t(x2) << 16) | (uint64_t(x1) << 8) | (uint64_t(x0) << 0); 1123 return _mm512_set1_epi64(int64_t(v)); 1124 #else 1125 int32_t hi = int32_t((uint32_t(x7) << 24) | (uint32_t(x6) << 16) | (uint32_t(x5) << 8) | uint32_t(x4)); 1126 int32_t lo = int32_t((uint32_t(x3) << 24) | (uint32_t(x2) << 16) | (uint32_t(x1) << 8) | uint32_t(x0)); 1127 return _mm512_broadcast_i32x4(_mm_set_epi32(hi, lo, hi, lo)); 1128 #endif 1129 } 1130 1131 BL_INLINE_NODEBUG __m512i simd_make512_u8(uint8_t x15, uint8_t x14, uint8_t x13, uint8_t x12, 1132 uint8_t x11, uint8_t x10, uint8_t x09, uint8_t x08, 1133 uint8_t x07, uint8_t x06, uint8_t x05, uint8_t x04, 1134 uint8_t x03, uint8_t x02, uint8_t x01, uint8_t x00) noexcept { 1135 int32_t v3 = int32_t((uint32_t(x15) << 24) | (uint32_t(x14) << 16) | (uint32_t(x13) << 8) | uint32_t(x12)); 1136 int32_t v2 = int32_t((uint32_t(x11) << 24) | (uint32_t(x10) << 16) | (uint32_t(x09) << 8) | uint32_t(x08)); 1137 int32_t v1 = int32_t((uint32_t(x07) << 24) | (uint32_t(x06) << 16) | (uint32_t(x05) << 8) | uint32_t(x04)); 1138 int32_t v0 = int32_t((uint32_t(x03) << 24) | (uint32_t(x02) << 16) | (uint32_t(x01) << 8) | uint32_t(x00)); 1139 return _mm512_broadcast_i32x4(_mm_set_epi32(v3, v2, v1, v0)); 1140 } 1141 1142 BL_INLINE_NODEBUG __m512i simd_make512_u8(uint8_t x31, uint8_t x30, uint8_t x29, uint8_t x28, 1143 uint8_t x27, uint8_t x26, uint8_t x25, uint8_t x24, 1144 uint8_t x23, uint8_t x22, uint8_t x21, uint8_t x20, 1145 uint8_t x19, uint8_t x18, uint8_t x17, uint8_t x16, 1146 uint8_t x15, uint8_t x14, uint8_t x13, uint8_t x12, 1147 uint8_t x11, uint8_t x10, uint8_t x09, uint8_t x08, 1148 uint8_t x07, uint8_t x06, uint8_t x05, uint8_t x04, 1149 uint8_t x03, uint8_t x02, uint8_t x01, uint8_t x00) noexcept { 1150 return _mm512_broadcast_i32x8( 1151 _mm256_set_epi8( 1152 int8_t(x31), int8_t(x30), int8_t(x29), int8_t(x28), 1153 int8_t(x27), int8_t(x26), int8_t(x25), int8_t(x24), 1154 int8_t(x23), int8_t(x22), int8_t(x21), int8_t(x20), 1155 int8_t(x19), int8_t(x18), int8_t(x17), int8_t(x16), 1156 int8_t(x15), int8_t(x14), int8_t(x13), int8_t(x12), 1157 int8_t(x11), int8_t(x10), int8_t(x09), int8_t(x08), 1158 int8_t(x07), int8_t(x06), int8_t(x05), int8_t(x04), 1159 int8_t(x03), int8_t(x02), int8_t(x01), int8_t(x00))); 1160 } 1161 1162 BL_INLINE_NODEBUG __m512i simd_make512_u8(uint8_t x63, uint8_t x62, uint8_t x61, uint8_t x60, 1163 uint8_t x59, uint8_t x58, uint8_t x57, uint8_t x56, 1164 uint8_t x55, uint8_t x54, uint8_t x53, uint8_t x52, 1165 uint8_t x51, uint8_t x50, uint8_t x49, uint8_t x48, 1166 uint8_t x47, uint8_t x46, uint8_t x45, uint8_t x44, 1167 uint8_t x43, uint8_t x42, uint8_t x41, uint8_t x40, 1168 uint8_t x39, uint8_t x38, uint8_t x37, uint8_t x36, 1169 uint8_t x35, uint8_t x34, uint8_t x33, uint8_t x32, 1170 uint8_t x31, uint8_t x30, uint8_t x29, uint8_t x28, 1171 uint8_t x27, uint8_t x26, uint8_t x25, uint8_t x24, 1172 uint8_t x23, uint8_t x22, uint8_t x21, uint8_t x20, 1173 uint8_t x19, uint8_t x18, uint8_t x17, uint8_t x16, 1174 uint8_t x15, uint8_t x14, uint8_t x13, uint8_t x12, 1175 uint8_t x11, uint8_t x10, uint8_t x09, uint8_t x08, 1176 uint8_t x07, uint8_t x06, uint8_t x05, uint8_t x04, 1177 uint8_t x03, uint8_t x02, uint8_t x01, uint8_t x00) noexcept { 1178 #if defined(__GNUC__) && __GNUC__ < 10 && !defined(__clang__) 1179 // GCC doesn't provide this intrinsic up to GCC 9.2. 1180 uint32_t u15 = scalar_u32_from_4x_u8(x63, x62, x61, x60); 1181 uint32_t u14 = scalar_u32_from_4x_u8(x59, x58, x57, x56); 1182 uint32_t u13 = scalar_u32_from_4x_u8(x55, x54, x53, x52); 1183 uint32_t u12 = scalar_u32_from_4x_u8(x51, x50, x49, x48); 1184 uint32_t u11 = scalar_u32_from_4x_u8(x47, x46, x45, x44); 1185 uint32_t u10 = scalar_u32_from_4x_u8(x43, x42, x41, x40); 1186 uint32_t u09 = scalar_u32_from_4x_u8(x39, x38, x37, x36); 1187 uint32_t u08 = scalar_u32_from_4x_u8(x35, x34, x33, x32); 1188 uint32_t u07 = scalar_u32_from_4x_u8(x31, x30, x29, x28); 1189 uint32_t u06 = scalar_u32_from_4x_u8(x27, x26, x25, x24); 1190 uint32_t u05 = scalar_u32_from_4x_u8(x23, x22, x21, x20); 1191 uint32_t u04 = scalar_u32_from_4x_u8(x19, x18, x17, x16); 1192 uint32_t u03 = scalar_u32_from_4x_u8(x15, x14, x13, x12); 1193 uint32_t u02 = scalar_u32_from_4x_u8(x11, x10, x09, x08); 1194 uint32_t u01 = scalar_u32_from_4x_u8(x07, x06, x05, x04); 1195 uint32_t u00 = scalar_u32_from_4x_u8(x03, x02, x01, x00); 1196 return _mm512_set_epi32(u15, u14, u13, u12, u11, u10, u09, u08, 1197 u07, u06, u05, u04, u03, u02, u01, u00); 1198 #else 1199 return _mm512_set_epi8( 1200 int8_t(x63), int8_t(x62), int8_t(x61), int8_t(x60), 1201 int8_t(x59), int8_t(x58), int8_t(x57), int8_t(x56), 1202 int8_t(x55), int8_t(x54), int8_t(x53), int8_t(x52), 1203 int8_t(x51), int8_t(x50), int8_t(x49), int8_t(x48), 1204 int8_t(x47), int8_t(x46), int8_t(x45), int8_t(x44), 1205 int8_t(x43), int8_t(x42), int8_t(x41), int8_t(x40), 1206 int8_t(x39), int8_t(x38), int8_t(x37), int8_t(x36), 1207 int8_t(x35), int8_t(x34), int8_t(x33), int8_t(x32), 1208 int8_t(x31), int8_t(x30), int8_t(x29), int8_t(x28), 1209 int8_t(x27), int8_t(x26), int8_t(x25), int8_t(x24), 1210 int8_t(x23), int8_t(x22), int8_t(x21), int8_t(x20), 1211 int8_t(x19), int8_t(x18), int8_t(x17), int8_t(x16), 1212 int8_t(x15), int8_t(x14), int8_t(x13), int8_t(x12), 1213 int8_t(x11), int8_t(x10), int8_t(x09), int8_t(x08), 1214 int8_t(x07), int8_t(x06), int8_t(x05), int8_t(x04), 1215 int8_t(x03), int8_t(x02), int8_t(x01), int8_t(x00)); 1216 #endif 1217 } 1218 1219 BL_INLINE_NODEBUG __m512 simd_make512_f32(float x0) noexcept { 1220 return _mm512_set1_ps(x0); 1221 } 1222 1223 BL_INLINE_NODEBUG __m512 simd_make512_f32(float x1, float x0) noexcept { 1224 return _mm512_broadcast_f32x4(_mm_set_ps(x1, x0, x1, x0)); 1225 } 1226 1227 BL_INLINE_NODEBUG __m512 simd_make512_f32(float x3, float x2, float x1, float x0) noexcept { 1228 return _mm512_broadcast_f32x4(_mm_set_ps(x3, x2, x1, x0)); 1229 } 1230 1231 BL_INLINE_NODEBUG __m512 simd_make512_f32(float x7, float x6, float x5, float x4, 1232 float x3, float x2, float x1, float x0) noexcept { 1233 return _mm512_set_ps(x7, x6, x5, x4, 1234 x3, x2, x1, x0, 1235 x7, x6, x5, x4, 1236 x3, x2, x1, x0); 1237 } 1238 1239 BL_INLINE_NODEBUG __m512 simd_make512_f32(float x15, float x14, float x13, float x12, 1240 float x11, float x10, float x09, float x08, 1241 float x07, float x06, float x05, float x04, 1242 float x03, float x02, float x01, float x00) noexcept { 1243 return _mm512_set_ps(x15, x14, x13, x12, 1244 x11, x10, x09, x08, 1245 x07, x06, x05, x04, 1246 x03, x02, x01, x00); 1247 } 1248 1249 BL_INLINE_NODEBUG __m512d simd_make512_f64(double x0) noexcept { 1250 return _mm512_set1_pd(x0); 1251 } 1252 1253 BL_INLINE_NODEBUG __m512d simd_make512_f64(double x1, double x0) noexcept { 1254 return _mm512_broadcast_f64x2(_mm_set_pd(x1, x0)); 1255 } 1256 1257 BL_INLINE_NODEBUG __m512d simd_make512_f64(double x3, double x2, double x1, double x0) noexcept { 1258 return _mm512_broadcast_f64x4(_mm256_set_pd(x3, x2, x1, x0)); 1259 } 1260 1261 BL_INLINE_NODEBUG __m512d simd_make512_f64(double x7, double x6, double x5, double x4, 1262 double x3, double x2, double x1, double x0) noexcept { 1263 return _mm512_set_pd(x7, x6, x5, x4, 1264 x3, x2, x1, x0); 1265 } 1266 1267 template<> 1268 struct SimdMake<64> { 1269 template<typename... Args> static BL_INLINE_NODEBUG __m512i make_u8(Args&&... args) noexcept { return simd_make512_u8(BLInternal::forward<Args>(args)...); } 1270 template<typename... Args> static BL_INLINE_NODEBUG __m512i make_u16(Args&&... args) noexcept { return simd_make512_u16(BLInternal::forward<Args>(args)...); } 1271 template<typename... Args> static BL_INLINE_NODEBUG __m512i make_u32(Args&&... args) noexcept { return simd_make512_u32(BLInternal::forward<Args>(args)...); } 1272 template<typename... Args> static BL_INLINE_NODEBUG __m512i make_u64(Args&&... args) noexcept { return simd_make512_u64(BLInternal::forward<Args>(args)...); } 1273 template<typename... Args> static BL_INLINE_NODEBUG __m512 make_f32(Args&&... args) noexcept { return simd_make512_f32(BLInternal::forward<Args>(args)...); } 1274 template<typename... Args> static BL_INLINE_NODEBUG __m512d make_f64(Args&&... args) noexcept { return simd_make512_f64(BLInternal::forward<Args>(args)...); } 1275 }; 1276 #endif // BL_TARGET_OPT_AVX512 1277 1278 template<typename SimdT, typename... Args> BL_INLINE_NODEBUG SimdT simd_make_i8(Args&&... args) noexcept { return simd_cast<SimdT>(SimdMake<sizeof(SimdT)>::make_u8(uint8_t(BLInternal::forward<Args>(args))...)); } 1279 template<typename SimdT, typename... Args> BL_INLINE_NODEBUG SimdT simd_make_i16(Args&&... args) noexcept { return simd_cast<SimdT>(SimdMake<sizeof(SimdT)>::make_u16(uint16_t(BLInternal::forward<Args>(args))...)); } 1280 template<typename SimdT, typename... Args> BL_INLINE_NODEBUG SimdT simd_make_i32(Args&&... args) noexcept { return simd_cast<SimdT>(SimdMake<sizeof(SimdT)>::make_u32(uint32_t(BLInternal::forward<Args>(args))...)); } 1281 template<typename SimdT, typename... Args> BL_INLINE_NODEBUG SimdT simd_make_i64(Args&&... args) noexcept { return simd_cast<SimdT>(SimdMake<sizeof(SimdT)>::make_u64(uint64_t(BLInternal::forward<Args>(args))...)); } 1282 template<typename SimdT, typename... Args> BL_INLINE_NODEBUG SimdT simd_make_u8(Args&&... args) noexcept { return simd_cast<SimdT>(SimdMake<sizeof(SimdT)>::make_u8(uint8_t(BLInternal::forward<Args>(args))...)); } 1283 template<typename SimdT, typename... Args> BL_INLINE_NODEBUG SimdT simd_make_u16(Args&&... args) noexcept { return simd_cast<SimdT>(SimdMake<sizeof(SimdT)>::make_u16(uint16_t(BLInternal::forward<Args>(args))...)); } 1284 template<typename SimdT, typename... Args> BL_INLINE_NODEBUG SimdT simd_make_u32(Args&&... args) noexcept { return simd_cast<SimdT>(SimdMake<sizeof(SimdT)>::make_u32(uint32_t(BLInternal::forward<Args>(args))...)); } 1285 template<typename SimdT, typename... Args> BL_INLINE_NODEBUG SimdT simd_make_u64(Args&&... args) noexcept { return simd_cast<SimdT>(SimdMake<sizeof(SimdT)>::make_u64(uint64_t(BLInternal::forward<Args>(args))...)); } 1286 template<typename SimdT, typename... Args> BL_INLINE_NODEBUG SimdT simd_make_f32(Args&&... args) noexcept { return simd_cast<SimdT>(SimdMake<sizeof(SimdT)>::make_f32(float(BLInternal::forward<Args>(args))...)); } 1287 template<typename SimdT, typename... Args> BL_INLINE_NODEBUG SimdT simd_make_f64(Args&&... args) noexcept { return simd_cast<SimdT>(SimdMake<sizeof(SimdT)>::make_f64(double(BLInternal::forward<Args>(args))...)); } 1288 1289 } // {Internal} 1290 1291 // SIMD - Internal - Cast Vector <-> Scalar 1292 // ======================================== 1293 1294 namespace Internal { 1295 1296 BL_INLINE_NODEBUG __m128i simd_cast_from_u32(uint32_t val) noexcept { return _mm_cvtsi32_si128(int(val)); } 1297 BL_INLINE_NODEBUG uint32_t simd_cast_to_u32(const __m128i& src) noexcept { return uint32_t(_mm_cvtsi128_si32(src)); } 1298 1299 #if BL_TARGET_ARCH_BITS >= 64 1300 BL_INLINE_NODEBUG __m128i simd_cast_from_u64(uint64_t val) noexcept { return _mm_cvtsi64_si128(int64_t(val)); } 1301 BL_INLINE_NODEBUG uint64_t simd_cast_to_u64(const __m128i& src) noexcept { return uint64_t(_mm_cvtsi128_si64(src)); } 1302 #else 1303 BL_INLINE_NODEBUG __m128i simd_cast_from_u64(uint64_t val) noexcept { return _mm_loadl_epi64(reinterpret_cast<const __m128i*>(&val)); } 1304 BL_INLINE_NODEBUG uint64_t simd_cast_to_u64(const __m128i& src) noexcept { 1305 uint64_t result; 1306 _mm_storel_epi64(reinterpret_cast<__m128i*>(&result), src); 1307 return result; 1308 } 1309 #endif 1310 1311 #if defined(__GNUC__) && !defined(__clang__) 1312 // See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70708 1313 BL_INLINE_NODEBUG __m128 simd_cast_from_f32(float val) noexcept { __m128 reg; __asm__("" : "=x" (reg) : "0" (val)); return reg; } 1314 BL_INLINE_NODEBUG __m128d simd_cast_from_f64(double val) noexcept { __m128d reg; __asm__("" : "=x" (reg) : "0" (val)); return reg; } 1315 #else 1316 BL_INLINE_NODEBUG __m128 simd_cast_from_f32(float val) noexcept { return _mm_set_ss(val); } 1317 BL_INLINE_NODEBUG __m128d simd_cast_from_f64(double val) noexcept { return _mm_set_sd(val); } 1318 #endif 1319 1320 BL_INLINE_NODEBUG float simd_cast_to_f32(const __m128& src) noexcept { return _mm_cvtss_f32(src); } 1321 BL_INLINE_NODEBUG double simd_cast_to_f64(const __m128d& src) noexcept { return _mm_cvtsd_f64(src); } 1322 1323 } // {Internal} 1324 1325 // SIMD - Internal - Convert Vector <-> Vector 1326 // =========================================== 1327 1328 namespace Internal { 1329 1330 BL_INLINE_NODEBUG __m128 simd_cvt_i32_f32(const __m128i& a) noexcept { return _mm_cvtepi32_ps(a); } 1331 BL_INLINE_NODEBUG __m128i simd_cvt_f32_i32(const __m128& a) noexcept { return _mm_cvtps_epi32(a); } 1332 BL_INLINE_NODEBUG __m128i simd_cvtt_f32_i32(const __m128& a) noexcept { return _mm_cvttps_epi32(a); } 1333 1334 /* 1335 // TODO: SIMD WRAP 1336 BL_INLINE_NODEBUG __m128i simd_cvt_f64_i32(const __m128d& a) noexcept { return _mm_cvtpd_epi32(a); } 1337 BL_INLINE_NODEBUG __m128i simd_cvtt_f64_i32(const __m128d& a) noexcept { return _mm_cvttpd_epi32(a); } 1338 BL_INLINE_NODEBUG __m128 simd_cvt_f64_f32(const __m128d& a) noexcept { return _mm_cvtpd_ps(a); } 1339 BL_INLINE_NODEBUG __m128d simd_cvt_2xi32_f64(const __m128i& a) noexcept { return _mm_cvtepi32_pd(a); } 1340 BL_INLINE_NODEBUG __m128d simd_cvt_f32x2_f64(const __m128& a) noexcept { return _mm_cvtps_pd(a); } 1341 */ 1342 1343 #if defined(BL_TARGET_OPT_AVX2) 1344 BL_INLINE_NODEBUG __m256 simd_cvt_i32_f32(const __m256i& a) noexcept { return _mm256_cvtepi32_ps(a); } 1345 BL_INLINE_NODEBUG __m256i simd_cvt_f32_i32(const __m256& a) noexcept { return _mm256_cvtps_epi32(a); } 1346 BL_INLINE_NODEBUG __m256i simd_cvtt_f32_i32(const __m256& a) noexcept { return _mm256_cvttps_epi32(a); } 1347 #endif // BL_TARGET_OPT_AVX2 1348 1349 #if defined(BL_TARGET_OPT_AVX512) 1350 BL_INLINE_NODEBUG __m512 simd_cvt_i32_f32(const __m512i& a) noexcept { return _mm512_cvtepi32_ps(a); } 1351 BL_INLINE_NODEBUG __m512i simd_cvt_f32_i32(const __m512& a) noexcept { return _mm512_cvtps_epi32(a); } 1352 BL_INLINE_NODEBUG __m512i simd_cvtt_f32_i32(const __m512& a) noexcept { return _mm512_cvttps_epi32(a); } 1353 #endif // BL_TARGET_OPT_AVX512 1354 1355 } // {Internal} 1356 1357 // SIMD - Internal - Convert Vector <-> Scalar 1358 // =========================================== 1359 1360 namespace Internal { 1361 1362 BL_INLINE_NODEBUG __m128 simd_cvt_f32_from_scalar_i32(int32_t val) noexcept { return _mm_cvtsi32_ss(_mm_setzero_ps(), val); } 1363 BL_INLINE_NODEBUG __m128d simd_cvt_f64_from_scalar_i32(int32_t val) noexcept { return _mm_cvtsi32_sd(_mm_setzero_pd(), val); } 1364 1365 BL_INLINE_NODEBUG int32_t simd_cvt_f32_to_scalar_i32(const __m128& src) noexcept { return _mm_cvtss_si32(src); } 1366 BL_INLINE_NODEBUG int32_t simd_cvt_f64_to_scalar_i32(const __m128d& src) noexcept { return _mm_cvtsd_si32(src); } 1367 BL_INLINE_NODEBUG int32_t simd_cvtt_f32_to_scalar_i32(const __m128& src) noexcept { return _mm_cvttss_si32(src); } 1368 BL_INLINE_NODEBUG int32_t simd_cvtt_f64_to_scalar_i32(const __m128d& src) noexcept { return _mm_cvttsd_si32(src); } 1369 1370 #if BL_TARGET_ARCH_BITS >= 64 1371 BL_INLINE_NODEBUG __m128 simd_cvt_f32_from_scalar_i64(int64_t val) noexcept { return _mm_cvtsi64_ss(_mm_setzero_ps(), val); } 1372 BL_INLINE_NODEBUG __m128d simd_cvt_f64_from_scalar_i64(int64_t val) noexcept { return _mm_cvtsi64_sd(_mm_setzero_pd(), val); } 1373 1374 BL_INLINE_NODEBUG int64_t simd_cvt_f32_to_scalar_i64(const __m128& src) noexcept { return _mm_cvtss_si64(src); } 1375 BL_INLINE_NODEBUG int64_t simd_cvt_f64_to_scalar_i64(const __m128d& src) noexcept { return _mm_cvtsd_si64(src); } 1376 BL_INLINE_NODEBUG int64_t simd_cvtt_f32_to_scalar_i64(const __m128& src) noexcept { return _mm_cvttss_si64(src); } 1377 BL_INLINE_NODEBUG int64_t simd_cvtt_f64_to_scalar_i64(const __m128d& src) noexcept { return _mm_cvttsd_si64(src); } 1378 #endif 1379 1380 } // {Internal} 1381 1382 // SIMD - Internal - Convert Vector <-> Mask 1383 // ========================================= 1384 1385 namespace Internal { 1386 1387 #if defined(BL_TARGET_OPT_AVX512) 1388 BL_INLINE_NODEBUG __m128i simd_128i_from_mask8(__mmask16 k) noexcept { return _mm_movm_epi8(k); } 1389 BL_INLINE_NODEBUG __m128 simd_128f_from_mask8(__mmask16 k) noexcept { return simd_as_f(simd_128i_from_mask8(k)); } 1390 BL_INLINE_NODEBUG __m128d simd_128d_from_mask8(__mmask16 k) noexcept { return simd_as_d(simd_128i_from_mask8(k)); } 1391 1392 BL_INLINE_NODEBUG __m256i simd_256i_from_mask8(__mmask32 k) noexcept { return _mm256_movm_epi8(k); } 1393 BL_INLINE_NODEBUG __m256 simd_256f_from_mask8(__mmask32 k) noexcept { return simd_as_f(simd_256i_from_mask8(k)); } 1394 BL_INLINE_NODEBUG __m256d simd_256d_from_mask8(__mmask32 k) noexcept { return simd_as_d(simd_256i_from_mask8(k)); } 1395 1396 BL_INLINE_NODEBUG __m512i simd_512i_from_mask8(__mmask64 k) noexcept { return _mm512_movm_epi8(k); } 1397 BL_INLINE_NODEBUG __m512 simd_512f_from_mask8(__mmask64 k) noexcept { return simd_as_f(simd_512i_from_mask8(k)); } 1398 BL_INLINE_NODEBUG __m512d simd_512d_from_mask8(__mmask64 k) noexcept { return simd_as_d(simd_512i_from_mask8(k)); } 1399 1400 BL_INLINE_NODEBUG __m128i simd_128i_from_mask16(__mmask8 k) noexcept { return _mm_movm_epi16(k); } 1401 BL_INLINE_NODEBUG __m128 simd_128f_from_mask16(__mmask8 k) noexcept { return simd_as_f(simd_128i_from_mask8(k)); } 1402 BL_INLINE_NODEBUG __m128d simd_128d_from_mask16(__mmask8 k) noexcept { return simd_as_d(simd_128i_from_mask8(k)); } 1403 1404 BL_INLINE_NODEBUG __m256i simd_256i_from_mask16(__mmask16 k) noexcept { return _mm256_movm_epi16(k); } 1405 BL_INLINE_NODEBUG __m256 simd_256f_from_mask16(__mmask16 k) noexcept { return simd_as_f(simd_256i_from_mask8(k)); } 1406 BL_INLINE_NODEBUG __m256d simd_256d_from_mask16(__mmask16 k) noexcept { return simd_as_d(simd_256i_from_mask8(k)); } 1407 1408 BL_INLINE_NODEBUG __m512i simd_512i_from_mask16(__mmask32 k) noexcept { return _mm512_movm_epi16(k); } 1409 BL_INLINE_NODEBUG __m512 simd_512f_from_mask16(__mmask32 k) noexcept { return simd_as_f(simd_512i_from_mask8(k)); } 1410 BL_INLINE_NODEBUG __m512d simd_512d_from_mask16(__mmask32 k) noexcept { return simd_as_d(simd_512i_from_mask8(k)); } 1411 1412 BL_INLINE_NODEBUG __m128i simd_128i_from_mask32(__mmask8 k) noexcept { return _mm_movm_epi32(k); } 1413 BL_INLINE_NODEBUG __m128 simd_128f_from_mask32(__mmask8 k) noexcept { return simd_as_f(simd_128i_from_mask8(k)); } 1414 BL_INLINE_NODEBUG __m128d simd_128d_from_mask32(__mmask8 k) noexcept { return simd_as_d(simd_128i_from_mask8(k)); } 1415 1416 BL_INLINE_NODEBUG __m256i simd_256i_from_mask32(__mmask8 k) noexcept { return _mm256_movm_epi32(k); } 1417 BL_INLINE_NODEBUG __m256 simd_256f_from_mask32(__mmask8 k) noexcept { return simd_as_f(simd_256i_from_mask8(k)); } 1418 BL_INLINE_NODEBUG __m256d simd_256d_from_mask32(__mmask8 k) noexcept { return simd_as_d(simd_256i_from_mask8(k)); } 1419 1420 BL_INLINE_NODEBUG __m512i simd_512i_from_mask32(__mmask16 k) noexcept { return _mm512_movm_epi32(k); } 1421 BL_INLINE_NODEBUG __m512 simd_512f_from_mask32(__mmask16 k) noexcept { return simd_as_f(simd_512i_from_mask8(k)); } 1422 BL_INLINE_NODEBUG __m512d simd_512d_from_mask32(__mmask16 k) noexcept { return simd_as_d(simd_512i_from_mask8(k)); } 1423 1424 BL_INLINE_NODEBUG __m128i simd_128i_from_mask64(__mmask8 k) noexcept { return _mm_movm_epi64(k); } 1425 BL_INLINE_NODEBUG __m128 simd_128f_from_mask64(__mmask8 k) noexcept { return simd_as_f(simd_128i_from_mask8(k)); } 1426 BL_INLINE_NODEBUG __m128d simd_128d_from_mask64(__mmask8 k) noexcept { return simd_as_d(simd_128i_from_mask8(k)); } 1427 1428 BL_INLINE_NODEBUG __m256i simd_256i_from_mask64(__mmask8 k) noexcept { return _mm256_movm_epi64(k); } 1429 BL_INLINE_NODEBUG __m256 simd_256f_from_mask64(__mmask8 k) noexcept { return simd_as_f(simd_256i_from_mask8(k)); } 1430 BL_INLINE_NODEBUG __m256d simd_256d_from_mask64(__mmask8 k) noexcept { return simd_as_d(simd_256i_from_mask8(k)); } 1431 1432 BL_INLINE_NODEBUG __m512i simd_512i_from_mask64(__mmask8 k) noexcept { return _mm512_movm_epi64(k); } 1433 BL_INLINE_NODEBUG __m512 simd_512f_from_mask64(__mmask8 k) noexcept { return simd_as_f(simd_512i_from_mask8(k)); } 1434 BL_INLINE_NODEBUG __m512d simd_512d_from_mask64(__mmask8 k) noexcept { return simd_as_d(simd_512i_from_mask8(k)); } 1435 #endif // BL_TARGET_OPT_AVX512 1436 1437 } // {Internal} 1438 1439 // SIMD - Internal - Shuffle & Permute 1440 // =================================== 1441 1442 namespace Internal { 1443 1444 template<size_t kW> BL_INLINE_NODEBUG typename SimdInfo<kW>::RegI simd_broadcast_u8(const __m128i& a) noexcept; 1445 template<size_t kW> BL_INLINE_NODEBUG typename SimdInfo<kW>::RegI simd_broadcast_u16(const __m128i& a) noexcept; 1446 template<size_t kW> BL_INLINE_NODEBUG typename SimdInfo<kW>::RegI simd_broadcast_u32(const __m128i& a) noexcept; 1447 template<size_t kW> BL_INLINE_NODEBUG typename SimdInfo<kW>::RegI simd_broadcast_u64(const __m128i& a) noexcept; 1448 template<size_t kW> BL_INLINE_NODEBUG typename SimdInfo<kW>::RegF simd_broadcast_f32(const __m128& a) noexcept; 1449 template<size_t kW> BL_INLINE_NODEBUG typename SimdInfo<kW>::RegD simd_broadcast_f64(const __m128d& a) noexcept; 1450 1451 #if defined(BL_TARGET_OPT_SSSE3) 1452 BL_INLINE_NODEBUG __m128i simd_swizzlev_u8(const __m128i& a, const __m128i& b) noexcept { return _mm_shuffle_epi8(a, b); } 1453 #endif // BL_TARGET_OPT_SSSE3 1454 1455 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1456 BL_INLINE_NODEBUG __m128i simd_swizzle_lo_u16(const __m128i& a) noexcept { return _mm_shufflelo_epi16(a, _MM_SHUFFLE(D, C, B, A)); } 1457 1458 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1459 BL_INLINE_NODEBUG __m128i simd_swizzle_hi_u16(const __m128i& a) noexcept { return _mm_shufflehi_epi16(a, _MM_SHUFFLE(D, C, B, A)); } 1460 1461 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1462 BL_INLINE_NODEBUG __m128i simd_swizzle_u16(const __m128i& a) noexcept { return simd_swizzle_hi_u16<D, C, B, A>(simd_swizzle_lo_u16<D, C, B, A>(a)); } 1463 1464 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1465 BL_INLINE_NODEBUG __m128i simd_swizzle_u32(const __m128i& a) noexcept { return _mm_shuffle_epi32(a, _MM_SHUFFLE(D, C, B, A)); } 1466 1467 template<uint8_t B, uint8_t A> 1468 BL_INLINE_NODEBUG __m128i simd_swizzle_u64(const __m128i& a) noexcept { return simd_swizzle_u32<B*2 + 1, B*2, A*2 + 1, A*2>(a); } 1469 1470 #if defined(BL_TARGET_OPT_AVX) 1471 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1472 BL_INLINE_NODEBUG __m128 simd_swizzle_f32(const __m128& a) noexcept { return _mm_shuffle_ps(a, a, _MM_SHUFFLE(D, C, B, A)); } 1473 template<uint8_t B, uint8_t A> 1474 BL_INLINE_NODEBUG __m128d simd_swizzle_f64(const __m128d& a) noexcept { return _mm_shuffle_pd(a, a, (B << 1) | A); } 1475 #else 1476 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1477 BL_INLINE_NODEBUG __m128 simd_swizzle_f32(const __m128& a) noexcept { return simd_cast<__m128>(_mm_shuffle_epi32(simd_cast<__m128i>(a), _MM_SHUFFLE(D, C, B, A))); } 1478 template<uint8_t B, uint8_t A> 1479 BL_INLINE_NODEBUG __m128d simd_swizzle_f64(const __m128d& a) noexcept { return simd_cast<__m128d>(simd_swizzle_u32<B*2 + 1, B*2, A*2 + 1, A*2>(simd_cast<__m128i>(a))); } 1480 #endif // BL_TARGET_OPT_AVX 1481 1482 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1483 BL_INLINE_NODEBUG __m128 simd_shuffle_f32(const __m128& lo, const __m128& hi) noexcept { return _mm_shuffle_ps(lo, hi, _MM_SHUFFLE(D, C, B, A)); } 1484 1485 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1486 BL_INLINE_NODEBUG __m128i simd_shuffle_u32(const __m128i& lo, const __m128i& hi) noexcept { return simd_cast<__m128i>(_mm_shuffle_ps(simd_cast<__m128>(lo), simd_cast<__m128>(hi), _MM_SHUFFLE(D, C, B, A))); } 1487 1488 template<uint8_t B, uint8_t A> 1489 BL_INLINE_NODEBUG __m128d simd_shuffle_f64(const __m128d& lo, const __m128d& hi) noexcept { return _mm_shuffle_pd(lo, hi, (B << 1) | A); } 1490 1491 template<uint8_t B, uint8_t A> 1492 BL_INLINE_NODEBUG __m128i simd_shuffle_u64(const __m128i& lo, const __m128i& hi) noexcept { return simd_cast<__m128i>(_mm_shuffle_pd(simd_cast<__m128d>(lo), simd_cast<__m128d>(hi), (B << 1) | A)); } 1493 1494 #if defined(BL_TARGET_OPT_AVX2) && !defined(BL_SIMD_MISSING_INTRINSICS) 1495 BL_INLINE_NODEBUG __m256i simd_interleave_u128(const __m128i& a, const __m128i& b) noexcept { return _mm256_set_m128i(b, a); } 1496 #elif defined(BL_TARGET_OPT_AVX) 1497 BL_INLINE_NODEBUG __m256i simd_interleave_u128(const __m128i& a, const __m128i& b) noexcept { return _mm256_insertf128_si256(simd_cast<__m256i>(a), b, 1); } 1498 #endif 1499 1500 #if defined(BL_TARGET_OPT_AVX2) 1501 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1502 BL_INLINE_NODEBUG __m256i simd_permute_u64(const __m128i& a) noexcept { return _mm256_permute4x64_epi64(simd_cast<__m256i>(a), _MM_SHUFFLE(D, C, B, A)); } 1503 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1504 BL_INLINE_NODEBUG __m256i simd_permute_u64(const __m256i& a) noexcept { return _mm256_permute4x64_epi64(a, _MM_SHUFFLE(D, C, B, A)); } 1505 1506 template<uint8_t B, uint8_t A> 1507 BL_INLINE_NODEBUG __m256i simd_permute_u128(const __m256i& a, const __m256i& b) noexcept { return _mm256_permute2x128_si256(a, b, ((B & 0xF) << 4) + (A & 0xF)); } 1508 template<uint8_t B, uint8_t A> 1509 BL_INLINE_NODEBUG __m256i simd_permute_u128(const __m256i& a) noexcept { return simd_permute_u128<B, A>(a, a); } 1510 #endif // BL_TARGET_OPT_AVX2 1511 1512 #if defined(BL_TARGET_OPT_AVX2) 1513 template<> BL_INLINE_NODEBUG __m128i simd_broadcast_u8<16>(const __m128i& a) noexcept { return _mm_broadcastb_epi8(a); } 1514 template<> BL_INLINE_NODEBUG __m128i simd_broadcast_u16<16>(const __m128i& a) noexcept { return _mm_broadcastw_epi16(a); } 1515 template<> BL_INLINE_NODEBUG __m128i simd_broadcast_u32<16>(const __m128i& a) noexcept { return _mm_broadcastd_epi32(a); } 1516 template<> BL_INLINE_NODEBUG __m128i simd_broadcast_u64<16>(const __m128i& a) noexcept { return _mm_broadcastq_epi64(a); } 1517 #elif defined(BL_TARGET_OPT_SSSE3) 1518 template<> BL_INLINE_NODEBUG __m128i simd_broadcast_u8<16>(const __m128i& a) noexcept { return _mm_shuffle_epi8(a, _mm_setzero_si128()); } 1519 template<> BL_INLINE_NODEBUG __m128i simd_broadcast_u16<16>(const __m128i& a) noexcept { return _mm_shuffle_epi8(a, bl::common_table.p_0100010001000100.as<__m128i>()); } 1520 template<> BL_INLINE_NODEBUG __m128i simd_broadcast_u32<16>(const __m128i& a) noexcept { return _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 0)); } 1521 template<> BL_INLINE_NODEBUG __m128i simd_broadcast_u64<16>(const __m128i& a) noexcept { return _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 1, 0)); } 1522 #else 1523 template<> BL_INLINE_NODEBUG __m128i simd_broadcast_u16<16>(const __m128i& a) noexcept { return _mm_shuffle_epi32(_mm_unpacklo_epi16(a, a), _MM_SHUFFLE(0, 0, 0, 0)); } 1524 template<> BL_INLINE_NODEBUG __m128i simd_broadcast_u8<16>(const __m128i& a) noexcept { return simd_broadcast_u16<16>(_mm_unpacklo_epi8(a, a)); } 1525 template<> BL_INLINE_NODEBUG __m128i simd_broadcast_u32<16>(const __m128i& a) noexcept { return _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 0, 0, 0)); } 1526 template<> BL_INLINE_NODEBUG __m128i simd_broadcast_u64<16>(const __m128i& a) noexcept { return _mm_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 1, 0)); } 1527 #endif 1528 1529 #if defined(BL_TARGET_OPT_AVX2) && !defined(BL_SIMD_MISSING_INTRINSICS) 1530 template<> BL_INLINE_NODEBUG __m128 simd_broadcast_f32<16>(const __m128& a) noexcept { return _mm_broadcastss_ps(a); } 1531 template<> BL_INLINE_NODEBUG __m128d simd_broadcast_f64<16>(const __m128d& a) noexcept { return _mm_broadcastsd_pd(a); } 1532 #else 1533 template<> BL_INLINE_NODEBUG __m128 simd_broadcast_f32<16>(const __m128& a) noexcept { return simd_cast<__m128>(simd_broadcast_u32<16>(simd_cast<__m128i>(a))); } 1534 template<> BL_INLINE_NODEBUG __m128d simd_broadcast_f64<16>(const __m128d& a) noexcept { return simd_cast<__m128d>(simd_broadcast_u64<16>(simd_cast<__m128i>(a))); } 1535 #endif 1536 1537 BL_INLINE_NODEBUG __m128i simd_dup_lo_u32(const __m128i& a) noexcept { return simd_swizzle_u32<2, 2, 0, 0>(a); } 1538 BL_INLINE_NODEBUG __m128i simd_dup_hi_u32(const __m128i& a) noexcept { return simd_swizzle_u32<3, 3, 1, 1>(a); } 1539 1540 BL_INLINE_NODEBUG __m128i simd_dup_lo_u64(const __m128i& a) noexcept { return simd_swizzle_u64<0, 0>(a); } 1541 BL_INLINE_NODEBUG __m128i simd_dup_hi_u64(const __m128i& a) noexcept { return simd_swizzle_u64<1, 1>(a); } 1542 1543 BL_INLINE_NODEBUG __m128 simd_dup_lo_f32(const __m128& a) noexcept { return simd_swizzle_f32<2, 2, 0, 0>(a); } 1544 BL_INLINE_NODEBUG __m128 simd_dup_hi_f32(const __m128& a) noexcept { return simd_swizzle_f32<3, 3, 1, 1>(a); } 1545 1546 BL_INLINE_NODEBUG __m128 simd_dup_lo_f32x2(const __m128& a) noexcept { return simd_swizzle_f32<1, 0, 1, 0>(a); } 1547 BL_INLINE_NODEBUG __m128 simd_dup_hi_f32x2(const __m128& a) noexcept { return simd_swizzle_f32<3, 2, 3, 2>(a); } 1548 1549 BL_INLINE_NODEBUG __m128d simd_dup_lo_f64(const __m128d& a) noexcept { return simd_swizzle_f64<0, 0>(a); } 1550 BL_INLINE_NODEBUG __m128d simd_dup_hi_f64(const __m128d& a) noexcept { return simd_swizzle_f64<1, 1>(a); } 1551 1552 BL_INLINE_NODEBUG __m128i simd_swap_u32(const __m128i& a) noexcept { return simd_swizzle_u32<2, 3, 0, 1>(a); } 1553 BL_INLINE_NODEBUG __m128 simd_swap_f32(const __m128& a) noexcept { return simd_swizzle_f32<2, 3, 0, 1>(a); } 1554 BL_INLINE_NODEBUG __m128i simd_swap_u64(const __m128i& a) noexcept { return simd_swizzle_u64<0, 1>(a); } 1555 BL_INLINE_NODEBUG __m128d simd_swap_f64(const __m128d& a) noexcept { return simd_swizzle_f64<0, 1>(a); } 1556 1557 BL_INLINE_NODEBUG __m128i simd_interleave_lo_u8(const __m128i& a, const __m128i& b) noexcept { return _mm_unpacklo_epi8(a, b); } 1558 BL_INLINE_NODEBUG __m128i simd_interleave_hi_u8(const __m128i& a, const __m128i& b) noexcept { return _mm_unpackhi_epi8(a, b); } 1559 BL_INLINE_NODEBUG __m128i simd_interleave_lo_u16(const __m128i& a, const __m128i& b) noexcept { return _mm_unpacklo_epi16(a, b); } 1560 BL_INLINE_NODEBUG __m128i simd_interleave_hi_u16(const __m128i& a, const __m128i& b) noexcept { return _mm_unpackhi_epi16(a, b); } 1561 BL_INLINE_NODEBUG __m128i simd_interleave_lo_u32(const __m128i& a, const __m128i& b) noexcept { return _mm_unpacklo_epi32(a, b); } 1562 BL_INLINE_NODEBUG __m128i simd_interleave_hi_u32(const __m128i& a, const __m128i& b) noexcept { return _mm_unpackhi_epi32(a, b); } 1563 BL_INLINE_NODEBUG __m128i simd_interleave_lo_u64(const __m128i& a, const __m128i& b) noexcept { return _mm_unpacklo_epi64(a, b); } 1564 BL_INLINE_NODEBUG __m128i simd_interleave_hi_u64(const __m128i& a, const __m128i& b) noexcept { return _mm_unpackhi_epi64(a, b); } 1565 1566 BL_INLINE_NODEBUG __m128 simd_interleave_lo_f32(const __m128& a, const __m128& b) noexcept { return _mm_unpacklo_ps(a, b); } 1567 BL_INLINE_NODEBUG __m128 simd_interleave_hi_f32(const __m128& a, const __m128& b) noexcept { return _mm_unpackhi_ps(a, b); } 1568 BL_INLINE_NODEBUG __m128d simd_interleave_lo_f64(const __m128d& a, const __m128d& b) noexcept { return _mm_unpacklo_pd(a, b); } 1569 BL_INLINE_NODEBUG __m128d simd_interleave_hi_f64(const __m128d& a, const __m128d& b) noexcept { return _mm_unpackhi_pd(a, b); } 1570 1571 #if defined(BL_TARGET_OPT_SSSE3) 1572 template<int kNumBytes> 1573 BL_INLINE_NODEBUG __m128i simd_alignr_u128(const __m128i& a, const __m128i& b) noexcept { return _mm_alignr_epi8(a, b, kNumBytes); } 1574 #else 1575 template<int kNumBytes> 1576 BL_INLINE_NODEBUG __m128i simd_alignr_u128(const __m128i& a, const __m128i& b) noexcept { 1577 __m128i a_shifted = _mm_slli_si128(a, (16u - kNumBytes) % 16u); 1578 __m128i b_shifted = _mm_srli_si128(b, kNumBytes); 1579 return kNumBytes > 0u ? _mm_or_si128(a_shifted, b_shifted) : a; 1580 } 1581 #endif 1582 1583 #if defined(BL_TARGET_OPT_AVX) 1584 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1585 BL_INLINE_NODEBUG __m256 simd_swizzle_f32(const __m256& a) noexcept { return _mm256_shuffle_ps(a, a, _MM_SHUFFLE(D, C, B, A)); } 1586 1587 template<uint8_t B, uint8_t A> 1588 BL_INLINE_NODEBUG __m256d simd_swizzle_f64(const __m256d& a) noexcept { return _mm256_shuffle_pd(a, a, (B << 3) | (A << 2) | (B << 1) | A); } 1589 1590 #if defined(BL_TARGET_OPT_AVX2) 1591 template<> BL_INLINE_NODEBUG __m256i simd_broadcast_u8<32>(const __m128i& a) noexcept { return _mm256_broadcastb_epi8(a); } 1592 template<> BL_INLINE_NODEBUG __m256i simd_broadcast_u16<32>(const __m128i& a) noexcept { return _mm256_broadcastw_epi16(a); } 1593 template<> BL_INLINE_NODEBUG __m256i simd_broadcast_u32<32>(const __m128i& a) noexcept { return _mm256_broadcastd_epi32(a); } 1594 template<> BL_INLINE_NODEBUG __m256i simd_broadcast_u64<32>(const __m128i& a) noexcept { return _mm256_broadcastq_epi64(a); } 1595 #else 1596 template<> BL_INLINE_NODEBUG __m256i simd_broadcast_u32<32>(const __m128i& a) noexcept { return simd_cast<__m256i>(_mm256_broadcastss_ps(simd_cast<__m128>(a))); } 1597 template<> BL_INLINE_NODEBUG __m256i simd_broadcast_u64<32>(const __m128i& a) noexcept { return simd_cast<__m256i>(_mm256_broadcastsd_pd(simd_cast<__m128d>(a))); } 1598 template<> BL_INLINE_NODEBUG __m256i simd_broadcast_u8<32>(const __m128i& a) noexcept { return simd_broadcast_u64<32>(_mm_shuffle_epi8(a, _mm_setzero_si128())); } 1599 template<> BL_INLINE_NODEBUG __m256i simd_broadcast_u16<32>(const __m128i& a) noexcept { return simd_broadcast_u64<32>(_mm_shufflelo_epi16(a, _MM_SHUFFLE(0, 0, 0, 0))); } 1600 #endif 1601 1602 template<> BL_INLINE_NODEBUG __m256 simd_broadcast_f32<32>(const __m128& a) noexcept { return _mm256_broadcastss_ps(a); } 1603 template<> BL_INLINE_NODEBUG __m256d simd_broadcast_f64<32>(const __m128d& a) noexcept { return _mm256_broadcastsd_pd(a); } 1604 1605 BL_INLINE_NODEBUG __m256 simd_dup_lo_f32(const __m256& a) noexcept { return simd_swizzle_f32<2, 2, 0, 0>(a); } 1606 BL_INLINE_NODEBUG __m256 simd_dup_hi_f32(const __m256& a) noexcept { return simd_swizzle_f32<3, 3, 1, 1>(a); } 1607 1608 BL_INLINE_NODEBUG __m256 simd_dup_lo_f32x2(const __m256& a) noexcept { return simd_swizzle_f32<1, 0, 1, 0>(a); } 1609 BL_INLINE_NODEBUG __m256 simd_dup_hi_f32x2(const __m256& a) noexcept { return simd_swizzle_f32<3, 2, 3, 2>(a); } 1610 1611 BL_INLINE_NODEBUG __m256d simd_dup_lo_f64(const __m256d& a) noexcept { return simd_swizzle_f64<0, 0>(a); } 1612 BL_INLINE_NODEBUG __m256d simd_dup_hi_f64(const __m256d& a) noexcept { return simd_swizzle_f64<1, 1>(a); } 1613 1614 BL_INLINE_NODEBUG __m256 simd_swap_f32(const __m256& a) noexcept { return simd_swizzle_f32<2, 3, 0, 1>(a); } 1615 BL_INLINE_NODEBUG __m256d simd_swap_f64(const __m256d& a) noexcept { return simd_swizzle_f64<0, 1>(a); } 1616 1617 BL_INLINE_NODEBUG __m256 simd_interleave_lo_f32(const __m256& a, const __m256& b) noexcept { return _mm256_unpacklo_ps(a, b); } 1618 BL_INLINE_NODEBUG __m256 simd_interleave_hi_f32(const __m256& a, const __m256& b) noexcept { return _mm256_unpackhi_ps(a, b); } 1619 BL_INLINE_NODEBUG __m256d simd_interleave_lo_f64(const __m256d& a, const __m256d& b) noexcept { return _mm256_unpacklo_pd(a, b); } 1620 BL_INLINE_NODEBUG __m256d simd_interleave_hi_f64(const __m256d& a, const __m256d& b) noexcept { return _mm256_unpackhi_pd(a, b); } 1621 #endif // BL_TARGET_OPT_AVX 1622 1623 #if defined(BL_TARGET_OPT_AVX2) 1624 BL_INLINE_NODEBUG __m256i simd_swizzlev_u8(const __m256i& a, const __m256i& b) noexcept { return _mm256_shuffle_epi8(a, b); } 1625 1626 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1627 BL_INLINE_NODEBUG __m256i simd_swizzle_lo_u16(const __m256i& a) noexcept { return _mm256_shufflelo_epi16(a, _MM_SHUFFLE(D, C, B, A)); } 1628 1629 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1630 BL_INLINE_NODEBUG __m256i simd_swizzle_hi_u16(const __m256i& a) noexcept { return _mm256_shufflehi_epi16(a, _MM_SHUFFLE(D, C, B, A)); } 1631 1632 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1633 BL_INLINE_NODEBUG __m256i simd_swizzle_u16(const __m256i& a) noexcept { return simd_swizzle_hi_u16<D, C, B, A>(simd_swizzle_lo_u16<D, C, B, A>(a)); } 1634 1635 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1636 BL_INLINE_NODEBUG __m256i simd_swizzle_u32(const __m256i& a) noexcept { return _mm256_shuffle_epi32(a, _MM_SHUFFLE(D, C, B, A)); } 1637 1638 template<uint8_t B, uint8_t A> 1639 BL_INLINE_NODEBUG __m256i simd_swizzle_u64(const __m256i& a) noexcept { return simd_swizzle_u32<B*2 + 1, B*2, A*2 + 1, A*2>(a); } 1640 1641 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1642 BL_INLINE_NODEBUG __m256 simd_shuffle_f32(const __m256& lo, const __m256& hi) noexcept { return _mm256_shuffle_ps(lo, hi, _MM_SHUFFLE(D, C, B, A)); } 1643 1644 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1645 BL_INLINE_NODEBUG __m256i simd_shuffle_u32(const __m256i& lo, const __m256i& hi) noexcept { return simd_cast<__m256i>(_mm256_shuffle_ps(simd_cast<__m256>(lo), simd_cast<__m256>(hi), _MM_SHUFFLE(D, C, B, A))); } 1646 1647 template<uint8_t B, uint8_t A> 1648 BL_INLINE_NODEBUG __m256d simd_shuffle_f64(const __m256d& lo, const __m256d& hi) noexcept { return _mm256_shuffle_pd(lo, hi, (B << 3) | (A << 2) | (B << 1) | A); } 1649 1650 template<uint8_t B, uint8_t A> 1651 BL_INLINE_NODEBUG __m256i simd_shuffle_u64(const __m256i& lo, const __m256i& hi) noexcept { return simd_cast<__m256i>(simd_shuffle_f64<B, A>(simd_cast<__m256d>(lo), simd_cast<__m256d>(hi))); } 1652 1653 BL_INLINE_NODEBUG __m256i simd_broadcast256_u8(const __m128i& a) noexcept { return _mm256_broadcastb_epi8(a); } 1654 BL_INLINE_NODEBUG __m256i simd_broadcast256_u16(const __m128i& a) noexcept { return _mm256_broadcastw_epi16(a); } 1655 BL_INLINE_NODEBUG __m256i simd_broadcast256_u32(const __m128i& a) noexcept { return _mm256_broadcastd_epi32(a); } 1656 BL_INLINE_NODEBUG __m256i simd_broadcast256_u64(const __m128i& a) noexcept { return _mm256_broadcastq_epi64(a); } 1657 1658 BL_INLINE_NODEBUG __m256 simd_broadcast256_f32(const __m256& a) noexcept { return simd_cast<__m256>(simd_broadcast256_u32(simd_cast<__m128i>(a))); } 1659 BL_INLINE_NODEBUG __m256d simd_broadcast256_f64(const __m256d& a) noexcept { return simd_cast<__m256d>(simd_broadcast256_u64(simd_cast<__m128i>(a))); } 1660 1661 BL_INLINE_NODEBUG __m256i simd_dup_lo_u32(const __m256i& a) noexcept { return simd_swizzle_u32<2, 2, 0, 0>(a); } 1662 BL_INLINE_NODEBUG __m256i simd_dup_hi_u32(const __m256i& a) noexcept { return simd_swizzle_u32<3, 3, 1, 1>(a); } 1663 1664 BL_INLINE_NODEBUG __m256i simd_dup_lo_u64(const __m256i& a) noexcept { return simd_swizzle_u64<0, 0>(a); } 1665 BL_INLINE_NODEBUG __m256i simd_dup_hi_u64(const __m256i& a) noexcept { return simd_swizzle_u64<1, 1>(a); } 1666 1667 BL_INLINE_NODEBUG __m256i simd_swap_u32(const __m256i& a) noexcept { return simd_swizzle_u32<2, 3, 0, 1>(a); } 1668 BL_INLINE_NODEBUG __m256i simd_swap_u64(const __m256i& a) noexcept { return simd_swizzle_u64<0, 1>(a); } 1669 1670 BL_INLINE_NODEBUG __m256i simd_interleave_lo_u8(const __m256i& a, const __m256i& b) noexcept { return _mm256_unpacklo_epi8(a, b); } 1671 BL_INLINE_NODEBUG __m256i simd_interleave_hi_u8(const __m256i& a, const __m256i& b) noexcept { return _mm256_unpackhi_epi8(a, b); } 1672 BL_INLINE_NODEBUG __m256i simd_interleave_lo_u16(const __m256i& a, const __m256i& b) noexcept { return _mm256_unpacklo_epi16(a, b); } 1673 BL_INLINE_NODEBUG __m256i simd_interleave_hi_u16(const __m256i& a, const __m256i& b) noexcept { return _mm256_unpackhi_epi16(a, b); } 1674 BL_INLINE_NODEBUG __m256i simd_interleave_lo_u32(const __m256i& a, const __m256i& b) noexcept { return _mm256_unpacklo_epi32(a, b); } 1675 BL_INLINE_NODEBUG __m256i simd_interleave_hi_u32(const __m256i& a, const __m256i& b) noexcept { return _mm256_unpackhi_epi32(a, b); } 1676 BL_INLINE_NODEBUG __m256i simd_interleave_lo_u64(const __m256i& a, const __m256i& b) noexcept { return _mm256_unpacklo_epi64(a, b); } 1677 BL_INLINE_NODEBUG __m256i simd_interleave_hi_u64(const __m256i& a, const __m256i& b) noexcept { return _mm256_unpackhi_epi64(a, b); } 1678 1679 template<int kNumBytes> 1680 BL_INLINE_NODEBUG __m256i simd_alignr_u128(const __m256i& a, const __m256i& b) noexcept { return _mm256_alignr_epi8(a, b, kNumBytes); } 1681 1682 #endif // BL_TARGET_OPT_AVX2 1683 1684 #if defined(BL_TARGET_OPT_AVX512) 1685 BL_INLINE_NODEBUG __m512i simd_swizzlev_u8(const __m512i& a, const __m512i& b) noexcept { return _mm512_shuffle_epi8(a, b); } 1686 1687 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1688 BL_INLINE_NODEBUG __m512i simd_swizzle_lo_u16(const __m512i& a) noexcept { return _mm512_shufflelo_epi16(a, _MM_SHUFFLE(D, C, B, A)); } 1689 1690 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1691 BL_INLINE_NODEBUG __m512i simd_swizzle_hi_u16(const __m512i& a) noexcept { return _mm512_shufflehi_epi16(a, _MM_SHUFFLE(D, C, B, A)); } 1692 1693 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1694 BL_INLINE_NODEBUG __m512i simd_swizzle_u16(const __m512i& a) noexcept { return simd_swizzle_hi_u16<D, C, B, A>(simd_swizzle_lo_u16<D, C, B, A>(a)); } 1695 1696 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1697 BL_INLINE_NODEBUG __m512i simd_swizzle_u32(const __m512i& a) noexcept { return _mm512_shuffle_epi32(a, _MM_PERM_ENUM(_MM_SHUFFLE(D, C, B, A))); } 1698 1699 template<uint8_t B, uint8_t A> 1700 BL_INLINE_NODEBUG __m512i simd_swizzle_u64(const __m512i& a) noexcept { return simd_swizzle_u32<B*2 + 1, B*2, A*2 + 1, A*2>(a); } 1701 1702 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1703 BL_INLINE_NODEBUG __m512 simd_shuffle_f32(const __m512& lo, const __m512& hi) noexcept { return _mm512_shuffle_ps(lo, hi, _MM_SHUFFLE(D, C, B, A)); } 1704 1705 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A> 1706 BL_INLINE_NODEBUG __m512i simd_shuffle_u32(const __m512i& lo, const __m512i& hi) noexcept { return simd_cast<__m512i>(_mm512_shuffle_ps(simd_cast<__m512>(lo), simd_cast<__m512>(hi), _MM_SHUFFLE(D, C, B, A))); } 1707 1708 template<uint8_t B, uint8_t A> 1709 BL_INLINE_NODEBUG __m512d simd_shuffle_f64(const __m512d& lo, const __m512d& hi) noexcept { return _mm512_shuffle_pd(lo, hi, (B << 1) | A); } 1710 1711 template<uint8_t B, uint8_t A> 1712 BL_INLINE_NODEBUG __m512i simd_shuffle_u64(const __m512i& lo, const __m512i& hi) noexcept { return simd_cast<__m512i>(_mm512_shuffle_pd(simd_cast<__m512d>(lo), simd_cast<__m512d>(hi), (B << 1) | A)); } 1713 1714 BL_INLINE_NODEBUG __m512i simd_broadcast512_u8(const __m128i& a) noexcept { return _mm512_broadcastb_epi8(a); } 1715 BL_INLINE_NODEBUG __m512i simd_broadcast512_u16(const __m128i& a) noexcept { return _mm512_broadcastw_epi16(a); } 1716 BL_INLINE_NODEBUG __m512i simd_broadcast512_u32(const __m128i& a) noexcept { return _mm512_broadcastd_epi32(a); } 1717 BL_INLINE_NODEBUG __m512i simd_broadcast512_u64(const __m128i& a) noexcept { return _mm512_broadcastq_epi64(a); } 1718 1719 BL_INLINE_NODEBUG __m512 simd_broadcast512_f32(const __m128& a) noexcept { return simd_cast<__m512>(simd_broadcast512_u32(simd_cast<__m128i>(a))); } 1720 BL_INLINE_NODEBUG __m512d simd_broadcast512_f64(const __m128d& a) noexcept { return simd_cast<__m512d>(simd_broadcast512_u64(simd_cast<__m128i>(a))); } 1721 1722 template<> BL_INLINE_NODEBUG __m512i simd_broadcast_u8<64>(const __m128i& a) noexcept { return _mm512_broadcastb_epi8(a); } 1723 template<> BL_INLINE_NODEBUG __m512i simd_broadcast_u16<64>(const __m128i& a) noexcept { return _mm512_broadcastw_epi16(a); } 1724 template<> BL_INLINE_NODEBUG __m512i simd_broadcast_u32<64>(const __m128i& a) noexcept { return _mm512_broadcastd_epi32(a); } 1725 template<> BL_INLINE_NODEBUG __m512i simd_broadcast_u64<64>(const __m128i& a) noexcept { return _mm512_broadcastq_epi64(a); } 1726 template<> BL_INLINE_NODEBUG __m512 simd_broadcast_f32<64>(const __m128& a) noexcept { return _mm512_broadcastss_ps(a); } 1727 template<> BL_INLINE_NODEBUG __m512d simd_broadcast_f64<64>(const __m128d& a) noexcept { return _mm512_broadcastsd_pd(a); } 1728 1729 BL_INLINE_NODEBUG __m512i simd_dup_lo_u32(const __m512i& a) noexcept { return simd_swizzle_u32<2, 2, 0, 0>(a); } 1730 BL_INLINE_NODEBUG __m512i simd_dup_hi_u32(const __m512i& a) noexcept { return simd_swizzle_u32<3, 3, 1, 1>(a); } 1731 1732 BL_INLINE_NODEBUG __m512i simd_dup_lo_u64(const __m512i& a) noexcept { return simd_swizzle_u64<0, 0>(a); } 1733 BL_INLINE_NODEBUG __m512i simd_dup_hi_u64(const __m512i& a) noexcept { return simd_swizzle_u64<1, 1>(a); } 1734 1735 BL_INLINE_NODEBUG __m512i simd_swap_u32(const __m512i& a) noexcept { return simd_swizzle_u32<2, 3, 0, 1>(a); } 1736 BL_INLINE_NODEBUG __m512i simd_swap_u64(const __m512i& a) noexcept { return simd_swizzle_u64<0, 1>(a); } 1737 1738 BL_INLINE_NODEBUG __m512i simd_interleave_lo_u8(const __m512i& a, const __m512i& b) noexcept { return _mm512_unpacklo_epi8(a, b); } 1739 BL_INLINE_NODEBUG __m512i simd_interleave_hi_u8(const __m512i& a, const __m512i& b) noexcept { return _mm512_unpackhi_epi8(a, b); } 1740 BL_INLINE_NODEBUG __m512i simd_interleave_lo_u16(const __m512i& a, const __m512i& b) noexcept { return _mm512_unpacklo_epi16(a, b); } 1741 BL_INLINE_NODEBUG __m512i simd_interleave_hi_u16(const __m512i& a, const __m512i& b) noexcept { return _mm512_unpackhi_epi16(a, b); } 1742 BL_INLINE_NODEBUG __m512i simd_interleave_lo_u32(const __m512i& a, const __m512i& b) noexcept { return _mm512_unpacklo_epi32(a, b); } 1743 BL_INLINE_NODEBUG __m512i simd_interleave_hi_u32(const __m512i& a, const __m512i& b) noexcept { return _mm512_unpackhi_epi32(a, b); } 1744 BL_INLINE_NODEBUG __m512i simd_interleave_lo_u64(const __m512i& a, const __m512i& b) noexcept { return _mm512_unpacklo_epi64(a, b); } 1745 BL_INLINE_NODEBUG __m512i simd_interleave_hi_u64(const __m512i& a, const __m512i& b) noexcept { return _mm512_unpackhi_epi64(a, b); } 1746 1747 template<int kNumBytes> 1748 BL_INLINE_NODEBUG __m512i simd_alignr_u128(const __m512i& a, const __m512i& b) noexcept { return _mm512_alignr_epi8(a, b, kNumBytes); } 1749 1750 #endif // BL_TARGET_OPT_AVX512 1751 1752 } // {Internal} 1753 1754 // SIMD - Internal - Integer Packing & Unpacking 1755 // ============================================= 1756 1757 namespace Internal { 1758 1759 BL_INLINE_NODEBUG __m128i simd_packs_128_i16_i8(const __m128i& a, const __m128i& b) noexcept { return _mm_packs_epi16(a, b); } 1760 BL_INLINE_NODEBUG __m128i simd_packs_128_i16_u8(const __m128i& a, const __m128i& b) noexcept { return _mm_packus_epi16(a, b); } 1761 BL_INLINE_NODEBUG __m128i simd_packs_128_i32_i16(const __m128i& a, const __m128i& b) noexcept { return _mm_packs_epi32(a, b); } 1762 1763 BL_INLINE_NODEBUG __m128i simd_packs_128_i16_i8(const __m128i& a) noexcept { return _mm_packs_epi16(a, a); } 1764 BL_INLINE_NODEBUG __m128i simd_packs_128_i16_u8(const __m128i& a) noexcept { return _mm_packus_epi16(a, a); } 1765 BL_INLINE_NODEBUG __m128i simd_packs_128_i32_i16(const __m128i& a) noexcept { return _mm_packs_epi32(a, a); } 1766 1767 #if defined(BL_TARGET_OPT_SSE4_1) 1768 BL_INLINE_NODEBUG __m128i simd_packs_128_i32_u16(const __m128i& a) noexcept { return _mm_packus_epi32(a, a); } 1769 BL_INLINE_NODEBUG __m128i simd_packs_128_i32_u16(const __m128i& a, const __m128i& b) noexcept { return _mm_packus_epi32(a, b); } 1770 #else 1771 BL_INLINE_NODEBUG __m128i simd_packs_128_i32_u16(const __m128i& a) noexcept { 1772 __m128i a_shifted = _mm_srai_epi32(_mm_slli_epi32(a, 16), 16); 1773 return _mm_packs_epi32(a_shifted, a_shifted); 1774 } 1775 BL_INLINE_NODEBUG __m128i simd_packs_128_i32_u16(const __m128i& a, const __m128i& b) noexcept { 1776 __m128i a_shifted = _mm_srai_epi32(_mm_slli_epi32(a, 16), 16); 1777 __m128i b_shifted = _mm_srai_epi32(_mm_slli_epi32(b, 16), 16); 1778 return _mm_packs_epi32(a_shifted, b_shifted); 1779 } 1780 #endif 1781 1782 BL_INLINE_NODEBUG __m128i simd_packs_128_i32_i8(const __m128i& a) noexcept { return simd_packs_128_i16_i8(_mm_packs_epi32(a, a)); } 1783 BL_INLINE_NODEBUG __m128i simd_packs_128_i32_u8(const __m128i& a) noexcept { return simd_packs_128_i16_u8(_mm_packs_epi32(a, a)); } 1784 1785 BL_INLINE_NODEBUG __m128i simd_packs_128_i32_i8(const __m128i& a, const __m128i& b) noexcept { return simd_packs_128_i16_i8(_mm_packs_epi32(a, b)); } 1786 BL_INLINE_NODEBUG __m128i simd_packs_128_i32_u8(const __m128i& a, const __m128i& b) noexcept { return simd_packs_128_i32_i16(_mm_packs_epi32(a, b)); } 1787 1788 BL_INLINE_NODEBUG __m128i simd_packs_128_i32_i8(const __m128i& a, const __m128i& b, const __m128i& c, const __m128i& d) noexcept { return _mm_packs_epi16(_mm_packs_epi32(a, b), _mm_packs_epi32(c, d)); } 1789 BL_INLINE_NODEBUG __m128i simd_packs_128_i32_u8(const __m128i& a, const __m128i& b, const __m128i& c, const __m128i& d) noexcept { return _mm_packus_epi16(_mm_packs_epi32(a, b), _mm_packs_epi32(c, d)); } 1790 1791 // These assume that HI bytes of all inputs are always zero, so the implementation 1792 // can decide between packing with signed/unsigned saturation or vector swizzling. 1793 BL_INLINE_NODEBUG __m128i simd_packz_128_u16_u8(const __m128i& a) noexcept { return _mm_packus_epi16(a, a); } 1794 BL_INLINE_NODEBUG __m128i simd_packz_128_u16_u8(const __m128i& a, const __m128i& b) noexcept { return _mm_packus_epi16(a, b); } 1795 1796 #if defined(BL_TARGET_OPT_SSE4_1) || !defined(BL_TARGET_OPT_SSSE3) 1797 BL_INLINE_NODEBUG __m128i simd_packz_128_u32_u16(const __m128i& a) noexcept { return simd_packs_128_i32_u16(a); } 1798 BL_INLINE_NODEBUG __m128i simd_packz_128_u32_u16(const __m128i& a, const __m128i& b) noexcept { return simd_packs_128_i32_u16(a, b); } 1799 #else 1800 BL_INLINE_NODEBUG __m128i simd_packz_128_u32_u16(const __m128i& a) noexcept { 1801 return simd_swizzlev_u8(a, vec_const<__m128i>(&bl::common_table.swizu8_xx76xx54xx32xx10_to_7654321076543210)); 1802 } 1803 1804 BL_INLINE_NODEBUG __m128i simd_packz_128_u32_u16(const __m128i& a, const __m128i& b) noexcept { 1805 __m128i a_lo = simd_swizzlev_u8(a, vec_const<__m128i>(&bl::common_table.swizu8_xx76xx54xx32xx10_to_7654321076543210)); 1806 __m128i b_lo = simd_swizzlev_u8(b, vec_const<__m128i>(&bl::common_table.swizu8_xx76xx54xx32xx10_to_7654321076543210)); 1807 return _mm_unpacklo_epi64(a_lo, b_lo); 1808 } 1809 #endif 1810 1811 #if defined(BL_TARGET_OPT_SSSE3) 1812 BL_INLINE_NODEBUG __m128i simd_packz_128_u32_u8(const __m128i& a) noexcept { return simd_swizzlev_u8(a, vec_const<__m128i>(&bl::common_table.swizu8_xxx3xxx2xxx1xxx0_to_3210321032103210)); } 1813 #else 1814 BL_INLINE_NODEBUG __m128i simd_packz_128_u32_u8(const __m128i& a) noexcept { return simd_packs_128_i16_u8(_mm_packs_epi32(a, a)); } 1815 #endif 1816 1817 BL_INLINE_NODEBUG __m128i simd_packz_128_u32_u8(const __m128i& a, const __m128i& b) noexcept { return simd_packz_128_u16_u8(_mm_packs_epi32(a, b)); } 1818 BL_INLINE_NODEBUG __m128i simd_packz_128_u32_u8(const __m128i& a, const __m128i& b, const __m128i& c, const __m128i& d) noexcept { return _mm_packus_epi16(_mm_packs_epi32(a, b), _mm_packs_epi32(c, d)); } 1819 1820 #if defined(BL_TARGET_OPT_SSE4_1) 1821 BL_INLINE_NODEBUG __m128i simd_unpack_lo64_i8_i16(const __m128i& a) noexcept { return _mm_cvtepi8_epi16(a); } 1822 BL_INLINE_NODEBUG __m128i simd_unpack_lo64_u8_u16(const __m128i& a) noexcept { return _mm_cvtepu8_epi16(a); } 1823 BL_INLINE_NODEBUG __m128i simd_unpack_lo64_i16_i32(const __m128i& a) noexcept { return _mm_cvtepi16_epi32(a); } 1824 BL_INLINE_NODEBUG __m128i simd_unpack_lo64_u16_u32(const __m128i& a) noexcept { return _mm_cvtepu16_epi32(a); } 1825 BL_INLINE_NODEBUG __m128i simd_unpack_lo64_i32_i64(const __m128i& a) noexcept { return _mm_cvtepi32_epi64(a); } 1826 BL_INLINE_NODEBUG __m128i simd_unpack_lo64_u32_u64(const __m128i& a) noexcept { return _mm_cvtepu32_epi64(a); } 1827 BL_INLINE_NODEBUG __m128i simd_unpack_lo32_i8_i32(const __m128i& a) noexcept { return _mm_cvtepi8_epi32(a); } 1828 BL_INLINE_NODEBUG __m128i simd_unpack_lo32_u8_u32(const __m128i& a) noexcept { return _mm_cvtepu8_epi32(a); } 1829 #else 1830 BL_INLINE_NODEBUG __m128i simd_unpack_lo64_i8_i16(const __m128i& a) noexcept { return _mm_srai_epi16(_mm_unpacklo_epi8(a, a), 8); } 1831 BL_INLINE_NODEBUG __m128i simd_unpack_lo64_u8_u16(const __m128i& a) noexcept { return _mm_unpacklo_epi8(a, _mm_setzero_si128()); } 1832 BL_INLINE_NODEBUG __m128i simd_unpack_lo64_i16_i32(const __m128i& a) noexcept { return _mm_srai_epi32(_mm_unpacklo_epi16(a, a), 16); } 1833 BL_INLINE_NODEBUG __m128i simd_unpack_lo64_u16_u32(const __m128i& a) noexcept { return _mm_unpacklo_epi16(a, _mm_setzero_si128()); } 1834 BL_INLINE_NODEBUG __m128i simd_unpack_lo64_i32_i64(const __m128i& a) noexcept { return _mm_unpacklo_epi32(a, _mm_srai_epi32(a, 31)); } 1835 BL_INLINE_NODEBUG __m128i simd_unpack_lo64_u32_u64(const __m128i& a) noexcept { return _mm_unpacklo_epi32(a, _mm_setzero_si128()); } 1836 1837 BL_INLINE_NODEBUG __m128i simd_unpack_lo32_i8_i32(const __m128i& a) noexcept { 1838 __m128i x = _mm_unpacklo_epi8(a, a); 1839 __m128i y = _mm_unpacklo_epi8(x, x); 1840 return _mm_srai_epi32(y, 24); 1841 } 1842 1843 BL_INLINE_NODEBUG __m128i simd_unpack_lo32_u8_u32(const __m128i& a) noexcept { 1844 return simd_unpack_lo64_u16_u32(simd_unpack_lo64_u8_u16(a)); 1845 } 1846 #endif 1847 1848 BL_INLINE_NODEBUG __m128i simd_unpack_hi64_i8_i16(const __m128i& a) noexcept { return _mm_srai_epi16(_mm_unpackhi_epi8(a, a), 8); } 1849 BL_INLINE_NODEBUG __m128i simd_unpack_hi64_u8_u16(const __m128i& a) noexcept { return _mm_unpackhi_epi8(a, _mm_setzero_si128()); } 1850 BL_INLINE_NODEBUG __m128i simd_unpack_hi64_i16_i32(const __m128i& a) noexcept { return _mm_srai_epi32(_mm_unpackhi_epi16(a, a), 16); } 1851 BL_INLINE_NODEBUG __m128i simd_unpack_hi64_u16_u32(const __m128i& a) noexcept { return _mm_unpackhi_epi16(a, _mm_setzero_si128()); } 1852 BL_INLINE_NODEBUG __m128i simd_unpack_hi64_i32_i64(const __m128i& a) noexcept { return _mm_unpackhi_epi32(a, _mm_srai_epi32(a, 31)); } 1853 BL_INLINE_NODEBUG __m128i simd_unpack_hi64_u32_u64(const __m128i& a) noexcept { return _mm_unpackhi_epi32(a, _mm_setzero_si128()); } 1854 1855 BL_INLINE_NODEBUG __m128i simd_movw_i8_i16(const __m128i& a) noexcept { return simd_unpack_lo64_i8_i16(a); } 1856 BL_INLINE_NODEBUG __m128i simd_movw_u8_u16(const __m128i& a) noexcept { return simd_unpack_lo64_u8_u16(a); } 1857 BL_INLINE_NODEBUG __m128i simd_movw_i16_i32(const __m128i& a) noexcept { return simd_unpack_lo64_i16_i32(a); } 1858 BL_INLINE_NODEBUG __m128i simd_movw_u16_u32(const __m128i& a) noexcept { return simd_unpack_lo64_u16_u32(a); } 1859 BL_INLINE_NODEBUG __m128i simd_movw_i32_i64(const __m128i& a) noexcept { return simd_unpack_lo64_i32_i64(a); } 1860 BL_INLINE_NODEBUG __m128i simd_movw_u32_u64(const __m128i& a) noexcept { return simd_unpack_lo64_u32_u64(a); } 1861 BL_INLINE_NODEBUG __m128i simd_movw_i8_i32(const __m128i& a) noexcept { return simd_unpack_lo32_i8_i32(a); } 1862 BL_INLINE_NODEBUG __m128i simd_movw_u8_u32(const __m128i& a) noexcept { return simd_unpack_lo32_u8_u32(a); } 1863 1864 #if defined(BL_TARGET_OPT_AVX2) 1865 BL_INLINE_NODEBUG __m256i simd_packs_128_i16_i8(const __m256i& a) noexcept { return _mm256_packs_epi16(a, a); } 1866 BL_INLINE_NODEBUG __m256i simd_packs_128_i16_u8(const __m256i& a) noexcept { return _mm256_packus_epi16(a, a); } 1867 BL_INLINE_NODEBUG __m256i simd_packs_128_i32_i16(const __m256i& a) noexcept { return _mm256_packs_epi32(a, a); } 1868 BL_INLINE_NODEBUG __m256i simd_packs_128_i32_u16(const __m256i& a) noexcept { return _mm256_packus_epi32(a, a); } 1869 1870 BL_INLINE_NODEBUG __m256i simd_packs_128_i16_i8(const __m256i& a, const __m256i& b) noexcept { return _mm256_packs_epi16(a, b); } 1871 BL_INLINE_NODEBUG __m256i simd_packs_128_i16_u8(const __m256i& a, const __m256i& b) noexcept { return _mm256_packus_epi16(a, b); } 1872 BL_INLINE_NODEBUG __m256i simd_packs_128_i32_i16(const __m256i& a, const __m256i& b) noexcept { return _mm256_packs_epi32(a, b); } 1873 BL_INLINE_NODEBUG __m256i simd_packs_128_i32_u16(const __m256i& a, const __m256i& b) noexcept { return _mm256_packus_epi32(a, b); } 1874 1875 BL_INLINE_NODEBUG __m256i simd_packs_128_i32_i8(const __m256i& a) noexcept { return simd_packs_128_i16_i8(_mm256_packs_epi32(a, a)); } 1876 BL_INLINE_NODEBUG __m256i simd_packs_128_i32_u8(const __m256i& a) noexcept { return simd_packs_128_i16_u8(_mm256_packs_epi32(a, a)); } 1877 1878 BL_INLINE_NODEBUG __m256i simd_packs_128_i32_i8(const __m256i& a, const __m256i& b) noexcept { return simd_packs_128_i16_i8(_mm256_packs_epi32(a, b)); } 1879 BL_INLINE_NODEBUG __m256i simd_packs_128_i32_u8(const __m256i& a, const __m256i& b) noexcept { return simd_packs_128_i16_u8(_mm256_packs_epi32(a, b)); } 1880 1881 BL_INLINE_NODEBUG __m256i simd_packs_128_i32_i8(const __m256i& a, const __m256i& b, const __m256i& c, const __m256i& d) noexcept { return _mm256_packs_epi16(_mm256_packs_epi32(a, b), _mm256_packs_epi32(c, d)); } 1882 BL_INLINE_NODEBUG __m256i simd_packs_128_i32_u8(const __m256i& a, const __m256i& b, const __m256i& c, const __m256i& d) noexcept { return _mm256_packus_epi16(_mm256_packs_epi32(a, b), _mm256_packs_epi32(c, d)); } 1883 1884 BL_INLINE_NODEBUG __m256i simd_packz_128_u32_u16(const __m256i& a) noexcept { return _mm256_packus_epi32(a, a); } 1885 BL_INLINE_NODEBUG __m256i simd_packz_128_u32_u16(const __m256i& a, const __m256i& b) noexcept { return _mm256_packus_epi32(a, b); } 1886 1887 BL_INLINE_NODEBUG __m256i simd_packz_128_u32_u8(const __m256i& a) noexcept { return simd_packs_128_i16_u8(_mm256_packs_epi32(a, a)); } 1888 BL_INLINE_NODEBUG __m256i simd_packz_128_u32_u8(const __m256i& a, const __m256i& b) noexcept { return simd_packs_128_i16_u8(_mm256_packs_epi32(a, b)); } 1889 BL_INLINE_NODEBUG __m256i simd_packz_128_u32_u8(const __m256i& a, const __m256i& b, const __m256i& c, const __m256i& d) noexcept { return _mm256_packus_epi16(_mm256_packs_epi32(a, b), _mm256_packs_epi32(c, d)); } 1890 1891 BL_INLINE_NODEBUG __m256i simd_unpack_lo64_i8_i16(const __m256i& a) noexcept { return _mm256_srai_epi16(_mm256_unpacklo_epi8(a, a), 8); } 1892 BL_INLINE_NODEBUG __m256i simd_unpack_lo64_u8_u16(const __m256i& a) noexcept { return _mm256_unpacklo_epi8(a, _mm256_setzero_si256()); } 1893 BL_INLINE_NODEBUG __m256i simd_unpack_lo64_u8_u32(const __m256i& a) noexcept { return _mm256_unpacklo_epi8(a, _mm256_setzero_si256()); } 1894 BL_INLINE_NODEBUG __m256i simd_unpack_lo64_i16_i32(const __m256i& a) noexcept { return _mm256_srai_epi32(_mm256_unpacklo_epi16(a, a), 16); } 1895 BL_INLINE_NODEBUG __m256i simd_unpack_lo64_u16_u32(const __m256i& a) noexcept { return _mm256_unpacklo_epi16(a, _mm256_setzero_si256()); } 1896 BL_INLINE_NODEBUG __m256i simd_unpack_lo64_i32_i64(const __m256i& a) noexcept { return _mm256_unpacklo_epi32(a, _mm256_srai_epi32(a, 31)); } 1897 BL_INLINE_NODEBUG __m256i simd_unpack_lo64_u32_u64(const __m256i& a) noexcept { return _mm256_unpacklo_epi32(a, _mm256_setzero_si256()); } 1898 1899 BL_INLINE_NODEBUG __m256i simd_unpack_hi64_i8_i16(const __m256i& a) noexcept { return _mm256_srai_epi16(_mm256_unpackhi_epi8(a, a), 8); } 1900 BL_INLINE_NODEBUG __m256i simd_unpack_hi64_u8_u16(const __m256i& a) noexcept { return _mm256_unpackhi_epi8(a, _mm256_setzero_si256()); } 1901 BL_INLINE_NODEBUG __m256i simd_unpack_hi64_i16_i32(const __m256i& a) noexcept { return _mm256_srai_epi32(_mm256_unpackhi_epi16(a, a), 16); } 1902 BL_INLINE_NODEBUG __m256i simd_unpack_hi64_u16_u32(const __m256i& a) noexcept { return _mm256_unpackhi_epi16(a, _mm256_setzero_si256()); } 1903 BL_INLINE_NODEBUG __m256i simd_unpack_hi64_i32_i64(const __m256i& a) noexcept { return _mm256_unpackhi_epi32(a, _mm256_srai_epi32(a, 31)); } 1904 BL_INLINE_NODEBUG __m256i simd_unpack_hi64_u32_u64(const __m256i& a) noexcept { return _mm256_unpackhi_epi32(a, _mm256_setzero_si256()); } 1905 1906 BL_INLINE_NODEBUG __m256i simd_movw_i8_i16(const __m256i& a) noexcept { return _mm256_cvtepi8_epi16(simd_cast<__m128i>(a)); } 1907 BL_INLINE_NODEBUG __m256i simd_movw_u8_u16(const __m256i& a) noexcept { return _mm256_cvtepu8_epi16(simd_cast<__m128i>(a)); } 1908 BL_INLINE_NODEBUG __m256i simd_movw_i8_i32(const __m256i& a) noexcept { return _mm256_cvtepi8_epi32(simd_cast<__m128i>(a)); } 1909 BL_INLINE_NODEBUG __m256i simd_movw_u8_u32(const __m256i& a) noexcept { return _mm256_cvtepu8_epi32(simd_cast<__m128i>(a)); } 1910 BL_INLINE_NODEBUG __m256i simd_movw_i8_i64(const __m256i& a) noexcept { return _mm256_cvtepi8_epi64(simd_cast<__m128i>(a)); } 1911 BL_INLINE_NODEBUG __m256i simd_movw_u8_u64(const __m256i& a) noexcept { return _mm256_cvtepu8_epi64(simd_cast<__m128i>(a)); } 1912 BL_INLINE_NODEBUG __m256i simd_movw_i16_i32(const __m256i& a) noexcept { return _mm256_cvtepi16_epi32(simd_cast<__m128i>(a)); } 1913 BL_INLINE_NODEBUG __m256i simd_movw_u16_u32(const __m256i& a) noexcept { return _mm256_cvtepu16_epi32(simd_cast<__m128i>(a)); } 1914 BL_INLINE_NODEBUG __m256i simd_movw_i16_i64(const __m256i& a) noexcept { return _mm256_cvtepi16_epi64(simd_cast<__m128i>(a)); } 1915 BL_INLINE_NODEBUG __m256i simd_movw_u16_u64(const __m256i& a) noexcept { return _mm256_cvtepu16_epi64(simd_cast<__m128i>(a)); } 1916 BL_INLINE_NODEBUG __m256i simd_movw_i32_i64(const __m256i& a) noexcept { return _mm256_cvtepi32_epi64(simd_cast<__m128i>(a)); } 1917 BL_INLINE_NODEBUG __m256i simd_movw_u32_u64(const __m256i& a) noexcept { return _mm256_cvtepu32_epi64(simd_cast<__m128i>(a)); } 1918 #endif // BL_TARGET_OPT_AVX2 1919 1920 #if defined(BL_TARGET_OPT_AVX512) 1921 BL_INLINE_NODEBUG __m512i simd_packs_128_i16_i8(const __m512i& a) noexcept { return _mm512_packs_epi16(a, a); } 1922 BL_INLINE_NODEBUG __m512i simd_packs_128_i16_u8(const __m512i& a) noexcept { return _mm512_packus_epi16(a, a); } 1923 BL_INLINE_NODEBUG __m512i simd_packs_128_i32_i16(const __m512i& a) noexcept { return _mm512_packs_epi32(a, a); } 1924 BL_INLINE_NODEBUG __m512i simd_packs_128_i32_u16(const __m512i& a) noexcept { return _mm512_packus_epi32(a, a); } 1925 1926 BL_INLINE_NODEBUG __m512i simd_packs_128_i16_i8(const __m512i& a, const __m512i& b) noexcept { return _mm512_packs_epi16(a, b); } 1927 BL_INLINE_NODEBUG __m512i simd_packs_128_i16_u8(const __m512i& a, const __m512i& b) noexcept { return _mm512_packus_epi16(a, b); } 1928 BL_INLINE_NODEBUG __m512i simd_packs_128_i32_i16(const __m512i& a, const __m512i& b) noexcept { return _mm512_packs_epi32(a, b); } 1929 BL_INLINE_NODEBUG __m512i simd_packs_128_i32_u16(const __m512i& a, const __m512i& b) noexcept { return _mm512_packus_epi32(a, b); } 1930 1931 BL_INLINE_NODEBUG __m512i simd_packs_128_i32_i8(const __m512i& a) noexcept { return simd_packs_128_i16_i8(_mm512_packs_epi32(a, a)); } 1932 BL_INLINE_NODEBUG __m512i simd_packs_128_i32_u8(const __m512i& a) noexcept { return simd_packs_128_i16_u8(_mm512_packs_epi32(a, a)); } 1933 1934 BL_INLINE_NODEBUG __m512i simd_packs_128_i32_i8(const __m512i& a, const __m512i& b) noexcept { return simd_packs_128_i16_i8(_mm512_packs_epi32(a, b)); } 1935 BL_INLINE_NODEBUG __m512i simd_packs_128_i32_u8(const __m512i& a, const __m512i& b) noexcept { return simd_packs_128_i16_u8(_mm512_packs_epi32(a, b)); } 1936 1937 BL_INLINE_NODEBUG __m512i simd_packs_128_i32_i8(const __m512i& a, const __m512i& b, const __m512i& c, const __m512i& d) noexcept { return simd_packs_128_i16_i8(_mm512_packs_epi32(a, b), _mm512_packs_epi32(c, d)); } 1938 BL_INLINE_NODEBUG __m512i simd_packs_128_i32_u8(const __m512i& a, const __m512i& b, const __m512i& c, const __m512i& d) noexcept { return simd_packs_128_i16_u8(_mm512_packs_epi32(a, b), _mm512_packs_epi32(c, d)); } 1939 1940 BL_INLINE_NODEBUG __m512i simd_packz_128_u32_u16(const __m512i& a) noexcept { return simd_packs_128_i32_u16(a); } 1941 BL_INLINE_NODEBUG __m512i simd_packz_128_u32_u16(const __m512i& a, const __m512i& b) noexcept { return _mm512_packus_epi32(a, b); } 1942 1943 BL_INLINE_NODEBUG __m512i simd_packz_128_u32_u8(const __m512i& a) noexcept { return simd_packs_128_i16_u8(_mm512_packs_epi32(a, a)); } 1944 BL_INLINE_NODEBUG __m512i simd_packz_128_u32_u8(const __m512i& a, const __m512i& b) noexcept { return simd_packs_128_i16_u8(_mm512_packs_epi32(a, b)); } 1945 BL_INLINE_NODEBUG __m512i simd_packz_128_u32_u8(const __m512i& a, const __m512i& b, const __m512i& c, const __m512i& d) noexcept { return simd_packs_128_i16_u8(_mm512_packs_epi32(a, b), _mm512_packs_epi32(c, d)); } 1946 1947 BL_INLINE_NODEBUG __m512i simd_unpack_lo64_i8_i16(const __m512i& a) noexcept { return _mm512_srai_epi16(_mm512_unpacklo_epi8(a, a), 8); } 1948 BL_INLINE_NODEBUG __m512i simd_unpack_lo64_u8_u16(const __m512i& a) noexcept { return _mm512_unpacklo_epi8(a, _mm512_setzero_si512()); } 1949 BL_INLINE_NODEBUG __m512i simd_unpack_lo64_i16_i32(const __m512i& a) noexcept { return _mm512_srai_epi32(_mm512_unpacklo_epi16(a, a), 16); } 1950 BL_INLINE_NODEBUG __m512i simd_unpack_lo64_u16_u32(const __m512i& a) noexcept { return _mm512_unpacklo_epi16(a, _mm512_setzero_si512()); } 1951 BL_INLINE_NODEBUG __m512i simd_unpack_lo64_i32_i64(const __m512i& a) noexcept { return _mm512_unpacklo_epi32(a, _mm512_srai_epi32(a, 31)); } 1952 BL_INLINE_NODEBUG __m512i simd_unpack_lo64_u32_u64(const __m512i& a) noexcept { return _mm512_unpacklo_epi32(a, _mm512_setzero_si512()); } 1953 1954 BL_INLINE_NODEBUG __m512i simd_unpack_hi64_i8_i16(const __m512i& a) noexcept { return _mm512_srai_epi16(_mm512_unpackhi_epi8(a, a), 8); } 1955 BL_INLINE_NODEBUG __m512i simd_unpack_hi64_u8_u16(const __m512i& a) noexcept { return _mm512_unpackhi_epi8(a, _mm512_setzero_si512()); } 1956 BL_INLINE_NODEBUG __m512i simd_unpack_hi64_i16_i32(const __m512i& a) noexcept { return _mm512_srai_epi32(_mm512_unpackhi_epi16(a, a), 16); } 1957 BL_INLINE_NODEBUG __m512i simd_unpack_hi64_u16_u32(const __m512i& a) noexcept { return _mm512_unpackhi_epi16(a, _mm512_setzero_si512()); } 1958 BL_INLINE_NODEBUG __m512i simd_unpack_hi64_i32_i64(const __m512i& a) noexcept { return _mm512_unpackhi_epi32(a, _mm512_srai_epi32(a, 31)); } 1959 BL_INLINE_NODEBUG __m512i simd_unpack_hi64_u32_u64(const __m512i& a) noexcept { return _mm512_unpackhi_epi32(a, _mm512_setzero_si512()); } 1960 1961 BL_INLINE_NODEBUG __m512i simd_movw_i8_i16(const __m512i& a) noexcept { return _mm512_cvtepi8_epi16(simd_cast<__m256i>(a)); } 1962 BL_INLINE_NODEBUG __m512i simd_movw_u8_u16(const __m512i& a) noexcept { return _mm512_cvtepu8_epi16(simd_cast<__m256i>(a)); } 1963 BL_INLINE_NODEBUG __m512i simd_movw_i8_i32(const __m512i& a) noexcept { return _mm512_cvtepi8_epi32(simd_cast<__m128i>(a)); } 1964 BL_INLINE_NODEBUG __m512i simd_movw_u8_u32(const __m512i& a) noexcept { return _mm512_cvtepu8_epi32(simd_cast<__m128i>(a)); } 1965 BL_INLINE_NODEBUG __m512i simd_movw_i8_i64(const __m512i& a) noexcept { return _mm512_cvtepi8_epi64(simd_cast<__m128i>(a)); } 1966 BL_INLINE_NODEBUG __m512i simd_movw_u8_u64(const __m512i& a) noexcept { return _mm512_cvtepu8_epi64(simd_cast<__m128i>(a)); } 1967 BL_INLINE_NODEBUG __m512i simd_movw_i16_i32(const __m512i& a) noexcept { return _mm512_cvtepi16_epi32(simd_cast<__m256i>(a)); } 1968 BL_INLINE_NODEBUG __m512i simd_movw_u16_u32(const __m512i& a) noexcept { return _mm512_cvtepu16_epi32(simd_cast<__m256i>(a)); } 1969 BL_INLINE_NODEBUG __m512i simd_movw_i16_i64(const __m512i& a) noexcept { return _mm512_cvtepi16_epi64(simd_cast<__m128i>(a)); } 1970 BL_INLINE_NODEBUG __m512i simd_movw_u16_u64(const __m512i& a) noexcept { return _mm512_cvtepu16_epi64(simd_cast<__m128i>(a)); } 1971 BL_INLINE_NODEBUG __m512i simd_movw_i32_i64(const __m512i& a) noexcept { return _mm512_cvtepi32_epi64(simd_cast<__m256i>(a)); } 1972 BL_INLINE_NODEBUG __m512i simd_movw_u32_u64(const __m512i& a) noexcept { return _mm512_cvtepu32_epi64(simd_cast<__m256i>(a)); } 1973 #endif // BL_TARGET_OPT_AVX512 1974 1975 } // {Internal} 1976 1977 // SIMD - Extract & Insert 1978 // ======================= 1979 1980 template<uint32_t kIndex, typename V> 1981 BL_INLINE_NODEBUG uint32_t extract_u16(const V& src) noexcept { 1982 return uint32_t(_mm_extract_epi16(to_simd<__m128i>(src), kIndex)); 1983 } 1984 1985 template<uint32_t kIndex, typename V> 1986 BL_INLINE_NODEBUG Vec<16, typename V::ElementType> insert_u16(const V& dst, uint16_t val) noexcept { 1987 typedef Vec<16, typename V::ElementType> Vec128; 1988 return from_simd<Vec128>(_mm_insert_epi16(to_simd<__m128i>(dst), int(unsigned(val)), kIndex)); 1989 } 1990 1991 template<uint32_t kIndex, typename V> 1992 BL_INLINE_NODEBUG Vec<16, typename V::ElementType> insert_i16(const V& dst, int16_t val) noexcept { 1993 return insert_u16<kIndex>(dst, uint16_t(val)); 1994 } 1995 1996 template<uint32_t kIndex, typename V> 1997 BL_INLINE_NODEBUG Vec<16, typename V::ElementType> insert_m16(const V& dst, const void* src) noexcept { 1998 return insert_u16<kIndex>(dst, bl::MemOps::readU16u(src)); 1999 } 2000 2001 #if defined(BL_TARGET_OPT_SSE4_1) 2002 2003 template<uint32_t kIndex, typename V> 2004 BL_INLINE_NODEBUG uint32_t extract_u8(const V& src) noexcept { return uint32_t(_mm_extract_epi8(to_simd<__m128i>(src), kIndex)); } 2005 template<uint32_t kIndex, typename V> 2006 BL_INLINE_NODEBUG uint32_t extract_u32(const V& src) noexcept { return uint32_t(_mm_extract_epi32(to_simd<__m128i>(src), kIndex)); } 2007 2008 template<uint32_t kIndex, typename V> 2009 BL_INLINE_NODEBUG Vec<16, typename V::ElementType> insert_u8(const V& dst, uint8_t val) noexcept { 2010 typedef Vec<16, typename V::ElementType> Vec128; 2011 return Vec128{simd_cast<typename Vec128::SimdType>(_mm_insert_epi8(to_simd<__m128i>(dst), int(unsigned(val)), kIndex))}; 2012 } 2013 2014 template<uint32_t kIndex, typename V> 2015 BL_INLINE_NODEBUG Vec<16, typename V::ElementType> insert_i8(const V& dst, int8_t val) noexcept { 2016 return insert_u8<kIndex>(dst, uint8_t(val)); 2017 } 2018 2019 template<uint32_t kIndex, typename V> 2020 BL_INLINE_NODEBUG Vec<16, typename V::ElementType> insert_m8(const V& dst, const void* src) noexcept { 2021 return insert_u8<kIndex>(dst, *static_cast<const uint8_t*>(src)); 2022 } 2023 2024 template<uint32_t kIndex, typename V> 2025 BL_INLINE_NODEBUG Vec<16, typename V::ElementType> insert_u32(const V& dst, uint32_t val) noexcept { 2026 typedef Vec<16, typename V::ElementType> Vec128; 2027 return Vec128{simd_cast<typename Vec128::SimdType>(_mm_insert_epi32(to_simd<__m128i>(dst), int(val), kIndex))}; 2028 } 2029 2030 template<uint32_t kIndex, typename V> 2031 BL_INLINE_NODEBUG Vec<16, typename V::ElementType> insert_i32(const V& dst, int32_t val) noexcept { 2032 return insert_u32<kIndex>(dst, uint32_t(val)); 2033 } 2034 2035 template<uint32_t kIndex, typename V> 2036 BL_INLINE_NODEBUG Vec<16, typename V::ElementType> insert_m32(const V& dst, const void* src) noexcept { 2037 return insert_u32<kIndex>(dst, bl::MemOps::readU32u(src)); 2038 } 2039 2040 // Convenience function used to insert RGB24 components. 2041 template<uint32_t kIndex, typename V> 2042 BL_INLINE_NODEBUG Vec<16, typename V::ElementType> insert_m24(const V& dst, const void* src) noexcept { 2043 typedef Vec<16, typename V::ElementType> Vec128; 2044 const uint8_t* src_u8 = static_cast<const uint8_t*>(src); 2045 2046 __m128i v = to_simd<__m128i>(dst); 2047 2048 if constexpr ((kIndex & 0x1) == 0) { 2049 uint32_t u16_val = bl::MemOps::readU16u(src_u8); 2050 v = _mm_insert_epi16(v, int16_t(u16_val), kIndex / 2u); 2051 2052 uint32_t u8_val = bl::MemOps::readU8(src_u8 + 2u); 2053 v = _mm_insert_epi8(v, int8_t(u8_val), kIndex + 2u); 2054 } 2055 else { 2056 uint32_t u8_val = bl::MemOps::readU8(src_u8); 2057 v = _mm_insert_epi8(v, int8_t(u8_val), kIndex); 2058 2059 uint32_t u16_val = bl::MemOps::readU16u(src_u8 + 1u); 2060 v = _mm_insert_epi16(v, int16_t(u16_val), (kIndex + 1u) / 2u); 2061 } 2062 2063 return from_simd<Vec128>(v); 2064 } 2065 2066 #else 2067 2068 // Emulation of PEXTRB. 2069 template<uint32_t kIndex, typename V> 2070 BL_INLINE_NODEBUG uint32_t extract_u8(const V& src) noexcept { 2071 if constexpr ((kIndex & 1) == 0) 2072 return uint32_t(_mm_extract_epi16(to_simd<__m128i>(src), kIndex / 2u)) & 0xFFu; 2073 else 2074 return uint32_t(_mm_extract_epi16(to_simd<__m128i>(src), kIndex / 2u)) >> 8; 2075 } 2076 2077 // Emulation of PEXTRD. 2078 template<uint32_t kIndex, typename V> 2079 BL_INLINE_NODEBUG uint32_t extract_u32(const V& src) noexcept { 2080 #if BL_TARGET_ARCH_BITS >= 64 2081 if constexpr (kIndex == 1) { 2082 return uint32_t(uint64_t(_mm_cvtsi128_si64x(to_simd<__m128i>(src))) >> 32); 2083 } 2084 else 2085 #endif 2086 if constexpr (kIndex == 0) { 2087 return uint32_t(_mm_cvtsi128_si32(to_simd<__m128i>(src))); 2088 } 2089 else { 2090 uint32_t lo = uint32_t(_mm_extract_epi16(to_simd<__m128i>(src), kIndex * 2u)); 2091 uint32_t hi = uint32_t(_mm_extract_epi16(to_simd<__m128i>(src), kIndex * 2u + 1u)); 2092 return (hi << 16) | lo; 2093 } 2094 } 2095 2096 // Emulation of PINSRD. 2097 template<uint32_t kIndex, typename V> 2098 BL_INLINE_NODEBUG Vec<16, typename V::ElementType> insert_u32(const V& dst, uint32_t val) noexcept { 2099 int lo = int(val & 0xFFFFu); 2100 int hi = int(val >> 16); 2101 2102 typedef Vec<16, typename V::ElementType> Vec128; 2103 return Vec128{ 2104 simd_cast<typename Vec128::SimdType>( 2105 _mm_insert_epi16( 2106 _mm_insert_epi16(to_simd<__m128i>(dst), lo, kIndex), hi, kIndex + 1))}; 2107 } 2108 2109 template<uint32_t kIndex, typename V> 2110 BL_INLINE_NODEBUG Vec<16, typename V::ElementType> insert_i32(const V& dst, int32_t val) noexcept { 2111 return insert_u32<kIndex>(dst, uint32_t(val)); 2112 } 2113 2114 template<uint32_t kIndex, typename V> 2115 BL_INLINE_NODEBUG Vec<16, typename V::ElementType> insert_m32(const V& dst, const void* src) noexcept { 2116 return insert_m16<kIndex + 1>(insert_m16<kIndex>(dst, src), static_cast<const uint8_t*>(src) + 2); 2117 } 2118 2119 #endif // BL_TARGET_OPT_SSE4_1 2120 2121 // SIMD - Internal - Arithmetic and Logical Operations 2122 // =================================================== 2123 2124 namespace Internal { 2125 2126 #if defined(BL_TARGET_OPT_AVX512) 2127 template<uint8_t kPred> 2128 BL_INLINE_NODEBUG __m128i simd_ternlog(const __m128i& a, const __m128i& b, const __m128i& c) noexcept { return _mm_ternarylogic_epi32(a, b, c, kPred); } 2129 2130 template<uint8_t kPred> 2131 BL_INLINE_NODEBUG __m128 simd_ternlog(const __m128& a, const __m128& b, const __m128& c) noexcept { return simd_as_f(simd_ternlog<kPred>(simd_as_i(a), simd_as_i(b), simd_as_i(c))); } 2132 2133 template<uint8_t kPred> 2134 BL_INLINE_NODEBUG __m128d simd_ternlog(const __m128d& a, const __m128d& b, const __m128d& c) noexcept { return simd_as_d(simd_ternlog<kPred>(simd_as_i(a), simd_as_i(b), simd_as_i(c))); } 2135 #endif // BL_TARGET_OPT_AVX512 2136 2137 BL_INLINE_NODEBUG __m128 simd_and(const __m128& a, const __m128& b) noexcept { return _mm_and_ps(a, b); } 2138 BL_INLINE_NODEBUG __m128d simd_and(const __m128d& a, const __m128d& b) noexcept { return _mm_and_pd(a, b); } 2139 BL_INLINE_NODEBUG __m128i simd_and(const __m128i& a, const __m128i& b) noexcept { return _mm_and_si128(a, b); } 2140 2141 BL_INLINE_NODEBUG __m128 simd_andnot(const __m128& a, const __m128& b) noexcept { return _mm_andnot_ps(a, b); } 2142 BL_INLINE_NODEBUG __m128d simd_andnot(const __m128d& a, const __m128d& b) noexcept { return _mm_andnot_pd(a, b); } 2143 BL_INLINE_NODEBUG __m128i simd_andnot(const __m128i& a, const __m128i& b) noexcept { return _mm_andnot_si128(a, b); } 2144 2145 BL_INLINE_NODEBUG __m128 simd_or(const __m128& a, const __m128& b) noexcept { return _mm_or_ps(a, b); } 2146 BL_INLINE_NODEBUG __m128d simd_or(const __m128d& a, const __m128d& b) noexcept { return _mm_or_pd(a, b); } 2147 BL_INLINE_NODEBUG __m128i simd_or(const __m128i& a, const __m128i& b) noexcept { return _mm_or_si128(a, b); } 2148 2149 BL_INLINE_NODEBUG __m128 simd_xor(const __m128& a, const __m128& b) noexcept { return _mm_xor_ps(a, b); } 2150 BL_INLINE_NODEBUG __m128d simd_xor(const __m128d& a, const __m128d& b) noexcept { return _mm_xor_pd(a, b); } 2151 BL_INLINE_NODEBUG __m128i simd_xor(const __m128i& a, const __m128i& b) noexcept { return _mm_xor_si128(a, b); } 2152 2153 #if defined(BL_TARGET_OPT_AVX512) 2154 BL_INLINE_NODEBUG __m128 simd_not(const __m128& a) noexcept { return simd_ternlog<0x55u>(a, a, a); } 2155 BL_INLINE_NODEBUG __m128d simd_not(const __m128d& a) noexcept { return simd_ternlog<0x55u>(a, a, a); } 2156 BL_INLINE_NODEBUG __m128i simd_not(const __m128i& a) noexcept { return simd_ternlog<0x55u>(a, a, a); } 2157 2158 BL_INLINE_NODEBUG __m128 simd_blendv_bits(const __m128& a, const __m128& b, const __m128& msk) noexcept { return simd_ternlog<0xD8>(a, b, msk); } 2159 BL_INLINE_NODEBUG __m128d simd_blendv_bits(const __m128d& a, const __m128d& b, const __m128d& msk) noexcept { return simd_ternlog<0xD8>(a, b, msk); } 2160 BL_INLINE_NODEBUG __m128i simd_blendv_bits(const __m128i& a, const __m128i& b, const __m128i& msk) noexcept { return simd_ternlog<0xD8>(a, b, msk); } 2161 #else 2162 BL_INLINE_NODEBUG __m128 simd_not(const __m128& a) noexcept { return simd_xor(a, simd_make_ones<__m128>()); } 2163 BL_INLINE_NODEBUG __m128d simd_not(const __m128d& a) noexcept { return simd_xor(a, simd_make_ones<__m128d>()); } 2164 BL_INLINE_NODEBUG __m128i simd_not(const __m128i& a) noexcept { return simd_xor(a, simd_make_ones<__m128i>()); } 2165 2166 BL_INLINE_NODEBUG __m128 simd_blendv_bits(const __m128& a, const __m128& b, const __m128& msk) noexcept { return simd_or(simd_and(b, msk), simd_andnot(msk, a)); } 2167 BL_INLINE_NODEBUG __m128d simd_blendv_bits(const __m128d& a, const __m128d& b, const __m128d& msk) noexcept { return simd_or(simd_and(b, msk), simd_andnot(msk, a)); } 2168 BL_INLINE_NODEBUG __m128i simd_blendv_bits(const __m128i& a, const __m128i& b, const __m128i& msk) noexcept { return simd_or(simd_and(b, msk), simd_andnot(msk, a)); } 2169 #endif 2170 2171 #if defined(BL_TARGET_OPT_SSE4_1) 2172 BL_INLINE_NODEBUG __m128i simd_blendv_u8(const __m128i& a, const __m128i& b, const __m128i& msk) noexcept { return _mm_blendv_epi8(a, b, msk); } 2173 #else 2174 BL_INLINE_NODEBUG __m128i simd_blendv_u8(const __m128i& a, const __m128i& b, const __m128i& msk) noexcept { return simd_blendv_bits(a, b, msk); } 2175 #endif 2176 2177 #if defined(BL_TARGET_OPT_SSE4_1) 2178 template<uint32_t H, uint32_t G, uint32_t F, uint32_t E, uint32_t D, uint32_t C, uint32_t B, uint32_t A> 2179 BL_INLINE_NODEBUG __m128i simd_blend_i16(const __m128i& a, const __m128i& b) noexcept { 2180 return _mm_blend_epi16(a, b, (H << 7) | (G << 6) | (F << 5) | (E << 4) | (D << 3) | (C << 2) | (B << 1) | A); 2181 } 2182 2183 template<uint32_t D, uint32_t C, uint32_t B, uint32_t A> 2184 BL_INLINE_NODEBUG __m128i simd_blend_i32(const __m128i& a, const __m128i& b) noexcept { 2185 return _mm_blend_epi16(a, b, ((D * 0x3) << 3) | ((C * 0x3) << 2) | ((B * 0x3) << 1) | (A * 0x3)); 2186 } 2187 2188 template<uint32_t B, uint32_t A> 2189 BL_INLINE_NODEBUG __m128i simd_blend_i64(const __m128i& a, const __m128i& b) noexcept { 2190 return _mm_blend_epi16(a, b, ((B * 0xF) << 1) | (A * 0xF)); 2191 } 2192 #endif 2193 2194 BL_INLINE_NODEBUG __m128i simd_flip_sign_i8(const __m128i& a) noexcept { return simd_xor(a, simd_make128_u32(0x80808080u)); } 2195 BL_INLINE_NODEBUG __m128i simd_flip_sign_i16(const __m128i& a) noexcept { return simd_xor(a, simd_make128_u32(0x80008000u)); } 2196 BL_INLINE_NODEBUG __m128i simd_flip_sign_i32(const __m128i& a) noexcept { return simd_xor(a, simd_make128_u32(0x80000000u)); } 2197 BL_INLINE_NODEBUG __m128i simd_flip_sign_i64(const __m128i& a) noexcept { return simd_xor(a, simd_make128_u64(uint64_t(1) << 63)); } 2198 2199 BL_INLINE_NODEBUG __m128 simd_add_f32(const __m128& a, const __m128& b) noexcept { return _mm_add_ps(a, b); } 2200 BL_INLINE_NODEBUG __m128d simd_add_f64(const __m128d& a, const __m128d& b) noexcept { return _mm_add_pd(a, b); } 2201 2202 BL_INLINE_NODEBUG __m128 simd_sub_f32(const __m128& a, const __m128& b) noexcept { return _mm_sub_ps(a, b); } 2203 BL_INLINE_NODEBUG __m128d simd_sub_f64(const __m128d& a, const __m128d& b) noexcept { return _mm_sub_pd(a, b); } 2204 2205 BL_INLINE_NODEBUG __m128 simd_mul_f32(const __m128& a, const __m128& b) noexcept { return _mm_mul_ps(a, b); } 2206 BL_INLINE_NODEBUG __m128d simd_mul_f64(const __m128d& a, const __m128d& b) noexcept { return _mm_mul_pd(a, b); } 2207 2208 BL_INLINE_NODEBUG __m128 simd_div_f32(const __m128& a, const __m128& b) noexcept { return _mm_div_ps(a, b); } 2209 BL_INLINE_NODEBUG __m128d simd_div_f64(const __m128d& a, const __m128d& b) noexcept { return _mm_div_pd(a, b); } 2210 2211 BL_INLINE_NODEBUG __m128 simd_cmp_eq_f32(const __m128& a, const __m128& b) noexcept { return _mm_cmpeq_ps(a, b); } 2212 BL_INLINE_NODEBUG __m128d simd_cmp_eq_f64(const __m128d& a, const __m128d& b) noexcept { return _mm_cmpeq_pd(a, b); } 2213 2214 BL_INLINE_NODEBUG __m128 simd_cmp_ne_f32(const __m128& a, const __m128& b) noexcept { return _mm_cmpneq_ps(a, b); } 2215 BL_INLINE_NODEBUG __m128d simd_cmp_ne_f64(const __m128d& a, const __m128d& b) noexcept { return _mm_cmpneq_pd(a, b); } 2216 2217 BL_INLINE_NODEBUG __m128 simd_cmp_lt_f32(const __m128& a, const __m128& b) noexcept { return _mm_cmplt_ps(a, b); } 2218 BL_INLINE_NODEBUG __m128d simd_cmp_lt_f64(const __m128d& a, const __m128d& b) noexcept { return _mm_cmplt_pd(a, b); } 2219 2220 BL_INLINE_NODEBUG __m128 simd_cmp_le_f32(const __m128& a, const __m128& b) noexcept { return _mm_cmple_ps(a, b); } 2221 BL_INLINE_NODEBUG __m128d simd_cmp_le_f64(const __m128d& a, const __m128d& b) noexcept { return _mm_cmple_pd(a, b); } 2222 2223 BL_INLINE_NODEBUG __m128 simd_cmp_gt_f32(const __m128& a, const __m128& b) noexcept { return _mm_cmpgt_ps(a, b); } 2224 BL_INLINE_NODEBUG __m128d simd_cmp_gt_f64(const __m128d& a, const __m128d& b) noexcept { return _mm_cmpgt_pd(a, b); } 2225 2226 BL_INLINE_NODEBUG __m128 simd_cmp_ge_f32(const __m128& a, const __m128& b) noexcept { return _mm_cmpge_ps(a, b); } 2227 BL_INLINE_NODEBUG __m128d simd_cmp_ge_f64(const __m128d& a, const __m128d& b) noexcept { return _mm_cmpge_pd(a, b); } 2228 2229 BL_INLINE_NODEBUG __m128 simd_min_f32(const __m128& a, const __m128& b) noexcept { return _mm_min_ps(a, b); } 2230 BL_INLINE_NODEBUG __m128d simd_min_f64(const __m128d& a, const __m128d& b) noexcept { return _mm_min_pd(a, b); } 2231 2232 BL_INLINE_NODEBUG __m128 simd_max_f32(const __m128& a, const __m128& b) noexcept { return _mm_max_ps(a, b); } 2233 BL_INLINE_NODEBUG __m128d simd_max_f64(const __m128d& a, const __m128d& b) noexcept { return _mm_max_pd(a, b); } 2234 2235 BL_INLINE_NODEBUG __m128 simd_abs_f32(const __m128& a) noexcept { return _mm_and_ps(a, bl::common_table.p_7FFFFFFF7FFFFFFF.as<__m128>()); } 2236 BL_INLINE_NODEBUG __m128d simd_abs_f64(const __m128d& a) noexcept { return _mm_and_pd(a, bl::common_table.p_7FFFFFFFFFFFFFFF.as<__m128d>()); } 2237 2238 BL_INLINE_NODEBUG __m128 simd_sqrt_f32(const __m128& a) noexcept { return _mm_sqrt_ps(a); } 2239 BL_INLINE_NODEBUG __m128d simd_sqrt_f64(const __m128d& a) noexcept { return _mm_sqrt_pd(a); } 2240 2241 BL_INLINE_NODEBUG __m128i simd_add_i8(const __m128i& a, const __m128i& b) noexcept { return _mm_add_epi8(a, b); } 2242 BL_INLINE_NODEBUG __m128i simd_add_i16(const __m128i& a, const __m128i& b) noexcept { return _mm_add_epi16(a, b); } 2243 BL_INLINE_NODEBUG __m128i simd_add_i32(const __m128i& a, const __m128i& b) noexcept { return _mm_add_epi32(a, b); } 2244 BL_INLINE_NODEBUG __m128i simd_add_i64(const __m128i& a, const __m128i& b) noexcept { return _mm_add_epi64(a, b); } 2245 BL_INLINE_NODEBUG __m128i simd_add_u8(const __m128i& a, const __m128i& b) noexcept { return _mm_add_epi8(a, b); } 2246 BL_INLINE_NODEBUG __m128i simd_add_u16(const __m128i& a, const __m128i& b) noexcept { return _mm_add_epi16(a, b); } 2247 BL_INLINE_NODEBUG __m128i simd_add_u32(const __m128i& a, const __m128i& b) noexcept { return _mm_add_epi32(a, b); } 2248 BL_INLINE_NODEBUG __m128i simd_add_u64(const __m128i& a, const __m128i& b) noexcept { return _mm_add_epi64(a, b); } 2249 2250 BL_INLINE_NODEBUG __m128i simd_adds_i8(const __m128i& a, const __m128i& b) noexcept { return _mm_adds_epi8(a, b); } 2251 BL_INLINE_NODEBUG __m128i simd_adds_i16(const __m128i& a, const __m128i& b) noexcept { return _mm_adds_epi16(a, b); } 2252 BL_INLINE_NODEBUG __m128i simd_adds_u8(const __m128i& a, const __m128i& b) noexcept { return _mm_adds_epu8(a, b); } 2253 BL_INLINE_NODEBUG __m128i simd_adds_u16(const __m128i& a, const __m128i& b) noexcept { return _mm_adds_epu16(a, b); } 2254 2255 BL_INLINE_NODEBUG __m128i simd_sub_i8(const __m128i& a, const __m128i& b) noexcept { return _mm_sub_epi8(a, b); } 2256 BL_INLINE_NODEBUG __m128i simd_sub_i16(const __m128i& a, const __m128i& b) noexcept { return _mm_sub_epi16(a, b); } 2257 BL_INLINE_NODEBUG __m128i simd_sub_i32(const __m128i& a, const __m128i& b) noexcept { return _mm_sub_epi32(a, b); } 2258 BL_INLINE_NODEBUG __m128i simd_sub_i64(const __m128i& a, const __m128i& b) noexcept { return _mm_sub_epi64(a, b); } 2259 BL_INLINE_NODEBUG __m128i simd_sub_u8(const __m128i& a, const __m128i& b) noexcept { return _mm_sub_epi8(a, b); } 2260 BL_INLINE_NODEBUG __m128i simd_sub_u16(const __m128i& a, const __m128i& b) noexcept { return _mm_sub_epi16(a, b); } 2261 BL_INLINE_NODEBUG __m128i simd_sub_u32(const __m128i& a, const __m128i& b) noexcept { return _mm_sub_epi32(a, b); } 2262 BL_INLINE_NODEBUG __m128i simd_sub_u64(const __m128i& a, const __m128i& b) noexcept { return _mm_sub_epi64(a, b); } 2263 2264 BL_INLINE_NODEBUG __m128i simd_subs_i8(const __m128i& a, const __m128i& b) noexcept { return _mm_subs_epi8(a, b); } 2265 BL_INLINE_NODEBUG __m128i simd_subs_i16(const __m128i& a, const __m128i& b) noexcept { return _mm_subs_epi16(a, b); } 2266 BL_INLINE_NODEBUG __m128i simd_subs_u8(const __m128i& a, const __m128i& b) noexcept { return _mm_subs_epu8(a, b); } 2267 BL_INLINE_NODEBUG __m128i simd_subs_u16(const __m128i& a, const __m128i& b) noexcept { return _mm_subs_epu16(a, b); } 2268 2269 BL_INLINE_NODEBUG __m128i simd_mul_i16(const __m128i& a, const __m128i& b) noexcept { return _mm_mullo_epi16(a, b); } 2270 BL_INLINE_NODEBUG __m128i simd_mul_u16(const __m128i& a, const __m128i& b) noexcept { return _mm_mullo_epi16(a, b); } 2271 2272 BL_INLINE_NODEBUG __m128i simd_mulh_i16(const __m128i& a, const __m128i& b) noexcept { return _mm_mulhi_epi16(a, b); } 2273 BL_INLINE_NODEBUG __m128i simd_mulh_u16(const __m128i& a, const __m128i& b) noexcept { return _mm_mulhi_epu16(a, b); } 2274 BL_INLINE_NODEBUG __m128i simd_mulw_u32(const __m128i& a, const __m128i& b) noexcept { return _mm_mul_epu32(a, b); } 2275 2276 #if defined(BL_TARGET_OPT_SSE4_1) 2277 BL_INLINE_NODEBUG __m128i simd_mulw_i32(const __m128i& a, const __m128i& b) noexcept { return _mm_mul_epi32(a, b); } 2278 BL_INLINE_NODEBUG __m128i simd_mul_i32(const __m128i& a, const __m128i& b) noexcept { return _mm_mullo_epi32(a, b); } 2279 BL_INLINE_NODEBUG __m128i simd_mul_u32(const __m128i& a, const __m128i& b) noexcept { return _mm_mullo_epi32(a, b); } 2280 #else 2281 BL_INLINE_NODEBUG __m128i simd_mul_i32(const __m128i& a, const __m128i& b) noexcept { 2282 __m128i hi = _mm_mul_epu32(_mm_shuffle_epi32(a, _MM_SHUFFLE(3, 3, 1, 1)), 2283 _mm_shuffle_epi32(b, _MM_SHUFFLE(3, 3, 1, 1))); 2284 __m128i lo = _mm_mul_epu32(a, b); 2285 __m128i result3120 = simd_cast<__m128i>(_mm_shuffle_ps(simd_cast<__m128>(lo), simd_cast<__m128>(hi), _MM_SHUFFLE(2, 0, 2, 0))); 2286 return _mm_shuffle_epi32(result3120, _MM_SHUFFLE(3, 1, 2, 0)); 2287 } 2288 BL_INLINE_NODEBUG __m128i simd_mul_u32(const __m128i& a, const __m128i& b) noexcept { return simd_mul_i32(a, b); } 2289 #endif // BL_TARGET_OPT_SSE4_1 2290 2291 #if defined(BL_TARGET_OPT_AVX512) 2292 BL_INLINE_NODEBUG __m128i simd_mul_i64(const __m128i& a, const __m128i& b) noexcept { return _mm_mullo_epi64(a, b); } 2293 #else 2294 BL_INLINE_NODEBUG __m128i simd_mul_i64(const __m128i& a, const __m128i& b) noexcept { 2295 union u64x2_view { __m128i reg; uint64_t elements[2]; }; 2296 2297 u64x2_view a_view{a}; 2298 u64x2_view b_view{b}; 2299 return simd_make128_u64(a_view.elements[1] * b_view.elements[1], a_view.elements[0] * b_view.elements[0]); 2300 } 2301 #endif 2302 BL_INLINE_NODEBUG __m128i simd_mul_u64(const __m128i& a, const __m128i& b) noexcept { return simd_mul_i64(a, b); } 2303 2304 BL_INLINE_NODEBUG __m128i simd_maddw_i16_i32(const __m128i& a, const __m128i& b) noexcept { return _mm_madd_epi16(a, b); } 2305 2306 BL_INLINE_NODEBUG __m128i simd_cmp_eq_i8(const __m128i& a, const __m128i& b) noexcept { return _mm_cmpeq_epi8(a, b); } 2307 BL_INLINE_NODEBUG __m128i simd_cmp_eq_i16(const __m128i& a, const __m128i& b) noexcept { return _mm_cmpeq_epi16(a, b); } 2308 BL_INLINE_NODEBUG __m128i simd_cmp_eq_i32(const __m128i& a, const __m128i& b) noexcept { return _mm_cmpeq_epi32(a, b); } 2309 2310 #if defined(BL_TARGET_OPT_SSE4_1) 2311 BL_INLINE_NODEBUG __m128i simd_cmp_eq_i64(const __m128i& a, const __m128i& b) noexcept { return _mm_cmpeq_epi64(a, b); } 2312 #else 2313 BL_INLINE_NODEBUG __m128i simd_cmp_eq_i64(const __m128i& a, const __m128i& b) noexcept { 2314 __m128i x = _mm_cmpeq_epi32(a, b); 2315 __m128i y = _mm_shuffle_epi32(x, _MM_SHUFFLE(2, 3, 0, 1)); 2316 return _mm_and_si128(x, y); 2317 } 2318 #endif 2319 2320 BL_INLINE_NODEBUG __m128i simd_cmp_ne_i8(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_eq_i8(a, b)); } 2321 BL_INLINE_NODEBUG __m128i simd_cmp_ne_i16(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_eq_i16(a, b)); } 2322 BL_INLINE_NODEBUG __m128i simd_cmp_ne_i32(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_eq_i32(a, b)); } 2323 BL_INLINE_NODEBUG __m128i simd_cmp_ne_i64(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_eq_i64(a, b)); } 2324 2325 BL_INLINE_NODEBUG __m128i simd_cmp_gt_i8(const __m128i& a, const __m128i& b) noexcept { return _mm_cmpgt_epi8(a, b); } 2326 BL_INLINE_NODEBUG __m128i simd_cmp_gt_i16(const __m128i& a, const __m128i& b) noexcept { return _mm_cmpgt_epi16(a, b); } 2327 BL_INLINE_NODEBUG __m128i simd_cmp_gt_i32(const __m128i& a, const __m128i& b) noexcept { return _mm_cmpgt_epi32(a, b); } 2328 2329 #if defined(BL_TARGET_OPT_SSE4_2) 2330 BL_INLINE_NODEBUG __m128i simd_cmp_gt_i64(const __m128i& a, const __m128i& b) noexcept { return _mm_cmpgt_epi64(a, b); } 2331 #else 2332 BL_INLINE_NODEBUG __m128i simd_cmp_gt_i64(const __m128i& a, const __m128i& b) noexcept { 2333 // Possibly the best solution: 2334 // https://stackoverflow.com/questions/65166174/how-to-simulate-pcmpgtq-on-sse2 2335 // 2336 // The good thing on this solution is that it doesn't need any constants, just temporaries. 2337 __m128i msk = _mm_and_si128(_mm_cmpeq_epi32(a, b), _mm_sub_epi64(b, a)); 2338 msk = _mm_or_si128(msk, _mm_cmpgt_epi32(a, b)); 2339 return _mm_shuffle_epi32(msk, _MM_SHUFFLE(3, 3, 1, 1)); 2340 } 2341 #endif 2342 2343 BL_INLINE_NODEBUG __m128i simd_cmp_lt_i8(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_gt_i8(b, a); } 2344 BL_INLINE_NODEBUG __m128i simd_cmp_lt_i16(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_gt_i16(b, a); } 2345 BL_INLINE_NODEBUG __m128i simd_cmp_lt_i32(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_gt_i32(b, a); } 2346 BL_INLINE_NODEBUG __m128i simd_cmp_lt_i64(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_gt_i64(b, a); } 2347 2348 #if defined(BL_TARGET_OPT_SSE4_1) 2349 BL_INLINE_NODEBUG __m128i simd_cmp_ge_i8(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i8(_mm_min_epi8(a, b), b); } 2350 BL_INLINE_NODEBUG __m128i simd_cmp_le_i8(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i8(_mm_max_epi8(a, b), b); } 2351 BL_INLINE_NODEBUG __m128i simd_cmp_ge_i16(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i16(_mm_min_epi16(a, b), b); } 2352 BL_INLINE_NODEBUG __m128i simd_cmp_le_i16(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i16(_mm_max_epi16(a, b), b); } 2353 BL_INLINE_NODEBUG __m128i simd_cmp_ge_i32(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i32(_mm_min_epi32(a, b), b); } 2354 BL_INLINE_NODEBUG __m128i simd_cmp_le_i32(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i32(_mm_max_epi32(a, b), b); } 2355 #else 2356 BL_INLINE_NODEBUG __m128i simd_cmp_ge_i8(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_lt_i8(a, b)); } 2357 BL_INLINE_NODEBUG __m128i simd_cmp_le_i8(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_gt_i8(a, b)); } 2358 BL_INLINE_NODEBUG __m128i simd_cmp_ge_i16(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_lt_i16(a, b)); } 2359 BL_INLINE_NODEBUG __m128i simd_cmp_le_i16(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_gt_i16(a, b)); } 2360 BL_INLINE_NODEBUG __m128i simd_cmp_ge_i32(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_lt_i32(a, b)); } 2361 BL_INLINE_NODEBUG __m128i simd_cmp_le_i32(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_gt_i32(a, b)); } 2362 #endif 2363 2364 #if defined(BL_TARGET_OPT_AVX512) 2365 BL_INLINE_NODEBUG __m128i simd_cmp_ge_i64(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i64(_mm_min_epi64(a, b), b); } 2366 BL_INLINE_NODEBUG __m128i simd_cmp_le_i64(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i64(_mm_max_epi64(a, b), b); } 2367 #else 2368 BL_INLINE_NODEBUG __m128i simd_cmp_ge_i64(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_lt_i64(a, b)); } 2369 BL_INLINE_NODEBUG __m128i simd_cmp_le_i64(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_gt_i64(a, b)); } 2370 #endif 2371 2372 BL_INLINE_NODEBUG __m128i simd_cmp_ge_u8(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i8(_mm_min_epu8(a, b), b); } 2373 BL_INLINE_NODEBUG __m128i simd_cmp_le_u8(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i8(_mm_max_epu8(a, b), b); } 2374 2375 #if defined(BL_TARGET_OPT_SSE4_1) 2376 BL_INLINE_NODEBUG __m128i simd_cmp_ge_u16(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i16(_mm_min_epu16(a, b), b); } 2377 BL_INLINE_NODEBUG __m128i simd_cmp_le_u16(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i16(_mm_max_epu16(a, b), b); } 2378 #else 2379 BL_INLINE_NODEBUG __m128i simd_cmp_ge_u16(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i16(_mm_subs_epu16(b, a), simd_make_zero<__m128i>()); } 2380 BL_INLINE_NODEBUG __m128i simd_cmp_le_u16(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i16(_mm_subs_epu16(a, b), simd_make_zero<__m128i>()); } 2381 #endif 2382 2383 BL_INLINE_NODEBUG __m128i simd_cmp_gt_u8(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_le_u8(a, b)); } 2384 BL_INLINE_NODEBUG __m128i simd_cmp_lt_u8(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_ge_u8(a, b)); } 2385 BL_INLINE_NODEBUG __m128i simd_cmp_gt_u16(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_le_u16(a, b)); } 2386 BL_INLINE_NODEBUG __m128i simd_cmp_lt_u16(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_ge_u16(a, b)); } 2387 2388 #if defined(BL_TARGET_OPT_SSE4_1) 2389 BL_INLINE_NODEBUG __m128i simd_cmp_ge_u32(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i32(_mm_min_epu32(a, b), b); } 2390 BL_INLINE_NODEBUG __m128i simd_cmp_le_u32(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i32(_mm_max_epu32(a, b), b); } 2391 BL_INLINE_NODEBUG __m128i simd_cmp_gt_u32(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_le_u32(a, b)); } 2392 BL_INLINE_NODEBUG __m128i simd_cmp_lt_u32(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_ge_u32(a, b)); } 2393 #else 2394 BL_INLINE_NODEBUG __m128i simd_cmp_gt_u32(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_gt_i32(simd_flip_sign_i32(a), simd_flip_sign_i32(b)); } 2395 BL_INLINE_NODEBUG __m128i simd_cmp_lt_u32(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_gt_i32(simd_flip_sign_i32(b), simd_flip_sign_i32(a)); } 2396 2397 BL_INLINE_NODEBUG __m128i simd_cmp_ge_u32(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_lt_u32(a, b)); } 2398 BL_INLINE_NODEBUG __m128i simd_cmp_le_u32(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_gt_u32(a, b)); } 2399 #endif 2400 2401 #if defined(BL_TARGET_OPT_AVX512) 2402 BL_INLINE_NODEBUG __m128i simd_cmp_gt_u64(const __m128i& a, const __m128i& b) noexcept { return simd_128i_from_mask64(_mm_cmp_epu64_mask(a, b, _MM_CMPINT_NLE)); } 2403 BL_INLINE_NODEBUG __m128i simd_cmp_lt_u64(const __m128i& a, const __m128i& b) noexcept { return simd_128i_from_mask64(_mm_cmp_epu64_mask(a, b, _MM_CMPINT_LT)); } 2404 #elif defined(BL_TARGET_OPT_SSE4_2) 2405 BL_INLINE_NODEBUG __m128i simd_cmp_gt_u64(const __m128i& a, const __m128i& b) noexcept { return _mm_cmpgt_epi64(simd_flip_sign_i64(a), simd_flip_sign_i64(b)); } 2406 BL_INLINE_NODEBUG __m128i simd_cmp_lt_u64(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_gt_u64(b, a); } 2407 #else 2408 BL_INLINE_NODEBUG __m128i simd_cmp_gt_u64(const __m128i& a, const __m128i& b) noexcept { 2409 __m128i msk = _mm_andnot_si128(_mm_xor_si128(b, a), _mm_sub_epi64(b, a)); 2410 msk = _mm_or_si128(msk, _mm_andnot_si128(b, a)); 2411 return _mm_shuffle_epi32(_mm_srai_epi32(msk, 31), _MM_SHUFFLE(3, 3, 1, 1)); 2412 } 2413 BL_INLINE_NODEBUG __m128i simd_cmp_lt_u64(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_gt_u64(b, a); } 2414 #endif 2415 2416 #if defined(BL_TARGET_OPT_AVX512) 2417 BL_INLINE_NODEBUG __m128i simd_cmp_ge_u64(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i64(_mm_min_epu64(a, b), b); } 2418 BL_INLINE_NODEBUG __m128i simd_cmp_le_u64(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_eq_i64(_mm_max_epu64(a, b), b); } 2419 #else 2420 BL_INLINE_NODEBUG __m128i simd_cmp_ge_u64(const __m128i& a, const __m128i& b) noexcept { return simd_not(simd_cmp_gt_u64(b, a)); } 2421 BL_INLINE_NODEBUG __m128i simd_cmp_le_u64(const __m128i& a, const __m128i& b) noexcept { return simd_cmp_ge_u64(b, a); } 2422 #endif 2423 2424 BL_INLINE_NODEBUG __m128i simd_min_i16(const __m128i& a, const __m128i& b) noexcept { return _mm_min_epi16(a, b); } 2425 BL_INLINE_NODEBUG __m128i simd_max_i16(const __m128i& a, const __m128i& b) noexcept { return _mm_max_epi16(a, b); } 2426 BL_INLINE_NODEBUG __m128i simd_min_u8(const __m128i& a, const __m128i& b) noexcept { return _mm_min_epu8(a, b); } 2427 BL_INLINE_NODEBUG __m128i simd_max_u8(const __m128i& a, const __m128i& b) noexcept { return _mm_max_epu8(a, b); } 2428 2429 #if defined(BL_TARGET_OPT_SSE4_1) 2430 BL_INLINE_NODEBUG __m128i simd_min_i8(const __m128i& a, const __m128i& b) noexcept { return _mm_min_epi8(a, b); } 2431 BL_INLINE_NODEBUG __m128i simd_max_i8(const __m128i& a, const __m128i& b) noexcept { return _mm_max_epi8(a, b); } 2432 BL_INLINE_NODEBUG __m128i simd_min_i32(const __m128i& a, const __m128i& b) noexcept { return _mm_min_epi32(a, b); } 2433 BL_INLINE_NODEBUG __m128i simd_max_i32(const __m128i& a, const __m128i& b) noexcept { return _mm_max_epi32(a, b); } 2434 BL_INLINE_NODEBUG __m128i simd_min_u16(const __m128i& a, const __m128i& b) noexcept { return _mm_min_epu16(a, b); } 2435 BL_INLINE_NODEBUG __m128i simd_max_u16(const __m128i& a, const __m128i& b) noexcept { return _mm_max_epu16(a, b); } 2436 BL_INLINE_NODEBUG __m128i simd_min_u32(const __m128i& a, const __m128i& b) noexcept { return _mm_min_epu32(a, b); } 2437 BL_INLINE_NODEBUG __m128i simd_max_u32(const __m128i& a, const __m128i& b) noexcept { return _mm_max_epu32(a, b); } 2438 #else 2439 BL_INLINE_NODEBUG __m128i simd_min_i8(const __m128i& a, const __m128i& b) noexcept { return simd_blendv_u8(a, b, _mm_cmpgt_epi8(a, b)); } 2440 BL_INLINE_NODEBUG __m128i simd_max_i8(const __m128i& a, const __m128i& b) noexcept { return simd_blendv_u8(b, a, _mm_cmpgt_epi8(a, b)); } 2441 BL_INLINE_NODEBUG __m128i simd_min_i32(const __m128i& a, const __m128i& b) noexcept { return simd_blendv_u8(a, b, _mm_cmpgt_epi32(a, b)); } 2442 BL_INLINE_NODEBUG __m128i simd_max_i32(const __m128i& a, const __m128i& b) noexcept { return simd_blendv_u8(b, a, _mm_cmpgt_epi32(a, b)); } 2443 BL_INLINE_NODEBUG __m128i simd_min_u16(const __m128i& a, const __m128i& b) noexcept { return _mm_sub_epi16(a, _mm_subs_epu16(a, b)); } 2444 BL_INLINE_NODEBUG __m128i simd_max_u16(const __m128i& a, const __m128i& b) noexcept { return _mm_add_epi16(a, _mm_subs_epu16(b, a)); } 2445 BL_INLINE_NODEBUG __m128i simd_min_u32(const __m128i& a, const __m128i& b) noexcept { return simd_blendv_u8(a, b, _mm_cmpgt_epi32(simd_flip_sign_i32(a), simd_flip_sign_i32(b))); } 2446 BL_INLINE_NODEBUG __m128i simd_max_u32(const __m128i& a, const __m128i& b) noexcept { return simd_blendv_u8(b, a, _mm_cmpgt_epi32(simd_flip_sign_i32(a), simd_flip_sign_i32(b))); } 2447 #endif 2448 2449 #if defined(BL_TARGET_OPT_AVX512) 2450 BL_INLINE_NODEBUG __m128i simd_min_i64(const __m128i& a, const __m128i& b) noexcept { return _mm_min_epi64(a, b); } 2451 BL_INLINE_NODEBUG __m128i simd_max_i64(const __m128i& a, const __m128i& b) noexcept { return _mm_max_epi64(a, b); } 2452 BL_INLINE_NODEBUG __m128i simd_min_u64(const __m128i& a, const __m128i& b) noexcept { return _mm_min_epu64(a, b); } 2453 BL_INLINE_NODEBUG __m128i simd_max_u64(const __m128i& a, const __m128i& b) noexcept { return _mm_max_epu64(a, b); } 2454 #else 2455 BL_INLINE_NODEBUG __m128i simd_min_i64(const __m128i& a, const __m128i& b) noexcept { return simd_blendv_u8(a, b, simd_cmp_gt_i64(a, b)); } 2456 BL_INLINE_NODEBUG __m128i simd_max_i64(const __m128i& a, const __m128i& b) noexcept { return simd_blendv_u8(a, b, simd_cmp_gt_i64(b, a)); } 2457 BL_INLINE_NODEBUG __m128i simd_min_u64(const __m128i& a, const __m128i& b) noexcept { return simd_blendv_u8(a, b, simd_cmp_gt_i64(simd_flip_sign_i64(a), simd_flip_sign_i64(b))); } 2458 BL_INLINE_NODEBUG __m128i simd_max_u64(const __m128i& a, const __m128i& b) noexcept { return simd_blendv_u8(b, a, simd_cmp_gt_i64(simd_flip_sign_i64(a), simd_flip_sign_i64(b))); } 2459 #endif 2460 2461 #if defined(BL_TARGET_OPT_SSSE3) 2462 BL_INLINE_NODEBUG __m128i simd_abs_i8(const __m128i& a) noexcept { return _mm_abs_epi8(a); } 2463 BL_INLINE_NODEBUG __m128i simd_abs_i16(const __m128i& a) noexcept { return _mm_abs_epi16(a); } 2464 BL_INLINE_NODEBUG __m128i simd_abs_i32(const __m128i& a) noexcept { return _mm_abs_epi32(a); } 2465 #else 2466 BL_INLINE_NODEBUG __m128i simd_abs_i8(const __m128i& a) noexcept { 2467 __m128i neg_val = _mm_sub_epi8(_mm_setzero_si128(), a); 2468 return _mm_min_epu8(neg_val, a); 2469 } 2470 BL_INLINE_NODEBUG __m128i simd_abs_i16(const __m128i& a) noexcept { 2471 __m128i neg_val = _mm_sub_epi16(_mm_setzero_si128(), a); 2472 return _mm_max_epi16(neg_val, a); 2473 } 2474 BL_INLINE_NODEBUG __m128i simd_abs_i32(const __m128i& a) noexcept { 2475 __m128i neg_msk = _mm_srai_epi32(a, 31); 2476 return _mm_sub_epi32(simd_xor(a, neg_msk), neg_msk); 2477 } 2478 #endif 2479 2480 #if defined(BL_TARGET_OPT_AVX512) 2481 BL_INLINE_NODEBUG __m128i simd_abs_i64(const __m128i& a) noexcept { return _mm_abs_epi64(a); } 2482 #else 2483 BL_INLINE_NODEBUG __m128i simd_abs_i64(const __m128i& a) noexcept { 2484 __m128i neg_mask = _mm_srai_epi32(_mm_shuffle_epi32(a, _MM_SHUFFLE(3, 3, 1, 1)), 31); 2485 return _mm_sub_epi64(simd_xor(a, neg_mask), neg_mask); 2486 } 2487 #endif 2488 2489 template<uint8_t kN> BL_INLINE_NODEBUG __m128i simd_slli_i8(const __m128i& a) noexcept { 2490 constexpr uint8_t kShift = uint8_t(kN & 0x7); 2491 2492 if constexpr (kShift == 0) { 2493 return a; 2494 } 2495 else if constexpr (kShift == 1) { 2496 return _mm_add_epi8(a, a); 2497 } 2498 else { 2499 __m128i msk = _mm_set1_epi8(int8_t((0xFFu << kShift) & 0xFFu)); 2500 return _mm_and_si128(_mm_slli_epi16(a, kShift), msk); 2501 } 2502 } 2503 2504 template<uint8_t kN> BL_INLINE_NODEBUG __m128i simd_srli_u8(const __m128i& a) noexcept { 2505 constexpr uint8_t kShift = uint8_t(kN & 0x7); 2506 2507 if constexpr (kShift == 0) { 2508 return a; 2509 } 2510 else { 2511 __m128i msk = _mm_set1_epi8(int8_t((0xFFu >> kShift) & 0xFFu)); 2512 return _mm_and_si128(_mm_srli_epi16(a, kShift), msk); 2513 } 2514 } 2515 2516 template<uint8_t kN> BL_INLINE_NODEBUG __m128i simd_srai_i8(const __m128i& a) noexcept { 2517 constexpr uint8_t kShift = uint8_t(kN & 0x7); 2518 2519 if constexpr (kShift == 0) { 2520 return a; 2521 } 2522 else if constexpr (kShift == 7) { 2523 return _mm_cmpgt_epi8(simd_make_zero<__m128i>(), a); 2524 } 2525 else { 2526 __m128i tmp = simd_srli_u8<kShift>(a); 2527 __m128i sgn = simd_make128_u8(0x80u >> kShift); 2528 return _mm_sub_epi8(simd_xor(tmp, sgn), sgn); 2529 } 2530 } 2531 2532 template<uint8_t kN> BL_INLINE_NODEBUG __m128i simd_slli_i16(const __m128i& a) noexcept { return kN ? _mm_slli_epi16(a, kN) : a; } 2533 template<uint8_t kN> BL_INLINE_NODEBUG __m128i simd_slli_i32(const __m128i& a) noexcept { return kN ? _mm_slli_epi32(a, kN) : a; } 2534 template<uint8_t kN> BL_INLINE_NODEBUG __m128i simd_slli_i64(const __m128i& a) noexcept { return kN ? _mm_slli_epi64(a, kN) : a; } 2535 2536 template<uint8_t kN> BL_INLINE_NODEBUG __m128i simd_srli_u16(const __m128i& a) noexcept { return kN ? _mm_srli_epi16(a, kN) : a; } 2537 template<uint8_t kN> BL_INLINE_NODEBUG __m128i simd_srli_u32(const __m128i& a) noexcept { return kN ? _mm_srli_epi32(a, kN) : a; } 2538 template<uint8_t kN> BL_INLINE_NODEBUG __m128i simd_srli_u64(const __m128i& a) noexcept { return kN ? _mm_srli_epi64(a, kN) : a; } 2539 2540 template<uint8_t kN> BL_INLINE_NODEBUG __m128i simd_srai_i16(const __m128i& a) noexcept { return kN ? _mm_srai_epi16(a, kN) : a; } 2541 template<uint8_t kN> BL_INLINE_NODEBUG __m128i simd_srai_i32(const __m128i& a) noexcept { return kN ? _mm_srai_epi32(a, kN) : a; } 2542 2543 template<uint8_t kN> BL_INLINE_NODEBUG __m128i simd_srai_i64(const __m128i& a) noexcept { 2544 #if defined(BL_TARGET_OPT_AVX512) 2545 return kN ? _mm_srai_epi64(a, kN) : a; 2546 #else 2547 if constexpr (kN == 0) { 2548 return a; 2549 } 2550 else if constexpr (kN == 63u) { 2551 return _mm_srai_epi32(_mm_shuffle_epi32(a, _MM_SHUFFLE(3, 3, 1, 1)), 31); 2552 } 2553 #if defined(BL_TARGET_OPT_SSE4_1) 2554 else if constexpr (kN < 32u) { 2555 __m128i hi = _mm_srai_epi32(a, kN & 31u); 2556 __m128i lo = _mm_srli_epi64(a, kN & 31u); 2557 return _mm_blend_epi16(lo, hi, 0xCCu); 2558 } 2559 #endif 2560 else { 2561 __m128i highs = _mm_shuffle_epi32(a, _MM_SHUFFLE(3, 3, 1, 1)); 2562 __m128i signs = _mm_srai_epi32(highs, 31); 2563 __m128i msk = _mm_slli_epi64(signs, (64 - kN) & 63); 2564 return _mm_or_si128(msk, _mm_srli_epi64(a, kN)); 2565 } 2566 #endif 2567 } 2568 2569 template<uint8_t kNumBytes> BL_INLINE_NODEBUG __m128i simd_sllb_u128(const __m128i& a) noexcept { return kNumBytes ? _mm_slli_si128(a, kNumBytes) : a; } 2570 template<uint8_t kNumBytes> BL_INLINE_NODEBUG __m128i simd_srlb_u128(const __m128i& a) noexcept { return kNumBytes ? _mm_srli_si128(a, kNumBytes) : a; } 2571 2572 BL_INLINE_NODEBUG __m128i simd_sad_u8_u64(const __m128i& a, const __m128i& b) noexcept { return _mm_sad_epu8(a, b); } 2573 2574 #if defined(BL_TARGET_OPT_SSSE3) 2575 BL_INLINE_NODEBUG __m128i simd_maddws_u8xi8_i16(const __m128i& a, const __m128i& b) noexcept { return _mm_maddubs_epi16(a, b); } 2576 #endif // BL_TARGET_OPT_SSSE3 2577 2578 #if defined(BL_TARGET_OPT_SSE4_2) 2579 BL_INLINE_NODEBUG __m128i simd_clmul_u128_ll(const __m128i& a, const __m128i& b) noexcept { return _mm_clmulepi64_si128(a, b, 0x00); } 2580 BL_INLINE_NODEBUG __m128i simd_clmul_u128_lh(const __m128i& a, const __m128i& b) noexcept { return _mm_clmulepi64_si128(a, b, 0x10); } 2581 BL_INLINE_NODEBUG __m128i simd_clmul_u128_hl(const __m128i& a, const __m128i& b) noexcept { return _mm_clmulepi64_si128(a, b, 0x01); } 2582 BL_INLINE_NODEBUG __m128i simd_clmul_u128_hh(const __m128i& a, const __m128i& b) noexcept { return _mm_clmulepi64_si128(a, b, 0x11); } 2583 #endif // BL_TARGET_OPT_SSE4_2 2584 2585 #if defined(BL_TARGET_OPT_AVX512) 2586 template<uint8_t kPred> 2587 BL_INLINE_NODEBUG __m256i simd_ternlog(const __m256i& a, const __m256i& b, const __m256i& c) noexcept { return _mm256_ternarylogic_epi32(a, b, c, kPred); } 2588 2589 template<uint8_t kPred> 2590 BL_INLINE_NODEBUG __m256 simd_ternlog(const __m256& a, const __m256& b, const __m256& c) noexcept { return simd_as_f(simd_ternlog<kPred>(simd_as_i(a), simd_as_i(b), simd_as_i(c))); } 2591 2592 template<uint8_t kPred> 2593 BL_INLINE_NODEBUG __m256d simd_ternlog(const __m256d& a, const __m256d& b, const __m256d& c) noexcept { return simd_as_d(simd_ternlog<kPred>(simd_as_i(a), simd_as_i(b), simd_as_i(c))); } 2594 #endif // BL_TARGET_OPT_AVX512 2595 2596 #if defined(BL_TARGET_OPT_AVX) 2597 BL_INLINE_NODEBUG __m256 simd_and(const __m256& a, const __m256& b) noexcept { return _mm256_and_ps(a, b); } 2598 BL_INLINE_NODEBUG __m256d simd_and(const __m256d& a, const __m256d& b) noexcept { return _mm256_and_pd(a, b); } 2599 2600 BL_INLINE_NODEBUG __m256 simd_andnot(const __m256& a, const __m256& b) noexcept { return _mm256_andnot_ps(a, b); } 2601 BL_INLINE_NODEBUG __m256d simd_andnot(const __m256d& a, const __m256d& b) noexcept { return _mm256_andnot_pd(a, b); } 2602 2603 BL_INLINE_NODEBUG __m256 simd_or(const __m256& a, const __m256& b) noexcept { return _mm256_or_ps(a, b); } 2604 BL_INLINE_NODEBUG __m256d simd_or(const __m256d& a, const __m256d& b) noexcept { return _mm256_or_pd(a, b); } 2605 2606 BL_INLINE_NODEBUG __m256 simd_xor(const __m256& a, const __m256& b) noexcept { return _mm256_xor_ps(a, b); } 2607 BL_INLINE_NODEBUG __m256d simd_xor(const __m256d& a, const __m256d& b) noexcept { return _mm256_xor_pd(a, b); } 2608 2609 #if defined(BL_TARGET_OPT_AVX512) 2610 BL_INLINE_NODEBUG __m256 simd_not(const __m256& a) noexcept { return simd_ternlog<0x55>(a, a, a); } 2611 BL_INLINE_NODEBUG __m256d simd_not(const __m256d& a) noexcept { return simd_ternlog<0x55>(a, a, a); } 2612 2613 BL_INLINE_NODEBUG __m256 simd_blendv_bits(const __m256& a, const __m256& b, const __m256& msk) noexcept { return simd_ternlog<0xD8>(a, b, msk); } 2614 BL_INLINE_NODEBUG __m256d simd_blendv_bits(const __m256d& a, const __m256d& b, const __m256d& msk) noexcept { return simd_ternlog<0xD8>(a, b, msk); } 2615 #else 2616 BL_INLINE_NODEBUG __m256 simd_not(const __m256& a) noexcept { return simd_xor(a, simd_make_ones<__m256>()); } 2617 BL_INLINE_NODEBUG __m256d simd_not(const __m256d& a) noexcept { return simd_xor(a, simd_make_ones<__m256d>()); } 2618 2619 BL_INLINE_NODEBUG __m256 simd_blendv_bits(const __m256& a, const __m256& b, const __m256& msk) noexcept { return simd_or(simd_and(b, msk), simd_andnot(msk, a)); } 2620 BL_INLINE_NODEBUG __m256d simd_blendv_bits(const __m256d& a, const __m256d& b, const __m256d& msk) noexcept { return simd_or(simd_and(b, msk), simd_andnot(msk, a)); } 2621 #endif 2622 2623 #if defined(BL_TARGET_OPT_AVX2) 2624 BL_INLINE_NODEBUG __m256i simd_and(const __m256i& a, const __m256i& b) noexcept { return _mm256_and_si256(a, b); } 2625 BL_INLINE_NODEBUG __m256i simd_andnot(const __m256i& a, const __m256i& b) noexcept { return _mm256_andnot_si256(a, b); } 2626 BL_INLINE_NODEBUG __m256i simd_or(const __m256i& a, const __m256i& b) noexcept { return _mm256_or_si256(a, b); } 2627 BL_INLINE_NODEBUG __m256i simd_xor(const __m256i& a, const __m256i& b) noexcept { return _mm256_xor_si256(a, b); } 2628 #else 2629 BL_INLINE_NODEBUG __m256i simd_and(const __m256i& a, const __m256i& b) noexcept { return simd_as_i(_mm256_and_ps(simd_as_f(a), simd_as_f(b))); } 2630 BL_INLINE_NODEBUG __m256i simd_andnot(const __m256i& a, const __m256i& b) noexcept { return simd_as_i(_mm256_andnot_ps(simd_as_f(a), simd_as_f(b))); } 2631 BL_INLINE_NODEBUG __m256i simd_or(const __m256i& a, const __m256i& b) noexcept { return simd_as_i(_mm256_or_ps(simd_as_f(a), simd_as_f(b))); } 2632 BL_INLINE_NODEBUG __m256i simd_xor(const __m256i& a, const __m256i& b) noexcept { return simd_as_i(_mm256_xor_ps(simd_as_f(a), simd_as_f(b))); } 2633 #endif 2634 2635 #if defined(BL_TARGET_OPT_AVX512) 2636 BL_INLINE_NODEBUG __m256i simd_not(const __m256i& a) noexcept { return simd_ternlog<0x55>(a, a, a); } 2637 BL_INLINE_NODEBUG __m256i simd_blendv_bits(const __m256i& a, const __m256i& b, const __m256i& msk) noexcept { return simd_ternlog<0xD8>(a, b, msk); } 2638 #else 2639 BL_INLINE_NODEBUG __m256i simd_not(const __m256i& a) noexcept { return simd_xor(a, simd_make_ones<__m256i>()); } 2640 BL_INLINE_NODEBUG __m256i simd_blendv_bits(const __m256i& a, const __m256i& b, const __m256i& msk) noexcept { return simd_or(simd_and(b, msk), simd_andnot(msk, a)); } 2641 #endif 2642 2643 #if defined(BL_TARGET_OPT_AVX2) 2644 BL_INLINE_NODEBUG __m256i simd_blendv_u8(const __m256i& a, const __m256i& b, const __m256i& msk) noexcept { return _mm256_blendv_epi8(a, b, msk); } 2645 #endif 2646 2647 BL_INLINE_NODEBUG __m256i simd_flip_sign_i8(const __m256i& a) noexcept { return simd_xor(a, simd_make256_u32(0x80808080u)); } 2648 BL_INLINE_NODEBUG __m256i simd_flip_sign_i16(const __m256i& a) noexcept { return simd_xor(a, simd_make256_u32(0x80008000u)); } 2649 BL_INLINE_NODEBUG __m256i simd_flip_sign_i32(const __m256i& a) noexcept { return simd_xor(a, simd_make256_u32(0x80000000u)); } 2650 BL_INLINE_NODEBUG __m256i simd_flip_sign_i64(const __m256i& a) noexcept { return simd_xor(a, simd_make256_u64(uint64_t(1) << 63)); } 2651 2652 BL_INLINE_NODEBUG __m256 simd_add_f32(const __m256& a, const __m256& b) noexcept { return _mm256_add_ps(a, b); } 2653 BL_INLINE_NODEBUG __m256d simd_add_f64(const __m256d& a, const __m256d& b) noexcept { return _mm256_add_pd(a, b); } 2654 2655 BL_INLINE_NODEBUG __m256 simd_sub_f32(const __m256& a, const __m256& b) noexcept { return _mm256_sub_ps(a, b); } 2656 BL_INLINE_NODEBUG __m256d simd_sub_f64(const __m256d& a, const __m256d& b) noexcept { return _mm256_sub_pd(a, b); } 2657 2658 BL_INLINE_NODEBUG __m256 simd_mul_f32(const __m256& a, const __m256& b) noexcept { return _mm256_mul_ps(a, b); } 2659 BL_INLINE_NODEBUG __m256d simd_mul_f64(const __m256d& a, const __m256d& b) noexcept { return _mm256_mul_pd(a, b); } 2660 2661 BL_INLINE_NODEBUG __m256 simd_div_f32(const __m256& a, const __m256& b) noexcept { return _mm256_div_ps(a, b); } 2662 BL_INLINE_NODEBUG __m256d simd_div_f64(const __m256d& a, const __m256d& b) noexcept { return _mm256_div_pd(a, b); } 2663 2664 BL_INLINE_NODEBUG __m256 simd_cmp_eq_f32(const __m256& a, const __m256& b) noexcept { return _mm256_cmp_ps(a, b, _CMP_EQ_OQ); } 2665 BL_INLINE_NODEBUG __m256d simd_cmp_eq_f64(const __m256d& a, const __m256d& b) noexcept { return _mm256_cmp_pd(a, b, _CMP_EQ_OQ); } 2666 2667 BL_INLINE_NODEBUG __m256 simd_cmp_ne_f32(const __m256& a, const __m256& b) noexcept { return _mm256_cmp_ps(a, b, _CMP_NEQ_OQ); } 2668 BL_INLINE_NODEBUG __m256d simd_cmp_ne_f64(const __m256d& a, const __m256d& b) noexcept { return _mm256_cmp_pd(a, b, _CMP_NEQ_OQ); } 2669 2670 BL_INLINE_NODEBUG __m256 simd_cmp_lt_f32(const __m256& a, const __m256& b) noexcept { return _mm256_cmp_ps(a, b, _CMP_LT_OQ); } 2671 BL_INLINE_NODEBUG __m256d simd_cmp_lt_f64(const __m256d& a, const __m256d& b) noexcept { return _mm256_cmp_pd(a, b, _CMP_LT_OQ); } 2672 2673 BL_INLINE_NODEBUG __m256 simd_cmp_le_f32(const __m256& a, const __m256& b) noexcept { return _mm256_cmp_ps(a, b, _CMP_LE_OQ); } 2674 BL_INLINE_NODEBUG __m256d simd_cmp_le_f64(const __m256d& a, const __m256d& b) noexcept { return _mm256_cmp_pd(a, b, _CMP_LE_OQ); } 2675 2676 BL_INLINE_NODEBUG __m256 simd_cmp_gt_f32(const __m256& a, const __m256& b) noexcept { return _mm256_cmp_ps(a, b, _CMP_GT_OQ); } 2677 BL_INLINE_NODEBUG __m256d simd_cmp_gt_f64(const __m256d& a, const __m256d& b) noexcept { return _mm256_cmp_pd(a, b, _CMP_GT_OQ); } 2678 2679 BL_INLINE_NODEBUG __m256 simd_cmp_ge_f32(const __m256& a, const __m256& b) noexcept { return _mm256_cmp_ps(a, b, _CMP_GE_OQ); } 2680 BL_INLINE_NODEBUG __m256d simd_cmp_ge_f64(const __m256d& a, const __m256d& b) noexcept { return _mm256_cmp_pd(a, b, _CMP_GE_OQ); } 2681 2682 BL_INLINE_NODEBUG __m256 simd_min_f32(const __m256& a, const __m256& b) noexcept { return _mm256_min_ps(a, b); } 2683 BL_INLINE_NODEBUG __m256d simd_min_f64(const __m256d& a, const __m256d& b) noexcept { return _mm256_min_pd(a, b); } 2684 2685 BL_INLINE_NODEBUG __m256 simd_max_f32(const __m256& a, const __m256& b) noexcept { return _mm256_max_ps(a, b); } 2686 BL_INLINE_NODEBUG __m256d simd_max_f64(const __m256d& a, const __m256d& b) noexcept { return _mm256_max_pd(a, b); } 2687 2688 BL_INLINE_NODEBUG __m256 simd_abs_f32(const __m256& a) noexcept { return _mm256_and_ps(a, bl::common_table.p_7FFFFFFF7FFFFFFF.as<__m256>()); } 2689 BL_INLINE_NODEBUG __m256d simd_abs_f64(const __m256d& a) noexcept { return _mm256_and_pd(a, bl::common_table.p_7FFFFFFFFFFFFFFF.as<__m256d>()); } 2690 2691 BL_INLINE_NODEBUG __m256 simd_sqrt_f32(const __m256& a) noexcept { return _mm256_sqrt_ps(a); } 2692 BL_INLINE_NODEBUG __m256d simd_sqrt_f64(const __m256d& a) noexcept { return _mm256_sqrt_pd(a); } 2693 #endif // BL_TARGET_OPT_AVX 2694 2695 #if defined(BL_TARGET_OPT_AVX2) 2696 BL_INLINE_NODEBUG __m256i simd_add_i8(const __m256i& a, const __m256i& b) noexcept { return _mm256_add_epi8(a, b); } 2697 BL_INLINE_NODEBUG __m256i simd_add_i16(const __m256i& a, const __m256i& b) noexcept { return _mm256_add_epi16(a, b); } 2698 BL_INLINE_NODEBUG __m256i simd_add_i32(const __m256i& a, const __m256i& b) noexcept { return _mm256_add_epi32(a, b); } 2699 BL_INLINE_NODEBUG __m256i simd_add_i64(const __m256i& a, const __m256i& b) noexcept { return _mm256_add_epi64(a, b); } 2700 BL_INLINE_NODEBUG __m256i simd_add_u8(const __m256i& a, const __m256i& b) noexcept { return _mm256_add_epi8(a, b); } 2701 BL_INLINE_NODEBUG __m256i simd_add_u16(const __m256i& a, const __m256i& b) noexcept { return _mm256_add_epi16(a, b); } 2702 BL_INLINE_NODEBUG __m256i simd_add_u32(const __m256i& a, const __m256i& b) noexcept { return _mm256_add_epi32(a, b); } 2703 BL_INLINE_NODEBUG __m256i simd_add_u64(const __m256i& a, const __m256i& b) noexcept { return _mm256_add_epi64(a, b); } 2704 2705 BL_INLINE_NODEBUG __m256i simd_adds_i8(const __m256i& a, const __m256i& b) noexcept { return _mm256_adds_epi8(a, b); } 2706 BL_INLINE_NODEBUG __m256i simd_adds_i16(const __m256i& a, const __m256i& b) noexcept { return _mm256_adds_epi16(a, b); } 2707 BL_INLINE_NODEBUG __m256i simd_adds_u8(const __m256i& a, const __m256i& b) noexcept { return _mm256_adds_epu8(a, b); } 2708 BL_INLINE_NODEBUG __m256i simd_adds_u16(const __m256i& a, const __m256i& b) noexcept { return _mm256_adds_epu16(a, b); } 2709 2710 BL_INLINE_NODEBUG __m256i simd_sub_i8(const __m256i& a, const __m256i& b) noexcept { return _mm256_sub_epi8(a, b); } 2711 BL_INLINE_NODEBUG __m256i simd_sub_i16(const __m256i& a, const __m256i& b) noexcept { return _mm256_sub_epi16(a, b); } 2712 BL_INLINE_NODEBUG __m256i simd_sub_i32(const __m256i& a, const __m256i& b) noexcept { return _mm256_sub_epi32(a, b); } 2713 BL_INLINE_NODEBUG __m256i simd_sub_i64(const __m256i& a, const __m256i& b) noexcept { return _mm256_sub_epi64(a, b); } 2714 BL_INLINE_NODEBUG __m256i simd_sub_u8(const __m256i& a, const __m256i& b) noexcept { return _mm256_sub_epi8(a, b); } 2715 BL_INLINE_NODEBUG __m256i simd_sub_u16(const __m256i& a, const __m256i& b) noexcept { return _mm256_sub_epi16(a, b); } 2716 BL_INLINE_NODEBUG __m256i simd_sub_u32(const __m256i& a, const __m256i& b) noexcept { return _mm256_sub_epi32(a, b); } 2717 BL_INLINE_NODEBUG __m256i simd_sub_u64(const __m256i& a, const __m256i& b) noexcept { return _mm256_sub_epi64(a, b); } 2718 2719 BL_INLINE_NODEBUG __m256i simd_subs_i8(const __m256i& a, const __m256i& b) noexcept { return _mm256_subs_epi8(a, b); } 2720 BL_INLINE_NODEBUG __m256i simd_subs_i16(const __m256i& a, const __m256i& b) noexcept { return _mm256_subs_epi16(a, b); } 2721 BL_INLINE_NODEBUG __m256i simd_subs_u8(const __m256i& a, const __m256i& b) noexcept { return _mm256_subs_epu8(a, b); } 2722 BL_INLINE_NODEBUG __m256i simd_subs_u16(const __m256i& a, const __m256i& b) noexcept { return _mm256_subs_epu16(a, b); } 2723 2724 BL_INLINE_NODEBUG __m256i simd_mul_i16(const __m256i& a, const __m256i& b) noexcept { return _mm256_mullo_epi16(a, b); } 2725 BL_INLINE_NODEBUG __m256i simd_mul_u16(const __m256i& a, const __m256i& b) noexcept { return _mm256_mullo_epi16(a, b); } 2726 2727 BL_INLINE_NODEBUG __m256i simd_mul_i32(const __m256i& a, const __m256i& b) noexcept { return _mm256_mullo_epi32(a, b); } 2728 BL_INLINE_NODEBUG __m256i simd_mul_u32(const __m256i& a, const __m256i& b) noexcept { return _mm256_mullo_epi32(a, b); } 2729 2730 #if defined(BL_TARGET_OPT_AVX512) 2731 BL_INLINE_NODEBUG __m256i simd_mul_i64(const __m256i& a, const __m256i& b) noexcept { return _mm256_mullo_epi64(a, b); } 2732 #else 2733 BL_INLINE_NODEBUG __m256i simd_mul_i64(const __m256i& a, const __m256i& b) noexcept { 2734 __m256i al_bh = _mm256_mul_epu32(a, _mm256_srli_epi64(b, 32)); 2735 __m256i ah_bl = _mm256_mul_epu32(b, _mm256_srli_epi64(a, 32)); 2736 __m256i al_bl = _mm256_mul_epu32(a, b); 2737 __m256i prod1 = _mm256_slli_epi64(_mm256_add_epi64(al_bh, ah_bl), 32); 2738 return _mm256_add_epi64(al_bl, prod1); 2739 } 2740 #endif 2741 BL_INLINE_NODEBUG __m256i simd_mul_u64(const __m256i& a, const __m256i& b) noexcept { return simd_mul_i64(a, b); } 2742 2743 BL_INLINE_NODEBUG __m256i simd_mulh_i16(const __m256i& a, const __m256i& b) noexcept { return _mm256_mulhi_epi16(a, b); } 2744 BL_INLINE_NODEBUG __m256i simd_mulh_u16(const __m256i& a, const __m256i& b) noexcept { return _mm256_mulhi_epu16(a, b); } 2745 BL_INLINE_NODEBUG __m256i simd_mulw_i32(const __m256i& a, const __m256i& b) noexcept { return _mm256_mul_epi32(a, b); } 2746 BL_INLINE_NODEBUG __m256i simd_mulw_u32(const __m256i& a, const __m256i& b) noexcept { return _mm256_mul_epu32(a, b); } 2747 2748 BL_INLINE_NODEBUG __m256i simd_maddw_i16_i32(const __m256i& a, const __m256i& b) noexcept { return _mm256_madd_epi16(a, b); } 2749 2750 BL_INLINE_NODEBUG __m256i simd_cmp_eq_i8(const __m256i& a, const __m256i& b) noexcept { return _mm256_cmpeq_epi8(a, b); } 2751 BL_INLINE_NODEBUG __m256i simd_cmp_eq_i16(const __m256i& a, const __m256i& b) noexcept { return _mm256_cmpeq_epi16(a, b); } 2752 BL_INLINE_NODEBUG __m256i simd_cmp_eq_i32(const __m256i& a, const __m256i& b) noexcept { return _mm256_cmpeq_epi32(a, b); } 2753 BL_INLINE_NODEBUG __m256i simd_cmp_eq_i64(const __m256i& a, const __m256i& b) noexcept { return _mm256_cmpeq_epi64(a, b); } 2754 2755 BL_INLINE_NODEBUG __m256i simd_cmp_ne_i8(const __m256i& a, const __m256i& b) noexcept { return simd_not(simd_cmp_eq_i8(a, b)); } 2756 BL_INLINE_NODEBUG __m256i simd_cmp_ne_i16(const __m256i& a, const __m256i& b) noexcept { return simd_not(simd_cmp_eq_i16(a, b)); } 2757 BL_INLINE_NODEBUG __m256i simd_cmp_ne_i32(const __m256i& a, const __m256i& b) noexcept { return simd_not(simd_cmp_eq_i32(a, b)); } 2758 BL_INLINE_NODEBUG __m256i simd_cmp_ne_i64(const __m256i& a, const __m256i& b) noexcept { return simd_not(simd_cmp_eq_i64(a, b)); } 2759 2760 BL_INLINE_NODEBUG __m256i simd_cmp_gt_i8(const __m256i& a, const __m256i& b) noexcept { return _mm256_cmpgt_epi8(a, b); } 2761 BL_INLINE_NODEBUG __m256i simd_cmp_lt_i8(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_gt_i8(b, a); } 2762 BL_INLINE_NODEBUG __m256i simd_cmp_gt_i16(const __m256i& a, const __m256i& b) noexcept { return _mm256_cmpgt_epi16(a, b); } 2763 BL_INLINE_NODEBUG __m256i simd_cmp_lt_i16(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_gt_i16(b, a); } 2764 BL_INLINE_NODEBUG __m256i simd_cmp_gt_i32(const __m256i& a, const __m256i& b) noexcept { return _mm256_cmpgt_epi32(a, b); } 2765 BL_INLINE_NODEBUG __m256i simd_cmp_lt_i32(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_gt_i32(b, a); } 2766 2767 BL_INLINE_NODEBUG __m256i simd_cmp_ge_i8(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_eq_i8(_mm256_min_epi8(a, b), b); } 2768 BL_INLINE_NODEBUG __m256i simd_cmp_le_i8(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_eq_i8(_mm256_max_epi8(a, b), b); } 2769 BL_INLINE_NODEBUG __m256i simd_cmp_ge_i16(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_eq_i16(_mm256_min_epi16(a, b), b); } 2770 BL_INLINE_NODEBUG __m256i simd_cmp_le_i16(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_eq_i16(_mm256_max_epi16(a, b), b); } 2771 BL_INLINE_NODEBUG __m256i simd_cmp_ge_i32(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_eq_i32(_mm256_min_epi32(a, b), b); } 2772 BL_INLINE_NODEBUG __m256i simd_cmp_le_i32(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_eq_i32(_mm256_max_epi32(a, b), b); } 2773 2774 BL_INLINE_NODEBUG __m256i simd_cmp_ge_u8(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_eq_i8(_mm256_min_epu8(a, b), b); } 2775 BL_INLINE_NODEBUG __m256i simd_cmp_le_u8(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_eq_i8(_mm256_max_epu8(a, b), b); } 2776 BL_INLINE_NODEBUG __m256i simd_cmp_ge_u16(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_eq_i16(_mm256_min_epu16(a, b), b); } 2777 BL_INLINE_NODEBUG __m256i simd_cmp_le_u16(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_eq_i16(_mm256_max_epu16(a, b), b); } 2778 BL_INLINE_NODEBUG __m256i simd_cmp_ge_u32(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_eq_i32(_mm256_min_epu32(a, b), b); } 2779 BL_INLINE_NODEBUG __m256i simd_cmp_le_u32(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_eq_i32(_mm256_max_epu32(a, b), b); } 2780 2781 BL_INLINE_NODEBUG __m256i simd_cmp_gt_u8(const __m256i& a, const __m256i& b) noexcept { return simd_not(simd_cmp_le_u8(a, b)); } 2782 BL_INLINE_NODEBUG __m256i simd_cmp_lt_u8(const __m256i& a, const __m256i& b) noexcept { return simd_not(simd_cmp_ge_u8(a, b)); } 2783 BL_INLINE_NODEBUG __m256i simd_cmp_gt_u16(const __m256i& a, const __m256i& b) noexcept { return simd_not(simd_cmp_le_u16(a, b)); } 2784 BL_INLINE_NODEBUG __m256i simd_cmp_lt_u16(const __m256i& a, const __m256i& b) noexcept { return simd_not(simd_cmp_ge_u16(a, b)); } 2785 BL_INLINE_NODEBUG __m256i simd_cmp_gt_u32(const __m256i& a, const __m256i& b) noexcept { return simd_not(simd_cmp_le_u32(a, b)); } 2786 BL_INLINE_NODEBUG __m256i simd_cmp_lt_u32(const __m256i& a, const __m256i& b) noexcept { return simd_not(simd_cmp_ge_u32(a, b)); } 2787 2788 BL_INLINE_NODEBUG __m256i simd_cmp_gt_i64(const __m256i& a, const __m256i& b) noexcept { return _mm256_cmpgt_epi64(a, b); } 2789 BL_INLINE_NODEBUG __m256i simd_cmp_lt_i64(const __m256i& a, const __m256i& b) noexcept { return _mm256_cmpgt_epi64(b, a); } 2790 2791 #if defined(BL_TARGET_OPT_AVX512) 2792 BL_INLINE_NODEBUG __m256i simd_cmp_ge_i64(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_eq_i64(_mm256_min_epi64(a, b), b); } 2793 BL_INLINE_NODEBUG __m256i simd_cmp_le_i64(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_eq_i64(_mm256_max_epi64(a, b), b); } 2794 BL_INLINE_NODEBUG __m256i simd_cmp_gt_u64(const __m256i& a, const __m256i& b) noexcept { return simd_256i_from_mask64(_mm256_cmp_epu64_mask(a, b, _MM_CMPINT_NLE)); } 2795 BL_INLINE_NODEBUG __m256i simd_cmp_ge_u64(const __m256i& a, const __m256i& b) noexcept { return simd_256i_from_mask64(_mm256_cmp_epu64_mask(a, b, _MM_CMPINT_NLT)); } 2796 #else 2797 BL_INLINE_NODEBUG __m256i simd_cmp_ge_i64(const __m256i& a, const __m256i& b) noexcept { return simd_not(simd_cmp_lt_i64(a, b)); } 2798 BL_INLINE_NODEBUG __m256i simd_cmp_le_i64(const __m256i& a, const __m256i& b) noexcept { return simd_not(simd_cmp_gt_i64(a, b)); } 2799 BL_INLINE_NODEBUG __m256i simd_cmp_gt_u64(const __m256i& a, const __m256i& b) noexcept { return _mm256_cmpgt_epi64(simd_flip_sign_i64(a), simd_flip_sign_i64(b)); } 2800 BL_INLINE_NODEBUG __m256i simd_cmp_ge_u64(const __m256i& a, const __m256i& b) noexcept { return simd_not(simd_cmp_gt_u64(b, a)); } 2801 #endif 2802 2803 BL_INLINE_NODEBUG __m256i simd_cmp_lt_u64(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_gt_u64(b, a); } 2804 BL_INLINE_NODEBUG __m256i simd_cmp_le_u64(const __m256i& a, const __m256i& b) noexcept { return simd_cmp_ge_u64(b, a); } 2805 2806 BL_INLINE_NODEBUG __m256i simd_min_i8(const __m256i& a, const __m256i& b) noexcept { return _mm256_min_epi8(a, b); } 2807 BL_INLINE_NODEBUG __m256i simd_min_i16(const __m256i& a, const __m256i& b) noexcept { return _mm256_min_epi16(a, b); } 2808 BL_INLINE_NODEBUG __m256i simd_min_i32(const __m256i& a, const __m256i& b) noexcept { return _mm256_min_epi32(a, b); } 2809 BL_INLINE_NODEBUG __m256i simd_min_u8(const __m256i& a, const __m256i& b) noexcept { return _mm256_min_epu8(a, b); } 2810 BL_INLINE_NODEBUG __m256i simd_min_u16(const __m256i& a, const __m256i& b) noexcept { return _mm256_min_epu16(a, b); } 2811 BL_INLINE_NODEBUG __m256i simd_min_u32(const __m256i& a, const __m256i& b) noexcept { return _mm256_min_epu32(a, b); } 2812 2813 BL_INLINE_NODEBUG __m256i simd_max_i8(const __m256i& a, const __m256i& b) noexcept { return _mm256_max_epi8(a, b); } 2814 BL_INLINE_NODEBUG __m256i simd_max_i16(const __m256i& a, const __m256i& b) noexcept { return _mm256_max_epi16(a, b); } 2815 BL_INLINE_NODEBUG __m256i simd_max_i32(const __m256i& a, const __m256i& b) noexcept { return _mm256_max_epi32(a, b); } 2816 BL_INLINE_NODEBUG __m256i simd_max_u8(const __m256i& a, const __m256i& b) noexcept { return _mm256_max_epu8(a, b); } 2817 BL_INLINE_NODEBUG __m256i simd_max_u16(const __m256i& a, const __m256i& b) noexcept { return _mm256_max_epu16(a, b); } 2818 BL_INLINE_NODEBUG __m256i simd_max_u32(const __m256i& a, const __m256i& b) noexcept { return _mm256_max_epu32(a, b); } 2819 2820 #if defined(BL_TARGET_OPT_AVX512) 2821 BL_INLINE_NODEBUG __m256i simd_min_i64(const __m256i& a, const __m256i& b) noexcept { return _mm256_min_epi64(a, b); } 2822 BL_INLINE_NODEBUG __m256i simd_max_i64(const __m256i& a, const __m256i& b) noexcept { return _mm256_max_epi64(a, b); } 2823 BL_INLINE_NODEBUG __m256i simd_min_u64(const __m256i& a, const __m256i& b) noexcept { return _mm256_min_epu64(a, b); } 2824 BL_INLINE_NODEBUG __m256i simd_max_u64(const __m256i& a, const __m256i& b) noexcept { return _mm256_max_epu64(a, b); } 2825 #else 2826 BL_INLINE_NODEBUG __m256i simd_min_i64(const __m256i& a, const __m256i& b) noexcept { return simd_blendv_u8(a, b, simd_cmp_gt_i64(a, b)); } 2827 BL_INLINE_NODEBUG __m256i simd_max_i64(const __m256i& a, const __m256i& b) noexcept { return simd_blendv_u8(b, a, simd_cmp_gt_i64(a, b)); } 2828 BL_INLINE_NODEBUG __m256i simd_min_u64(const __m256i& a, const __m256i& b) noexcept { return simd_blendv_u8(a, b, simd_cmp_gt_u64(a, b)); } 2829 BL_INLINE_NODEBUG __m256i simd_max_u64(const __m256i& a, const __m256i& b) noexcept { return simd_blendv_u8(b, a, simd_cmp_gt_u64(a, b)); } 2830 #endif 2831 2832 BL_INLINE_NODEBUG __m256i simd_abs_i8(const __m256i& a) noexcept { return _mm256_abs_epi8(a); } 2833 BL_INLINE_NODEBUG __m256i simd_abs_i16(const __m256i& a) noexcept { return _mm256_abs_epi16(a); } 2834 BL_INLINE_NODEBUG __m256i simd_abs_i32(const __m256i& a) noexcept { return _mm256_abs_epi32(a); } 2835 2836 #if defined(BL_TARGET_OPT_AVX512) 2837 BL_INLINE_NODEBUG __m256i simd_abs_i64(const __m256i& a) noexcept { return _mm256_abs_epi64(a); } 2838 #else 2839 BL_INLINE_NODEBUG __m256i simd_abs_i64(const __m256i& a) noexcept { 2840 __m256i neg_mask = _mm256_srai_epi32(_mm256_shuffle_epi32(a, _MM_SHUFFLE(3, 3, 1, 1)), 31); 2841 return _mm256_sub_epi64(simd_xor(a, neg_mask), neg_mask); 2842 } 2843 #endif 2844 2845 template<uint8_t kN> BL_INLINE_NODEBUG __m256i simd_slli_i8(const __m256i& a) noexcept { 2846 constexpr uint8_t kShift = uint8_t(kN & 0x7); 2847 2848 if constexpr (kShift == 0) { 2849 return a; 2850 } 2851 else if constexpr (kShift == 1) { 2852 return _mm256_add_epi8(a, a); 2853 } 2854 else { 2855 __m256i msk = _mm256_set1_epi8(int8_t((0xFFu << kShift) & 0xFFu)); 2856 return _mm256_and_si256(_mm256_slli_epi16(a, kShift), msk); 2857 } 2858 } 2859 2860 template<uint8_t kN> BL_INLINE_NODEBUG __m256i simd_srli_u8(const __m256i& a) noexcept { 2861 constexpr uint8_t kShift = uint8_t(kN & 0x7); 2862 2863 if constexpr (kShift == 0) { 2864 return a; 2865 } 2866 else { 2867 __m256i msk = _mm256_set1_epi8(int8_t((0xFFu >> kShift) & 0xFFu)); 2868 return _mm256_and_si256(_mm256_srli_epi16(a, kShift), msk); 2869 } 2870 } 2871 2872 template<uint8_t kN> BL_INLINE_NODEBUG __m256i simd_srai_i8(const __m256i& a) noexcept { 2873 constexpr uint8_t kShift = uint8_t(kN & 0x7); 2874 2875 if constexpr (kShift == 0) { 2876 return a; 2877 } 2878 else if constexpr (kShift == 7) { 2879 return _mm256_cmpgt_epi8(simd_make_zero<__m256i>(), a); 2880 } 2881 else { 2882 __m256i tmp = simd_srli_u8<kShift>(a); 2883 __m256i sgn = simd_make256_u8(0x80u >> kShift); 2884 return _mm256_sub_epi8(simd_xor(tmp, sgn), sgn); 2885 } 2886 } 2887 2888 template<uint8_t kN> BL_INLINE_NODEBUG __m256i simd_slli_i16(const __m256i& a) noexcept { return kN ? _mm256_slli_epi16(a, kN) : a; } 2889 template<uint8_t kN> BL_INLINE_NODEBUG __m256i simd_slli_i32(const __m256i& a) noexcept { return kN ? _mm256_slli_epi32(a, kN) : a; } 2890 template<uint8_t kN> BL_INLINE_NODEBUG __m256i simd_slli_i64(const __m256i& a) noexcept { return kN ? _mm256_slli_epi64(a, kN) : a; } 2891 2892 template<uint8_t kN> BL_INLINE_NODEBUG __m256i simd_srli_u16(const __m256i& a) noexcept { return kN ? _mm256_srli_epi16(a, kN) : a; } 2893 template<uint8_t kN> BL_INLINE_NODEBUG __m256i simd_srli_u32(const __m256i& a) noexcept { return kN ? _mm256_srli_epi32(a, kN) : a; } 2894 template<uint8_t kN> BL_INLINE_NODEBUG __m256i simd_srli_u64(const __m256i& a) noexcept { return kN ? _mm256_srli_epi64(a, kN) : a; } 2895 2896 template<uint8_t kN> BL_INLINE_NODEBUG __m256i simd_srai_i16(const __m256i& a) noexcept { return kN ? _mm256_srai_epi16(a, kN) : a; } 2897 template<uint8_t kN> BL_INLINE_NODEBUG __m256i simd_srai_i32(const __m256i& a) noexcept { return kN ? _mm256_srai_epi32(a, kN) : a; } 2898 2899 #if defined(BL_TARGET_OPT_AVX512) 2900 template<uint8_t kN> BL_INLINE_NODEBUG __m256i simd_srai_i64(const __m256i& a) noexcept { return kN ? _mm256_srai_epi64(a, kN) : a; } 2901 #else 2902 template<uint8_t kN> BL_INLINE_NODEBUG __m256i simd_srai_i64(const __m256i& a) noexcept { 2903 if constexpr (kN == 0) { 2904 return a; 2905 } 2906 else if constexpr (kN == 63u) { 2907 return _mm256_srai_epi32(_mm256_shuffle_epi32(a, _MM_SHUFFLE(3, 3, 1, 1)), 31); 2908 } 2909 else if constexpr (kN < 32u) { 2910 __m256i hi = _mm256_srai_epi32(a, kN & 31u); 2911 __m256i lo = _mm256_srli_epi64(a, kN & 31u); 2912 return _mm256_blend_epi16(lo, hi, 0xCCu); 2913 } 2914 else { 2915 __m256i highs = _mm256_shuffle_epi32(a, _MM_SHUFFLE(3, 3, 1, 1)); 2916 __m256i signs = _mm256_srai_epi32(highs, 31); 2917 __m256i msk = _mm256_slli_epi64(signs, (64 - kN) & 63); 2918 return _mm256_or_si256(msk, _mm256_srli_epi64(a, kN)); 2919 } 2920 } 2921 #endif 2922 2923 template<uint8_t kNumBytes> BL_INLINE_NODEBUG __m256i simd_sllb_u128(const __m256i& a) noexcept { return kNumBytes ? _mm256_slli_si256(a, kNumBytes) : a; } 2924 template<uint8_t kNumBytes> BL_INLINE_NODEBUG __m256i simd_srlb_u128(const __m256i& a) noexcept { return kNumBytes ? _mm256_srli_si256(a, kNumBytes) : a; } 2925 2926 BL_INLINE_NODEBUG __m256i simd_sad_u8_u64(const __m256i& a, const __m256i& b) noexcept { return _mm256_sad_epu8(a, b); } 2927 BL_INLINE_NODEBUG __m256i simd_maddws_u8xi8_i16(const __m256i& a, const __m256i& b) noexcept { return _mm256_maddubs_epi16(a, b); } 2928 2929 #endif // BL_TARGET_OPT_AVX2 2930 2931 #if defined(BL_TARGET_OPT_AVX512) 2932 template<uint8_t kPred> 2933 BL_INLINE_NODEBUG __m512i simd_ternlog(const __m512i& a, const __m512i& b, const __m512i& c) noexcept { return _mm512_ternarylogic_epi32(a, b, c, kPred); } 2934 2935 template<uint8_t kPred> 2936 BL_INLINE_NODEBUG __m512 simd_ternlog(const __m512& a, const __m512& b, const __m512& c) noexcept { return simd_as_f(simd_ternlog<kPred>(simd_as_i(a), simd_as_i(b), simd_as_i(c))); } 2937 2938 template<uint8_t kPred> 2939 BL_INLINE_NODEBUG __m512d simd_ternlog(const __m512d& a, const __m512d& b, const __m512d& c) noexcept { return simd_as_d(simd_ternlog<kPred>(simd_as_i(a), simd_as_i(b), simd_as_i(c))); } 2940 2941 BL_INLINE_NODEBUG __m512 simd_and(const __m512& a, const __m512& b) noexcept { return _mm512_and_ps(a, b); } 2942 BL_INLINE_NODEBUG __m512d simd_and(const __m512d& a, const __m512d& b) noexcept { return _mm512_and_pd(a, b); } 2943 BL_INLINE_NODEBUG __m512i simd_and(const __m512i& a, const __m512i& b) noexcept { return _mm512_and_si512(a, b); } 2944 2945 BL_INLINE_NODEBUG __m512 simd_andnot(const __m512& a, const __m512& b) noexcept { return _mm512_andnot_ps(a, b); } 2946 BL_INLINE_NODEBUG __m512d simd_andnot(const __m512d& a, const __m512d& b) noexcept { return _mm512_andnot_pd(a, b); } 2947 BL_INLINE_NODEBUG __m512i simd_andnot(const __m512i& a, const __m512i& b) noexcept { return _mm512_andnot_si512(a, b); } 2948 2949 BL_INLINE_NODEBUG __m512 simd_or(const __m512& a, const __m512& b) noexcept { return _mm512_or_ps(a, b); } 2950 BL_INLINE_NODEBUG __m512d simd_or(const __m512d& a, const __m512d& b) noexcept { return _mm512_or_pd(a, b); } 2951 BL_INLINE_NODEBUG __m512i simd_or(const __m512i& a, const __m512i& b) noexcept { return _mm512_or_si512(a, b); } 2952 2953 BL_INLINE_NODEBUG __m512 simd_xor(const __m512& a, const __m512& b) noexcept { return _mm512_xor_ps(a, b); } 2954 BL_INLINE_NODEBUG __m512d simd_xor(const __m512d& a, const __m512d& b) noexcept { return _mm512_xor_pd(a, b); } 2955 BL_INLINE_NODEBUG __m512i simd_xor(const __m512i& a, const __m512i& b) noexcept { return _mm512_xor_si512(a, b); } 2956 2957 BL_INLINE_NODEBUG __m512i simd_not(const __m512i& a) noexcept { return simd_ternlog<0x55>(a, a, a); } 2958 BL_INLINE_NODEBUG __m512 simd_not(const __m512& a) noexcept { return simd_ternlog<0x55>(a, a, a); } 2959 BL_INLINE_NODEBUG __m512d simd_not(const __m512d& a) noexcept { return simd_ternlog<0x55>(a, a, a); } 2960 2961 BL_INLINE_NODEBUG __m512 simd_blendv_bits(const __m512& a, const __m512& b, const __m512& msk) noexcept { return simd_ternlog<0xD8>(a, b, msk); } 2962 BL_INLINE_NODEBUG __m512d simd_blendv_bits(const __m512d& a, const __m512d& b, const __m512d& msk) noexcept { return simd_ternlog<0xD8>(a, b, msk); } 2963 BL_INLINE_NODEBUG __m512i simd_blendv_bits(const __m512i& a, const __m512i& b, const __m512i& msk) noexcept { return simd_ternlog<0xD8>(a, b, msk); } 2964 2965 BL_INLINE_NODEBUG __m512i simd_blendv_u8(const __m512i& a, const __m512i& b, const __m512i& msk) noexcept { return simd_blendv_bits(a, b, msk); } 2966 2967 BL_INLINE_NODEBUG __m512i simd_flip_sign_i8(const __m512i& a) noexcept { return simd_xor(a, simd_make512_u32(0x80808080u)); } 2968 BL_INLINE_NODEBUG __m512i simd_flip_sign_i16(const __m512i& a) noexcept { return simd_xor(a, simd_make512_u32(0x80008000u)); } 2969 BL_INLINE_NODEBUG __m512i simd_flip_sign_i32(const __m512i& a) noexcept { return simd_xor(a, simd_make512_u32(0x80000000u)); } 2970 BL_INLINE_NODEBUG __m512i simd_flip_sign_i64(const __m512i& a) noexcept { return simd_xor(a, simd_make512_u64(uint64_t(1) << 63)); } 2971 2972 BL_INLINE_NODEBUG __m512 simd_add_f32(const __m512& a, const __m512& b) noexcept { return _mm512_add_ps(a, b); } 2973 BL_INLINE_NODEBUG __m512d simd_add_f64(const __m512d& a, const __m512d& b) noexcept { return _mm512_add_pd(a, b); } 2974 2975 BL_INLINE_NODEBUG __m512 simd_sub_f32(const __m512& a, const __m512& b) noexcept { return _mm512_sub_ps(a, b); } 2976 BL_INLINE_NODEBUG __m512d simd_sub_f64(const __m512d& a, const __m512d& b) noexcept { return _mm512_sub_pd(a, b); } 2977 2978 BL_INLINE_NODEBUG __m512 simd_mul_f32(const __m512& a, const __m512& b) noexcept { return _mm512_mul_ps(a, b); } 2979 BL_INLINE_NODEBUG __m512d simd_mul_f64(const __m512d& a, const __m512d& b) noexcept { return _mm512_mul_pd(a, b); } 2980 2981 BL_INLINE_NODEBUG __m512 simd_div_f32(const __m512& a, const __m512& b) noexcept { return _mm512_div_ps(a, b); } 2982 BL_INLINE_NODEBUG __m512d simd_div_f64(const __m512d& a, const __m512d& b) noexcept { return _mm512_div_pd(a, b); } 2983 2984 BL_INLINE_NODEBUG __m512 simd_min_f32(const __m512& a, const __m512& b) noexcept { return _mm512_min_ps(a, b); } 2985 BL_INLINE_NODEBUG __m512d simd_min_f64(const __m512d& a, const __m512d& b) noexcept { return _mm512_min_pd(a, b); } 2986 2987 BL_INLINE_NODEBUG __m512 simd_max_f32(const __m512& a, const __m512& b) noexcept { return _mm512_max_ps(a, b); } 2988 BL_INLINE_NODEBUG __m512d simd_max_f64(const __m512d& a, const __m512d& b) noexcept { return _mm512_max_pd(a, b); } 2989 2990 BL_INLINE_NODEBUG __m512 simd_cmp_eq_f32(const __m512& a, const __m512& b) noexcept { return simd_512f_from_mask32(_mm512_cmp_ps_mask(a, b, _CMP_EQ_OQ)); } 2991 BL_INLINE_NODEBUG __m512d simd_cmp_eq_f64(const __m512d& a, const __m512d& b) noexcept { return simd_512d_from_mask64(_mm512_cmp_pd_mask(a, b, _CMP_EQ_OQ)); } 2992 2993 BL_INLINE_NODEBUG __m512 simd_cmp_ne_f32(const __m512& a, const __m512& b) noexcept { return simd_512f_from_mask32(_mm512_cmp_ps_mask(a, b, _CMP_NEQ_OQ)); } 2994 BL_INLINE_NODEBUG __m512d simd_cmp_ne_f64(const __m512d& a, const __m512d& b) noexcept { return simd_512d_from_mask64(_mm512_cmp_pd_mask(a, b, _CMP_NEQ_OQ)); } 2995 2996 BL_INLINE_NODEBUG __m512 simd_cmp_lt_f32(const __m512& a, const __m512& b) noexcept { return simd_512f_from_mask32(_mm512_cmp_ps_mask(a, b, _CMP_LT_OQ)); } 2997 BL_INLINE_NODEBUG __m512d simd_cmp_lt_f64(const __m512d& a, const __m512d& b) noexcept { return simd_512d_from_mask64(_mm512_cmp_pd_mask(a, b, _CMP_LT_OQ)); } 2998 2999 BL_INLINE_NODEBUG __m512 simd_cmp_le_f32(const __m512& a, const __m512& b) noexcept { return simd_512f_from_mask32(_mm512_cmp_ps_mask(a, b, _CMP_LE_OQ)); } 3000 BL_INLINE_NODEBUG __m512d simd_cmp_le_f64(const __m512d& a, const __m512d& b) noexcept { return simd_512d_from_mask64(_mm512_cmp_pd_mask(a, b, _CMP_LE_OQ)); } 3001 3002 BL_INLINE_NODEBUG __m512 simd_cmp_gt_f32(const __m512& a, const __m512& b) noexcept { return simd_512f_from_mask32(_mm512_cmp_ps_mask(a, b, _CMP_GT_OQ)); } 3003 BL_INLINE_NODEBUG __m512d simd_cmp_gt_f64(const __m512d& a, const __m512d& b) noexcept { return simd_512d_from_mask64(_mm512_cmp_pd_mask(a, b, _CMP_GT_OQ)); } 3004 3005 BL_INLINE_NODEBUG __m512 simd_cmp_ge_f32(const __m512& a, const __m512& b) noexcept { return simd_512f_from_mask32(_mm512_cmp_ps_mask(a, b, _CMP_GE_OQ)); } 3006 BL_INLINE_NODEBUG __m512d simd_cmp_ge_f64(const __m512d& a, const __m512d& b) noexcept { return simd_512d_from_mask64(_mm512_cmp_pd_mask(a, b, _CMP_GE_OQ)); } 3007 3008 BL_INLINE_NODEBUG __m512 simd_abs_f32(const __m512& a) noexcept { return _mm512_and_ps(a, _mm512_broadcastss_ps(_mm_load_ss(&bl::common_table.p_7FFFFFFF7FFFFFFF.as<float>()))); } 3009 BL_INLINE_NODEBUG __m512d simd_abs_f64(const __m512d& a) noexcept { return _mm512_and_pd(a, _mm512_broadcastsd_pd(_mm_load_sd(&bl::common_table.p_7FFFFFFFFFFFFFFF.as<double>()))); } 3010 3011 BL_INLINE_NODEBUG __m512 simd_sqrt_f32(const __m512& a) noexcept { return _mm512_sqrt_ps(a); } 3012 BL_INLINE_NODEBUG __m512d simd_sqrt_f64(const __m512d& a) noexcept { return _mm512_sqrt_pd(a); } 3013 3014 BL_INLINE_NODEBUG __m512i simd_add_i8(const __m512i& a, const __m512i& b) noexcept { return _mm512_add_epi8(a, b); } 3015 BL_INLINE_NODEBUG __m512i simd_add_i16(const __m512i& a, const __m512i& b) noexcept { return _mm512_add_epi16(a, b); } 3016 BL_INLINE_NODEBUG __m512i simd_add_i32(const __m512i& a, const __m512i& b) noexcept { return _mm512_add_epi32(a, b); } 3017 BL_INLINE_NODEBUG __m512i simd_add_i64(const __m512i& a, const __m512i& b) noexcept { return _mm512_add_epi64(a, b); } 3018 BL_INLINE_NODEBUG __m512i simd_add_u8(const __m512i& a, const __m512i& b) noexcept { return _mm512_add_epi8(a, b); } 3019 BL_INLINE_NODEBUG __m512i simd_add_u16(const __m512i& a, const __m512i& b) noexcept { return _mm512_add_epi16(a, b); } 3020 BL_INLINE_NODEBUG __m512i simd_add_u32(const __m512i& a, const __m512i& b) noexcept { return _mm512_add_epi32(a, b); } 3021 BL_INLINE_NODEBUG __m512i simd_add_u64(const __m512i& a, const __m512i& b) noexcept { return _mm512_add_epi64(a, b); } 3022 3023 BL_INLINE_NODEBUG __m512i simd_adds_i8(const __m512i& a, const __m512i& b) noexcept { return _mm512_adds_epi8(a, b); } 3024 BL_INLINE_NODEBUG __m512i simd_adds_i16(const __m512i& a, const __m512i& b) noexcept { return _mm512_adds_epi16(a, b); } 3025 BL_INLINE_NODEBUG __m512i simd_adds_u8(const __m512i& a, const __m512i& b) noexcept { return _mm512_adds_epu8(a, b); } 3026 BL_INLINE_NODEBUG __m512i simd_adds_u16(const __m512i& a, const __m512i& b) noexcept { return _mm512_adds_epu16(a, b); } 3027 3028 BL_INLINE_NODEBUG __m512i simd_sub_i8(const __m512i& a, const __m512i& b) noexcept { return _mm512_sub_epi8(a, b); } 3029 BL_INLINE_NODEBUG __m512i simd_sub_i16(const __m512i& a, const __m512i& b) noexcept { return _mm512_sub_epi16(a, b); } 3030 BL_INLINE_NODEBUG __m512i simd_sub_i32(const __m512i& a, const __m512i& b) noexcept { return _mm512_sub_epi32(a, b); } 3031 BL_INLINE_NODEBUG __m512i simd_sub_i64(const __m512i& a, const __m512i& b) noexcept { return _mm512_sub_epi64(a, b); } 3032 BL_INLINE_NODEBUG __m512i simd_sub_u8(const __m512i& a, const __m512i& b) noexcept { return _mm512_sub_epi8(a, b); } 3033 BL_INLINE_NODEBUG __m512i simd_sub_u16(const __m512i& a, const __m512i& b) noexcept { return _mm512_sub_epi16(a, b); } 3034 BL_INLINE_NODEBUG __m512i simd_sub_u32(const __m512i& a, const __m512i& b) noexcept { return _mm512_sub_epi32(a, b); } 3035 BL_INLINE_NODEBUG __m512i simd_sub_u64(const __m512i& a, const __m512i& b) noexcept { return _mm512_sub_epi64(a, b); } 3036 3037 BL_INLINE_NODEBUG __m512i simd_subs_i8(const __m512i& a, const __m512i& b) noexcept { return _mm512_subs_epi8(a, b); } 3038 BL_INLINE_NODEBUG __m512i simd_subs_i16(const __m512i& a, const __m512i& b) noexcept { return _mm512_subs_epi16(a, b); } 3039 BL_INLINE_NODEBUG __m512i simd_subs_u8(const __m512i& a, const __m512i& b) noexcept { return _mm512_subs_epu8(a, b); } 3040 BL_INLINE_NODEBUG __m512i simd_subs_u16(const __m512i& a, const __m512i& b) noexcept { return _mm512_subs_epu16(a, b); } 3041 3042 BL_INLINE_NODEBUG __m512i simd_mul_i16(const __m512i& a, const __m512i& b) noexcept { return _mm512_mullo_epi16(a, b); } 3043 BL_INLINE_NODEBUG __m512i simd_mul_u16(const __m512i& a, const __m512i& b) noexcept { return _mm512_mullo_epi16(a, b); } 3044 BL_INLINE_NODEBUG __m512i simd_mul_i32(const __m512i& a, const __m512i& b) noexcept { return _mm512_mullo_epi32(a, b); } 3045 BL_INLINE_NODEBUG __m512i simd_mul_u32(const __m512i& a, const __m512i& b) noexcept { return _mm512_mullo_epi32(a, b); } 3046 BL_INLINE_NODEBUG __m512i simd_mul_i64(const __m512i& a, const __m512i& b) noexcept { return _mm512_mullo_epi64(a, b); } 3047 BL_INLINE_NODEBUG __m512i simd_mul_u64(const __m512i& a, const __m512i& b) noexcept { return _mm512_mullo_epi64(a, b); } 3048 3049 BL_INLINE_NODEBUG __m512i simd_mulh_i16(const __m512i& a, const __m512i& b) noexcept { return _mm512_mulhi_epi16(a, b); } 3050 BL_INLINE_NODEBUG __m512i simd_mulh_u16(const __m512i& a, const __m512i& b) noexcept { return _mm512_mulhi_epu16(a, b); } 3051 BL_INLINE_NODEBUG __m512i simd_mulw_i32(const __m512i& a, const __m512i& b) noexcept { return _mm512_mul_epi32(a, b); } 3052 BL_INLINE_NODEBUG __m512i simd_mulw_u32(const __m512i& a, const __m512i& b) noexcept { return _mm512_mul_epu32(a, b); } 3053 3054 BL_INLINE_NODEBUG __m512i simd_maddw_i16_i32(const __m512i& a, const __m512i& b) noexcept { return _mm512_madd_epi16(a, b); } 3055 3056 BL_INLINE_NODEBUG __m512i simd_cmp_eq_i8(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask8(_mm512_cmpeq_epi8_mask(a, b)); } 3057 BL_INLINE_NODEBUG __m512i simd_cmp_eq_i16(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask16(_mm512_cmpeq_epi16_mask(a, b)); } 3058 BL_INLINE_NODEBUG __m512i simd_cmp_eq_i32(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask32(_mm512_cmpeq_epi32_mask(a, b)); } 3059 BL_INLINE_NODEBUG __m512i simd_cmp_eq_i64(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask64(_mm512_cmpeq_epi64_mask(a, b)); } 3060 3061 BL_INLINE_NODEBUG __m512i simd_cmp_ne_i8(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask8(_mm512_cmp_epi8_mask(a, b, _MM_CMPINT_NE)); } 3062 BL_INLINE_NODEBUG __m512i simd_cmp_ne_i16(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask16(_mm512_cmp_epi16_mask(a, b, _MM_CMPINT_NE)); } 3063 BL_INLINE_NODEBUG __m512i simd_cmp_ne_i32(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask32(_mm512_cmp_epi32_mask(a, b, _MM_CMPINT_NE)); } 3064 BL_INLINE_NODEBUG __m512i simd_cmp_ne_i64(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask64(_mm512_cmp_epi64_mask(a, b, _MM_CMPINT_NE)); } 3065 3066 BL_INLINE_NODEBUG __m512i simd_cmp_gt_i8(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask8(_mm512_cmpgt_epi8_mask(a, b)); } 3067 BL_INLINE_NODEBUG __m512i simd_cmp_gt_i16(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask16(_mm512_cmpgt_epi16_mask(a, b)); } 3068 BL_INLINE_NODEBUG __m512i simd_cmp_gt_i32(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask32(_mm512_cmpgt_epi32_mask(a, b)); } 3069 BL_INLINE_NODEBUG __m512i simd_cmp_gt_i64(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask64(_mm512_cmpgt_epi64_mask(a, b)); } 3070 3071 BL_INLINE_NODEBUG __m512i simd_cmp_lt_i8(const __m512i& a, const __m512i& b) noexcept { return simd_cmp_gt_i8(b, a); } 3072 BL_INLINE_NODEBUG __m512i simd_cmp_lt_i16(const __m512i& a, const __m512i& b) noexcept { return simd_cmp_gt_i16(b, a); } 3073 BL_INLINE_NODEBUG __m512i simd_cmp_lt_i32(const __m512i& a, const __m512i& b) noexcept { return simd_cmp_gt_i32(b, a); } 3074 BL_INLINE_NODEBUG __m512i simd_cmp_lt_i64(const __m512i& a, const __m512i& b) noexcept { return simd_cmp_gt_i64(b, a); } 3075 3076 BL_INLINE_NODEBUG __m512i simd_cmp_ge_i8(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask8(_mm512_cmp_epi8_mask(a, b, _MM_CMPINT_NLT)); } 3077 BL_INLINE_NODEBUG __m512i simd_cmp_ge_i16(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask16(_mm512_cmp_epi16_mask(a, b, _MM_CMPINT_NLT)); } 3078 BL_INLINE_NODEBUG __m512i simd_cmp_ge_i32(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask32(_mm512_cmp_epi32_mask(a, b, _MM_CMPINT_NLT)); } 3079 BL_INLINE_NODEBUG __m512i simd_cmp_ge_i64(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask64(_mm512_cmp_epi64_mask(a, b, _MM_CMPINT_NLT)); } 3080 3081 BL_INLINE_NODEBUG __m512i simd_cmp_le_i8(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask8(_mm512_cmp_epi8_mask(a, b, _MM_CMPINT_LE)); } 3082 BL_INLINE_NODEBUG __m512i simd_cmp_le_i16(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask16(_mm512_cmp_epi16_mask(a, b, _MM_CMPINT_LE)); } 3083 BL_INLINE_NODEBUG __m512i simd_cmp_le_i32(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask32(_mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LE)); } 3084 BL_INLINE_NODEBUG __m512i simd_cmp_le_i64(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask64(_mm512_cmp_epi64_mask(a, b, _MM_CMPINT_LE)); } 3085 3086 BL_INLINE_NODEBUG __m512i simd_cmp_gt_u8(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask8(_mm512_cmp_epu8_mask(a, b, _MM_CMPINT_NLE)); } 3087 BL_INLINE_NODEBUG __m512i simd_cmp_gt_u16(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask16(_mm512_cmp_epu16_mask(a, b, _MM_CMPINT_NLE)); } 3088 BL_INLINE_NODEBUG __m512i simd_cmp_gt_u32(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask32(_mm512_cmp_epu32_mask(a, b, _MM_CMPINT_NLE)); } 3089 BL_INLINE_NODEBUG __m512i simd_cmp_gt_u64(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask64(_mm512_cmp_epu64_mask(a, b, _MM_CMPINT_NLE)); } 3090 3091 BL_INLINE_NODEBUG __m512i simd_cmp_ge_u8(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask8(_mm512_cmp_epu8_mask(a, b, _MM_CMPINT_NLT)); } 3092 BL_INLINE_NODEBUG __m512i simd_cmp_ge_u16(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask16(_mm512_cmp_epu16_mask(a, b, _MM_CMPINT_NLT)); } 3093 BL_INLINE_NODEBUG __m512i simd_cmp_ge_u32(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask32(_mm512_cmp_epu32_mask(a, b, _MM_CMPINT_NLT)); } 3094 BL_INLINE_NODEBUG __m512i simd_cmp_ge_u64(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask64(_mm512_cmp_epu64_mask(a, b, _MM_CMPINT_NLT)); } 3095 3096 BL_INLINE_NODEBUG __m512i simd_cmp_lt_u8(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask8(_mm512_cmp_epu8_mask(a, b, _MM_CMPINT_LT)); } 3097 BL_INLINE_NODEBUG __m512i simd_cmp_lt_u16(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask16(_mm512_cmp_epu16_mask(a, b, _MM_CMPINT_LT)); } 3098 BL_INLINE_NODEBUG __m512i simd_cmp_lt_u32(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask32(_mm512_cmp_epu32_mask(a, b, _MM_CMPINT_LT)); } 3099 BL_INLINE_NODEBUG __m512i simd_cmp_lt_u64(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask64(_mm512_cmp_epu64_mask(a, b, _MM_CMPINT_LT)); } 3100 3101 BL_INLINE_NODEBUG __m512i simd_cmp_le_u8(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask8(_mm512_cmp_epu8_mask(a, b, _MM_CMPINT_LE)); } 3102 BL_INLINE_NODEBUG __m512i simd_cmp_le_u16(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask16(_mm512_cmp_epu16_mask(a, b, _MM_CMPINT_LE)); } 3103 BL_INLINE_NODEBUG __m512i simd_cmp_le_u32(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask32(_mm512_cmp_epu32_mask(a, b, _MM_CMPINT_LE)); } 3104 BL_INLINE_NODEBUG __m512i simd_cmp_le_u64(const __m512i& a, const __m512i& b) noexcept { return simd_512i_from_mask64(_mm512_cmp_epu64_mask(a, b, _MM_CMPINT_LE)); } 3105 3106 BL_INLINE_NODEBUG __m512i simd_min_i8(const __m512i& a, const __m512i& b) noexcept { return _mm512_min_epi8(a, b); } 3107 BL_INLINE_NODEBUG __m512i simd_min_i16(const __m512i& a, const __m512i& b) noexcept { return _mm512_min_epi16(a, b); } 3108 BL_INLINE_NODEBUG __m512i simd_min_i32(const __m512i& a, const __m512i& b) noexcept { return _mm512_min_epi32(a, b); } 3109 BL_INLINE_NODEBUG __m512i simd_min_i64(const __m512i& a, const __m512i& b) noexcept { return _mm512_min_epi64(a, b); } 3110 BL_INLINE_NODEBUG __m512i simd_min_u8(const __m512i& a, const __m512i& b) noexcept { return _mm512_min_epu8(a, b); } 3111 BL_INLINE_NODEBUG __m512i simd_min_u16(const __m512i& a, const __m512i& b) noexcept { return _mm512_min_epu16(a, b); } 3112 BL_INLINE_NODEBUG __m512i simd_min_u32(const __m512i& a, const __m512i& b) noexcept { return _mm512_min_epu32(a, b); } 3113 BL_INLINE_NODEBUG __m512i simd_min_u64(const __m512i& a, const __m512i& b) noexcept { return _mm512_min_epu64(a, b); } 3114 3115 BL_INLINE_NODEBUG __m512i simd_max_i8(const __m512i& a, const __m512i& b) noexcept { return _mm512_max_epi8(a, b); } 3116 BL_INLINE_NODEBUG __m512i simd_max_i16(const __m512i& a, const __m512i& b) noexcept { return _mm512_max_epi16(a, b); } 3117 BL_INLINE_NODEBUG __m512i simd_max_i32(const __m512i& a, const __m512i& b) noexcept { return _mm512_max_epi32(a, b); } 3118 BL_INLINE_NODEBUG __m512i simd_max_i64(const __m512i& a, const __m512i& b) noexcept { return _mm512_max_epi64(a, b); } 3119 BL_INLINE_NODEBUG __m512i simd_max_u8(const __m512i& a, const __m512i& b) noexcept { return _mm512_max_epu8(a, b); } 3120 BL_INLINE_NODEBUG __m512i simd_max_u16(const __m512i& a, const __m512i& b) noexcept { return _mm512_max_epu16(a, b); } 3121 BL_INLINE_NODEBUG __m512i simd_max_u32(const __m512i& a, const __m512i& b) noexcept { return _mm512_max_epu32(a, b); } 3122 BL_INLINE_NODEBUG __m512i simd_max_u64(const __m512i& a, const __m512i& b) noexcept { return _mm512_max_epu64(a, b); } 3123 3124 BL_INLINE_NODEBUG __m512i simd_abs_i8(const __m512i& a) noexcept { return _mm512_abs_epi8(a); } 3125 BL_INLINE_NODEBUG __m512i simd_abs_i16(const __m512i& a) noexcept { return _mm512_abs_epi16(a); } 3126 BL_INLINE_NODEBUG __m512i simd_abs_i32(const __m512i& a) noexcept { return _mm512_abs_epi32(a); } 3127 BL_INLINE_NODEBUG __m512i simd_abs_i64(const __m512i& a) noexcept { return _mm512_abs_epi64(a); } 3128 3129 template<uint8_t kN> BL_INLINE_NODEBUG __m512i simd_slli_i8(const __m512i& a) noexcept { 3130 constexpr uint8_t kShift = uint8_t(kN & 0x7); 3131 3132 if constexpr (kShift == 0u) { 3133 return a; 3134 } 3135 else if constexpr (kShift == 1) { 3136 return _mm512_add_epi8(a, a); 3137 } 3138 else { 3139 __m512i msk = _mm512_set1_epi8(int8_t((0xFFu << kShift) & 0xFFu)); 3140 return _mm512_and_si512(_mm512_slli_epi16(a, kShift), msk); 3141 } 3142 } 3143 3144 template<uint8_t kN> BL_INLINE_NODEBUG __m512i simd_srli_u8(const __m512i& a) noexcept { 3145 constexpr uint8_t kShift = uint8_t(kN & 0x7); 3146 3147 if constexpr (kShift == 0) { 3148 return a; 3149 } 3150 else { 3151 __m512i msk = _mm512_set1_epi8(int8_t((0xFFu >> kShift) & 0xFFu)); 3152 return _mm512_and_si512(_mm512_srli_epi16(a, kShift), msk); 3153 } 3154 } 3155 3156 template<uint8_t kN> BL_INLINE_NODEBUG __m512i simd_srai_i8(const __m512i& a) noexcept { 3157 constexpr uint8_t kShift = uint8_t(kN & 0x7); 3158 3159 if constexpr (kShift == 0) { 3160 return a; 3161 } 3162 else { 3163 __m512i tmp = simd_srli_u8<kShift>(a); 3164 __m512i sgn = simd_make512_u8(0x80u >> kShift); 3165 return _mm512_sub_epi8(simd_xor(tmp, sgn), sgn); 3166 } 3167 } 3168 3169 template<uint8_t kN> BL_INLINE_NODEBUG __m512i simd_slli_i16(const __m512i& a) noexcept { return kN ? _mm512_slli_epi16(a, kN) : a; } 3170 template<uint8_t kN> BL_INLINE_NODEBUG __m512i simd_slli_i32(const __m512i& a) noexcept { return kN ? _mm512_slli_epi32(a, kN) : a; } 3171 template<uint8_t kN> BL_INLINE_NODEBUG __m512i simd_slli_i64(const __m512i& a) noexcept { return kN ? _mm512_slli_epi64(a, kN) : a; } 3172 3173 template<uint8_t kN> BL_INLINE_NODEBUG __m512i simd_srli_u16(const __m512i& a) noexcept { return kN ? _mm512_srli_epi16(a, kN) : a; } 3174 template<uint8_t kN> BL_INLINE_NODEBUG __m512i simd_srli_u32(const __m512i& a) noexcept { return kN ? _mm512_srli_epi32(a, kN) : a; } 3175 template<uint8_t kN> BL_INLINE_NODEBUG __m512i simd_srli_u64(const __m512i& a) noexcept { return kN ? _mm512_srli_epi64(a, kN) : a; } 3176 3177 template<uint8_t kN> BL_INLINE_NODEBUG __m512i simd_srai_i16(const __m512i& a) noexcept { return kN ? _mm512_srai_epi16(a, kN) : a; } 3178 template<uint8_t kN> BL_INLINE_NODEBUG __m512i simd_srai_i32(const __m512i& a) noexcept { return kN ? _mm512_srai_epi32(a, kN) : a; } 3179 template<uint8_t kN> BL_INLINE_NODEBUG __m512i simd_srai_i64(const __m512i& a) noexcept { return kN ? _mm512_srai_epi64(a, kN) : a; } 3180 3181 template<uint8_t kNumBytes> BL_INLINE_NODEBUG __m512i simd_sllb_u128(const __m512i& a) noexcept { return kNumBytes ? _mm512_bslli_epi128(a, kNumBytes) : a; } 3182 template<uint8_t kNumBytes> BL_INLINE_NODEBUG __m512i simd_srlb_u128(const __m512i& a) noexcept { return kNumBytes ? _mm512_bsrli_epi128(a, kNumBytes) : a; } 3183 3184 BL_INLINE_NODEBUG __m512i simd_sad_u8_u64(const __m512i& a, const __m512i& b) noexcept { return _mm512_sad_epu8(a, b); } 3185 BL_INLINE_NODEBUG __m512i simd_maddws_u8xi8_i16(const __m512i& a, const __m512i& b) noexcept { return _mm512_maddubs_epi16(a, b); } 3186 3187 #endif // BL_TARGET_OPT_AVX512 3188 3189 } // {Internal} 3190 3191 // SIMD - Internal - Extract MSB 3192 // ============================= 3193 3194 namespace Internal { 3195 3196 BL_INLINE_NODEBUG uint32_t simd_extract_sign_bits_i8(const __m128i& a) noexcept { return uint32_t(_mm_movemask_epi8(a)); } 3197 BL_INLINE_NODEBUG uint32_t simd_extract_sign_bits_i32(const __m128i& a) noexcept { return uint32_t(_mm_movemask_ps(simd_cast<__m128>(a))); } 3198 BL_INLINE_NODEBUG uint32_t simd_extract_sign_bits_i64(const __m128i& a) noexcept { return uint32_t(_mm_movemask_pd(simd_cast<__m128d>(a))); } 3199 3200 #if defined(BL_TARGET_OPT_AVX2) 3201 BL_INLINE_NODEBUG uint32_t simd_extract_sign_bits_i8(const __m256i& a) noexcept { return uint32_t(_mm256_movemask_epi8(a)); } 3202 BL_INLINE_NODEBUG uint32_t simd_extract_sign_bits_i32(const __m256i& a) noexcept { return uint32_t(_mm256_movemask_ps(simd_cast<__m256>(a))); } 3203 BL_INLINE_NODEBUG uint32_t simd_extract_sign_bits_i64(const __m256i& a) noexcept { return uint32_t(_mm256_movemask_pd(simd_cast<__m256d>(a))); } 3204 #endif // BL_TARGET_OPT_AVX2 3205 3206 #if defined(BL_TARGET_OPT_AVX512) 3207 BL_INLINE_NODEBUG uint64_t simd_extract_sign_bits_i8(const __m512i& a) noexcept { return uint64_t(_cvtmask64_u64(_mm512_movepi8_mask(a))); } 3208 BL_INLINE_NODEBUG uint32_t simd_extract_sign_bits_i32(const __m512i& a) noexcept { return uint32_t(_cvtmask16_u32(_mm512_movepi32_mask(a))); } 3209 BL_INLINE_NODEBUG uint32_t simd_extract_sign_bits_i64(const __m512i& a) noexcept { return uint32_t(_cvtmask8_u32(_mm512_movepi64_mask(a))); } 3210 #endif // BL_TARGET_OPT_AVX512 3211 3212 } // {Internal} 3213 3214 // SIMD - Internal - Load & Store Operations 3215 // ========================================= 3216 3217 namespace Internal { 3218 3219 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_load_8(const void* src) noexcept; 3220 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loada_16(const void* src) noexcept; 3221 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loadu_16(const void* src) noexcept; 3222 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loada_32(const void* src) noexcept; 3223 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loadu_32(const void* src) noexcept; 3224 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loada_64(const void* src) noexcept; 3225 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loadu_64(const void* src) noexcept; 3226 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loadl_64(SimdT dst, const void* src) noexcept; 3227 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loadh_64(SimdT dst, const void* src) noexcept; 3228 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loada_128(const void* src) noexcept; 3229 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loadu_128(const void* src) noexcept; 3230 3231 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loada(const void* src) noexcept; 3232 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loadu(const void* src) noexcept; 3233 3234 template<size_t kW> BL_INLINE_NODEBUG typename SimdInfo<kW>::RegI simd_load_broadcast_u32(const void* src) noexcept; 3235 template<size_t kW> BL_INLINE_NODEBUG typename SimdInfo<kW>::RegI simd_load_broadcast_u64(const void* src) noexcept; 3236 template<size_t kW> BL_INLINE_NODEBUG typename SimdInfo<kW>::RegF simd_load_broadcast_f32(const void* src) noexcept; 3237 template<size_t kW> BL_INLINE_NODEBUG typename SimdInfo<kW>::RegD simd_load_broadcast_f64(const void* src) noexcept; 3238 3239 template<size_t kW> BL_INLINE_NODEBUG typename SimdInfo<kW>::RegI simd_load_broadcast_4xi32(const void* src) noexcept; 3240 template<size_t kW> BL_INLINE_NODEBUG typename SimdInfo<kW>::RegI simd_load_broadcast_2xi64(const void* src) noexcept; 3241 template<size_t kW> BL_INLINE_NODEBUG typename SimdInfo<kW>::RegF simd_load_broadcast_f32x4(const void* src) noexcept; 3242 template<size_t kW> BL_INLINE_NODEBUG typename SimdInfo<kW>::RegD simd_load_broadcast_f64x2(const void* src) noexcept; 3243 3244 template<> BL_INLINE_NODEBUG __m128i simd_loada(const void* src) noexcept { return _mm_load_si128(static_cast<const __m128i*>(src)); } 3245 template<> BL_INLINE_NODEBUG __m128i simd_loadu(const void* src) noexcept { return _mm_loadu_si128(static_cast<const __m128i*>(src)); } 3246 template<> BL_INLINE_NODEBUG __m128 simd_loada(const void* src) noexcept { return _mm_load_ps(static_cast<const float*>(src)); } 3247 template<> BL_INLINE_NODEBUG __m128 simd_loadu(const void* src) noexcept { return _mm_loadu_ps(static_cast<const float*>(src)); } 3248 template<> BL_INLINE_NODEBUG __m128d simd_loada(const void* src) noexcept { return _mm_load_pd(static_cast<const double*>(src)); } 3249 template<> BL_INLINE_NODEBUG __m128d simd_loadu(const void* src) noexcept { return _mm_loadu_pd(static_cast<const double*>(src)); } 3250 3251 #if defined(BL_TARGET_OPT_SSE4_1) 3252 template<> BL_INLINE_NODEBUG __m128i simd_load_8(const void* src) noexcept { return _mm_insert_epi8(_mm_setzero_si128(), *static_cast<const uint8_t*>(src), 0); } 3253 #else 3254 template<> BL_INLINE_NODEBUG __m128i simd_load_8(const void* src) noexcept { return _mm_cvtsi32_si128(int(unsigned(*static_cast<const uint8_t*>(src)))); } 3255 #endif 3256 3257 template<> BL_INLINE_NODEBUG __m128i simd_loada_16(const void* src) noexcept { return _mm_cvtsi32_si128(*static_cast<const uint16_t*>(src)); } 3258 template<> BL_INLINE_NODEBUG __m128i simd_loadu_16(const void* src) noexcept { return _mm_cvtsi32_si128(int(bl::MemOps::readU16u(src))); } 3259 template<> BL_INLINE_NODEBUG __m128i simd_loada_32(const void* src) noexcept { return _mm_cvtsi32_si128(*static_cast<const int*>(src)); } 3260 template<> BL_INLINE_NODEBUG __m128i simd_loadu_32(const void* src) noexcept { return _mm_cvtsi32_si128(int(bl::MemOps::readU32u(src))); } 3261 template<> BL_INLINE_NODEBUG __m128i simd_loada_64(const void* src) noexcept { return _mm_loadl_epi64(static_cast<const __m128i*>(src)); } 3262 template<> BL_INLINE_NODEBUG __m128i simd_loadu_64(const void* src) noexcept { return _mm_loadl_epi64(static_cast<const __m128i*>(src)); } 3263 template<> BL_INLINE_NODEBUG __m128i simd_loada_128(const void* src) noexcept { return _mm_load_si128(static_cast<const __m128i*>(src)); } 3264 template<> BL_INLINE_NODEBUG __m128i simd_loadu_128(const void* src) noexcept { return _mm_loadu_si128(static_cast<const __m128i*>(src)); } 3265 3266 template<> BL_INLINE_NODEBUG __m128 simd_loada_32(const void* src) noexcept { return _mm_load_ss(static_cast<const float*>(src)); } 3267 template<> BL_INLINE_NODEBUG __m128 simd_loadu_32(const void* src) noexcept { return _mm_castsi128_ps(simd_loadu_32<__m128i>(src)); } 3268 template<> BL_INLINE_NODEBUG __m128 simd_loada_64(const void* src) noexcept { return _mm_castsi128_ps(simd_loada_64<__m128i>(src)); } 3269 template<> BL_INLINE_NODEBUG __m128 simd_loadu_64(const void* src) noexcept { return _mm_castsi128_ps(simd_loadu_64<__m128i>(src)); } 3270 template<> BL_INLINE_NODEBUG __m128 simd_loada_128(const void* src) noexcept { return _mm_load_ps(static_cast<const float*>(src)); } 3271 template<> BL_INLINE_NODEBUG __m128 simd_loadu_128(const void* src) noexcept { return _mm_loadu_ps(static_cast<const float*>(src)); } 3272 3273 template<> BL_INLINE_NODEBUG __m128d simd_loada_64(const void* src) noexcept { return _mm_load_sd(static_cast<const double*>(src)); } 3274 template<> BL_INLINE_NODEBUG __m128d simd_loadu_64(const void* src) noexcept { return _mm_castsi128_pd(simd_loadu_64<__m128i>(src)); } 3275 template<> BL_INLINE_NODEBUG __m128d simd_loada_128(const void* src) noexcept { return _mm_load_pd(static_cast<const double*>(src)); } 3276 template<> BL_INLINE_NODEBUG __m128d simd_loadu_128(const void* src) noexcept { return _mm_loadu_pd(static_cast<const double*>(src)); } 3277 3278 template<> BL_INLINE_NODEBUG __m128i simd_loadl_64(__m128i dst, const void* src) noexcept { return _mm_castpd_si128(_mm_loadl_pd(_mm_castsi128_pd(dst), static_cast<const double*>(src))); } 3279 template<> BL_INLINE_NODEBUG __m128i simd_loadh_64(__m128i dst, const void* src) noexcept { return _mm_castpd_si128(_mm_loadh_pd(_mm_castsi128_pd(dst), static_cast<const double*>(src))); } 3280 template<> BL_INLINE_NODEBUG __m128 simd_loadl_64(__m128 dst, const void* src) noexcept { return _mm_loadl_pi(dst, static_cast<const __m64*>(src)); } 3281 template<> BL_INLINE_NODEBUG __m128 simd_loadh_64(__m128 dst, const void* src) noexcept { return _mm_loadh_pi(dst, static_cast<const __m64*>(src)); } 3282 template<> BL_INLINE_NODEBUG __m128d simd_loadl_64(__m128d dst, const void* src) noexcept { return _mm_loadl_pd(dst, static_cast<const double*>(src)); } 3283 template<> BL_INLINE_NODEBUG __m128d simd_loadh_64(__m128d dst, const void* src) noexcept { return _mm_loadh_pd(dst, static_cast<const double*>(src)); } 3284 3285 // MSCV won't emit a single instruction if load_XX() is used to load from memory first. 3286 #if defined(BL_TARGET_OPT_SSE4_1) && defined(_MSC_VER) && !defined(__clang__) 3287 BL_INLINE_NODEBUG __m128i simd_loadu_64_i8_i16(const void* src) noexcept { return _mm_cvtepi8_epi16(*static_cast<const unaligned_m128i*>(src)); } 3288 BL_INLINE_NODEBUG __m128i simd_loadu_64_u8_u16(const void* src) noexcept { return _mm_cvtepu8_epi16(*static_cast<const unaligned_m128i*>(src)); } 3289 BL_INLINE_NODEBUG __m128i simd_loadu_32_i8_i32(const void* src) noexcept { return _mm_cvtepi8_epi32(*static_cast<const unaligned_m128i*>(src)); } 3290 BL_INLINE_NODEBUG __m128i simd_loadu_32_u8_u32(const void* src) noexcept { return _mm_cvtepu8_epi32(*static_cast<const unaligned_m128i*>(src)); } 3291 BL_INLINE_NODEBUG __m128i simd_loadu_16_i8_i64(const void* src) noexcept { return _mm_cvtepi8_epi64(*static_cast<const unaligned_m128i*>(src)); } 3292 BL_INLINE_NODEBUG __m128i simd_loadu_16_u8_u64(const void* src) noexcept { return _mm_cvtepu8_epi64(*static_cast<const unaligned_m128i*>(src)); } 3293 #elif defined(BL_TARGET_OPT_SSE4_1) 3294 BL_INLINE_NODEBUG __m128i simd_loadu_64_i8_i16(const void* src) noexcept { return _mm_cvtepi8_epi16(simd_loadu_64<__m128i>(src)); } 3295 BL_INLINE_NODEBUG __m128i simd_loadu_64_u8_u16(const void* src) noexcept { return _mm_cvtepu8_epi16(simd_loadu_64<__m128i>(src)); } 3296 BL_INLINE_NODEBUG __m128i simd_loadu_32_i8_i32(const void* src) noexcept { return _mm_cvtepi8_epi32(simd_loadu_32<__m128i>(src)); } 3297 BL_INLINE_NODEBUG __m128i simd_loadu_32_u8_u32(const void* src) noexcept { return _mm_cvtepu8_epi32(simd_loadu_32<__m128i>(src)); } 3298 BL_INLINE_NODEBUG __m128i simd_loadu_16_i8_i64(const void* src) noexcept { return _mm_cvtepi8_epi64(simd_loadu_16<__m128i>(src)); } 3299 BL_INLINE_NODEBUG __m128i simd_loadu_16_u8_u64(const void* src) noexcept { return _mm_cvtepu8_epi64(simd_loadu_16<__m128i>(src)); } 3300 #else 3301 BL_INLINE_NODEBUG __m128i simd_loadu_64_i8_i16(const void* src) noexcept { return simd_unpack_lo64_i8_i16(simd_loadu_64<__m128i>(src)); } 3302 BL_INLINE_NODEBUG __m128i simd_loadu_64_u8_u16(const void* src) noexcept { return simd_unpack_lo64_u8_u16(simd_loadu_64<__m128i>(src)); } 3303 BL_INLINE_NODEBUG __m128i simd_loadu_32_i8_i32(const void* src) noexcept { return simd_unpack_lo32_i8_i32(simd_loadu_32<__m128i>(src)); } 3304 BL_INLINE_NODEBUG __m128i simd_loadu_32_u8_u32(const void* src) noexcept { return simd_unpack_lo32_u8_u32(simd_loadu_32<__m128i>(src)); } 3305 #endif 3306 3307 BL_INLINE_NODEBUG __m128i simd_loada_64_i8_i16(const void* src) noexcept { return simd_loadu_64_i8_i16(src); } 3308 BL_INLINE_NODEBUG __m128i simd_loada_64_u8_u16(const void* src) noexcept { return simd_loadu_64_u8_u16(src); } 3309 BL_INLINE_NODEBUG __m128i simd_loada_32_i8_i32(const void* src) noexcept { return simd_loadu_32_i8_i32(src); } 3310 BL_INLINE_NODEBUG __m128i simd_loada_32_u8_u32(const void* src) noexcept { return simd_loadu_32_u8_u32(src); } 3311 3312 #if defined(BL_TARGET_OPT_SSE4_1) 3313 BL_INLINE_NODEBUG __m128i simd_loada_16_i8_i64(const void* src) noexcept { return simd_loadu_16_i8_i64(src); } 3314 BL_INLINE_NODEBUG __m128i simd_loada_16_u8_u64(const void* src) noexcept { return simd_loadu_16_u8_u64(src); } 3315 #endif 3316 3317 BL_INLINE_NODEBUG void simd_store_8(void* dst, __m128i src) noexcept { *static_cast<uint8_t*>(dst) = uint8_t(_mm_cvtsi128_si32(src)); } 3318 BL_INLINE_NODEBUG void simd_storea_16(void* dst, __m128i src) noexcept { *static_cast<uint16_t*>(dst) = uint16_t(_mm_cvtsi128_si32(src)); } 3319 BL_INLINE_NODEBUG void simd_storeu_16(void* dst, __m128i src) noexcept { bl::MemOps::writeU16u(dst, uint16_t(_mm_cvtsi128_si32(src))); } 3320 BL_INLINE_NODEBUG void simd_storea_32(void* dst, __m128i src) noexcept { *static_cast<uint32_t*>(dst) = uint32_t(_mm_cvtsi128_si32(src)); } 3321 BL_INLINE_NODEBUG void simd_storeu_32(void* dst, __m128i src) noexcept { bl::MemOps::writeU32u(dst, uint32_t(_mm_cvtsi128_si32(src))); } 3322 BL_INLINE_NODEBUG void simd_storea_64(void* dst, __m128i src) noexcept { _mm_storel_epi64(static_cast<__m128i*>(dst), src); } 3323 BL_INLINE_NODEBUG void simd_storeu_64(void* dst, __m128i src) noexcept { _mm_storel_epi64(static_cast<__m128i*>(dst), src); } 3324 BL_INLINE_NODEBUG void simd_storeh_64(void* dst, __m128i src) noexcept { _mm_storeh_pd(static_cast<double*>(dst), _mm_castsi128_pd(src)); } 3325 BL_INLINE_NODEBUG void simd_storea_128(void* dst, __m128i src) noexcept { _mm_store_si128(static_cast<__m128i*>(dst), src); } 3326 BL_INLINE_NODEBUG void simd_storeu_128(void* dst, __m128i src) noexcept { _mm_storeu_si128(static_cast<__m128i*>(dst), src); } 3327 3328 BL_INLINE_NODEBUG void simd_storea_32(void* dst, __m128 src) noexcept { _mm_store_ss(static_cast<float*>(dst), src); } 3329 BL_INLINE_NODEBUG void simd_storeu_32(void* dst, __m128 src) noexcept { _mm_store_ss(static_cast<float*>(dst), src); } 3330 BL_INLINE_NODEBUG void simd_storea_64(void* dst, __m128 src) noexcept { _mm_storel_pi(static_cast<__m64*>(dst), src); } 3331 BL_INLINE_NODEBUG void simd_storeu_64(void* dst, __m128 src) noexcept { _mm_storel_pi(static_cast<__m64*>(dst), src); } 3332 BL_INLINE_NODEBUG void simd_storeh_64(void* dst, __m128 src) noexcept { _mm_storeh_pi(static_cast<__m64*>(dst), src); } 3333 BL_INLINE_NODEBUG void simd_storea_128(void* dst, __m128 src) noexcept { _mm_store_ps(static_cast<float*>(dst), src); } 3334 BL_INLINE_NODEBUG void simd_storeu_128(void* dst, __m128 src) noexcept { _mm_storeu_ps(static_cast<float*>(dst), src); } 3335 3336 BL_INLINE_NODEBUG void simd_storea_64(void* dst, __m128d src) noexcept { _mm_store_sd(static_cast<double*>(dst), src); } 3337 BL_INLINE_NODEBUG void simd_storeu_64(void* dst, __m128d src) noexcept { _mm_store_sd(static_cast<double*>(dst), src); } 3338 BL_INLINE_NODEBUG void simd_storeh_64(void* dst, __m128d src) noexcept { _mm_storeh_pd(static_cast<double*>(dst), src); } 3339 BL_INLINE_NODEBUG void simd_storea_128(void* dst, __m128d src) noexcept { _mm_store_pd(static_cast<double*>(dst), src); } 3340 BL_INLINE_NODEBUG void simd_storeu_128(void* dst, __m128d src) noexcept { _mm_storeu_pd(static_cast<double*>(dst), src); } 3341 3342 BL_INLINE_NODEBUG void simd_storea(void* dst, __m128i src) noexcept { _mm_store_si128(static_cast<__m128i*>(dst), src); } 3343 BL_INLINE_NODEBUG void simd_storeu(void* dst, __m128i src) noexcept { _mm_storeu_si128(static_cast<__m128i*>(dst), src); } 3344 BL_INLINE_NODEBUG void simd_storea(void* dst, __m128 src) noexcept { _mm_store_ps(static_cast<float*>(dst), src); } 3345 BL_INLINE_NODEBUG void simd_storeu(void* dst, __m128 src) noexcept { _mm_storeu_ps(static_cast<float*>(dst), src); } 3346 BL_INLINE_NODEBUG void simd_storea(void* dst, __m128d src) noexcept { _mm_store_pd(static_cast<double*>(dst), src); } 3347 BL_INLINE_NODEBUG void simd_storeu(void* dst, __m128d src) noexcept { _mm_storeu_pd(static_cast<double*>(dst), src); } 3348 3349 #if defined(BL_TARGET_OPT_AVX) 3350 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loada_256(const void* src) noexcept; 3351 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loadu_256(const void* src) noexcept; 3352 3353 template<> BL_INLINE_NODEBUG __m256i simd_loada(const void* src) noexcept { return _mm256_load_si256(static_cast<const __m256i*>(src)); } 3354 template<> BL_INLINE_NODEBUG __m256i simd_loadu(const void* src) noexcept { return _mm256_loadu_si256(static_cast<const __m256i*>(src)); } 3355 template<> BL_INLINE_NODEBUG __m256 simd_loada(const void* src) noexcept { return _mm256_load_ps(static_cast<const float*>(src)); } 3356 template<> BL_INLINE_NODEBUG __m256 simd_loadu(const void* src) noexcept { return _mm256_loadu_ps(static_cast<const float*>(src)); } 3357 template<> BL_INLINE_NODEBUG __m256d simd_loada(const void* src) noexcept { return _mm256_load_pd(static_cast<const double*>(src)); } 3358 template<> BL_INLINE_NODEBUG __m256d simd_loadu(const void* src) noexcept { return _mm256_loadu_pd(static_cast<const double*>(src)); } 3359 3360 template<> BL_INLINE_NODEBUG __m256i simd_load_8(const void* src) noexcept { return _mm256_castsi128_si256(simd_load_8<__m128i>(src)); } 3361 template<> BL_INLINE_NODEBUG __m256i simd_loada_16(const void* src) noexcept { return _mm256_castsi128_si256(simd_loada_16<__m128i>(src)); } 3362 template<> BL_INLINE_NODEBUG __m256i simd_loadu_16(const void* src) noexcept { return _mm256_castsi128_si256(simd_loadu_16<__m128i>(src)); } 3363 template<> BL_INLINE_NODEBUG __m256i simd_loada_32(const void* src) noexcept { return _mm256_castsi128_si256(simd_loada_32<__m128i>(src)); } 3364 template<> BL_INLINE_NODEBUG __m256i simd_loadu_32(const void* src) noexcept { return _mm256_castsi128_si256(simd_loadu_32<__m128i>(src)); } 3365 template<> BL_INLINE_NODEBUG __m256i simd_loada_64(const void* src) noexcept { return _mm256_castsi128_si256(simd_loada_64<__m128i>(src)); } 3366 template<> BL_INLINE_NODEBUG __m256i simd_loadu_64(const void* src) noexcept { return _mm256_castsi128_si256(simd_loadu_64<__m128i>(src)); } 3367 template<> BL_INLINE_NODEBUG __m256i simd_loada_128(const void* src) noexcept { return _mm256_castsi128_si256(simd_loada_128<__m128i>(src)); } 3368 template<> BL_INLINE_NODEBUG __m256i simd_loadu_128(const void* src) noexcept { return _mm256_castsi128_si256(simd_loadu_128<__m128i>(src)); } 3369 template<> BL_INLINE_NODEBUG __m256i simd_loada_256(const void* src) noexcept { return _mm256_load_si256(static_cast<const __m256i*>(src)); } 3370 template<> BL_INLINE_NODEBUG __m256i simd_loadu_256(const void* src) noexcept { return _mm256_loadu_si256(static_cast<const __m256i*>(src)); } 3371 3372 template<> BL_INLINE_NODEBUG __m256 simd_loada_32(const void* src) noexcept { return _mm256_castps128_ps256(simd_loada_32<__m128>(src)); } 3373 template<> BL_INLINE_NODEBUG __m256 simd_loadu_32(const void* src) noexcept { return _mm256_castps128_ps256(simd_loadu_32<__m128>(src)); } 3374 template<> BL_INLINE_NODEBUG __m256 simd_loada_64(const void* src) noexcept { return _mm256_castps128_ps256(simd_loada_64<__m128>(src)); } 3375 template<> BL_INLINE_NODEBUG __m256 simd_loadu_64(const void* src) noexcept { return _mm256_castps128_ps256(simd_loadu_64<__m128>(src)); } 3376 template<> BL_INLINE_NODEBUG __m256 simd_loada_128(const void* src) noexcept { return _mm256_castps128_ps256(simd_loada_128<__m128>(src)); } 3377 template<> BL_INLINE_NODEBUG __m256 simd_loadu_128(const void* src) noexcept { return _mm256_castps128_ps256(simd_loadu_128<__m128>(src)); } 3378 template<> BL_INLINE_NODEBUG __m256 simd_loada_256(const void* src) noexcept { return _mm256_load_ps(static_cast<const float*>(src)); } 3379 template<> BL_INLINE_NODEBUG __m256 simd_loadu_256(const void* src) noexcept { return _mm256_loadu_ps(static_cast<const float*>(src)); } 3380 3381 template<> BL_INLINE_NODEBUG __m256d simd_loada_64(const void* src) noexcept { return _mm256_castpd128_pd256(simd_loada_64<__m128d>(src)); } 3382 template<> BL_INLINE_NODEBUG __m256d simd_loadu_64(const void* src) noexcept { return _mm256_castpd128_pd256(simd_loadu_64<__m128d>(src)); } 3383 template<> BL_INLINE_NODEBUG __m256d simd_loada_128(const void* src) noexcept { return _mm256_castpd128_pd256(simd_loada_128<__m128d>(src)); } 3384 template<> BL_INLINE_NODEBUG __m256d simd_loadu_128(const void* src) noexcept { return _mm256_castpd128_pd256(simd_loadu_128<__m128d>(src)); } 3385 template<> BL_INLINE_NODEBUG __m256d simd_loada_256(const void* src) noexcept { return _mm256_load_pd(static_cast<const double*>(src)); } 3386 template<> BL_INLINE_NODEBUG __m256d simd_loadu_256(const void* src) noexcept { return _mm256_loadu_pd(static_cast<const double*>(src)); } 3387 3388 // MSCV won't emit a single instruction if load_XX() is used to load from memory first. 3389 #if defined(_MSC_VER) && !defined(__clang__) 3390 BL_INLINE_NODEBUG __m256i simd_loadu_64_i8_i32(const void* src) noexcept { return _mm256_cvtepi8_epi32(*static_cast<const unaligned_m128i*>(src)); } 3391 BL_INLINE_NODEBUG __m256i simd_loadu_64_u8_u32(const void* src) noexcept { return _mm256_cvtepu8_epi32(*static_cast<const unaligned_m128i*>(src)); } 3392 BL_INLINE_NODEBUG __m256i simd_loadu_32_i8_i64(const void* src) noexcept { return _mm256_cvtepi8_epi64(*static_cast<const unaligned_m128i*>(src)); } 3393 BL_INLINE_NODEBUG __m256i simd_loadu_32_u8_u64(const void* src) noexcept { return _mm256_cvtepu8_epi64(*static_cast<const unaligned_m128i*>(src)); } 3394 #else 3395 BL_INLINE_NODEBUG __m256i simd_loadu_64_i8_i32(const void* src) noexcept { return _mm256_cvtepi8_epi32(simd_loadu_64<__m128i>(src)); } 3396 BL_INLINE_NODEBUG __m256i simd_loadu_64_u8_u32(const void* src) noexcept { return _mm256_cvtepu8_epi32(simd_loadu_64<__m128i>(src)); } 3397 BL_INLINE_NODEBUG __m256i simd_loadu_32_i8_i64(const void* src) noexcept { return _mm256_cvtepi8_epi64(simd_loadu_32<__m128i>(src)); } 3398 BL_INLINE_NODEBUG __m256i simd_loadu_32_u8_u64(const void* src) noexcept { return _mm256_cvtepu8_epi64(simd_loadu_32<__m128i>(src)); } 3399 #endif 3400 3401 BL_INLINE_NODEBUG __m256i simd_loada_64_i8_i32(const void* src) noexcept { return simd_loadu_64_i8_i32(src); } 3402 BL_INLINE_NODEBUG __m256i simd_loada_64_u8_u32(const void* src) noexcept { return simd_loadu_64_u8_u32(src); } 3403 BL_INLINE_NODEBUG __m256i simd_loada_32_i8_i64(const void* src) noexcept { return simd_loadu_32_i8_i64(src); } 3404 BL_INLINE_NODEBUG __m256i simd_loada_32_u8_u64(const void* src) noexcept { return simd_loadu_32_u8_u64(src); } 3405 3406 BL_INLINE_NODEBUG __m256i simd_loada_128_i8_i16(const void* src) noexcept { return _mm256_cvtepi8_epi16(*static_cast<const __m128i*>(src)); } 3407 BL_INLINE_NODEBUG __m256i simd_loadu_128_i8_i16(const void* src) noexcept { return _mm256_cvtepi8_epi16(*static_cast<const unaligned_m128i*>(src)); } 3408 BL_INLINE_NODEBUG __m256i simd_loada_128_u8_u16(const void* src) noexcept { return _mm256_cvtepu8_epi16(*static_cast<const __m128i*>(src)); } 3409 BL_INLINE_NODEBUG __m256i simd_loadu_128_u8_u16(const void* src) noexcept { return _mm256_cvtepu8_epi16(*static_cast<const unaligned_m128i*>(src)); } 3410 3411 BL_INLINE_NODEBUG __m128i simd_load_broadcast128_i32(const void* src) noexcept { return simd_cast<__m128i>(_mm_broadcast_ss(static_cast<const float*>(src))); } 3412 BL_INLINE_NODEBUG __m128i simd_load_broadcast128_i64(const void* src) noexcept { return simd_cast<__m128i>(_mm_movedup_pd(simd_loadu_64<__m128d>(src))); } 3413 BL_INLINE_NODEBUG __m128 simd_load_broadcast128_f32(const void* src) noexcept { return _mm_broadcast_ss(static_cast<const float*>(src)); } 3414 BL_INLINE_NODEBUG __m128d simd_load_broadcast128_f64(const void* src) noexcept { return _mm_movedup_pd(simd_loadu_64<__m128d>(src)); } 3415 3416 BL_INLINE_NODEBUG __m256i simd_load_broadcast256_i32(const void* src) noexcept { return simd_cast<__m256i>(_mm256_broadcast_ss(static_cast<const float*>(src))); } 3417 BL_INLINE_NODEBUG __m256i simd_load_broadcast256_i64(const void* src) noexcept { return simd_cast<__m256i>(_mm256_broadcast_sd(static_cast<const double*>(src))); } 3418 BL_INLINE_NODEBUG __m256 simd_load_broadcast256_f32(const void* src) noexcept { return _mm256_broadcast_ss(static_cast<const float*>(src)); } 3419 BL_INLINE_NODEBUG __m256d simd_load_broadcast256_f64(const void* src) noexcept { return _mm256_broadcast_sd(static_cast<const double*>(src)); } 3420 3421 BL_INLINE_NODEBUG __m256i simd_load_broadcast256_4xi32(const void* src) noexcept { return _mm256_broadcastsi128_si256(simd_loadu_128<__m128i>(src)); } 3422 BL_INLINE_NODEBUG __m256i simd_load_broadcast256_2xi64(const void* src) noexcept { return _mm256_broadcastsi128_si256(simd_loadu_128<__m128i>(src)); } 3423 BL_INLINE_NODEBUG __m256 simd_load_broadcast256_f32x4(const void* src) noexcept { return _mm256_broadcast_ps(static_cast<const __m128*>(src)); } 3424 BL_INLINE_NODEBUG __m256d simd_load_broadcast256_f64x2(const void* src) noexcept { return _mm256_broadcast_pd(static_cast<const __m128d*>(src)); } 3425 3426 template<> BL_INLINE_NODEBUG __m128i simd_load_broadcast_u32<16>(const void* src) noexcept { return simd_load_broadcast128_i32(src); } 3427 template<> BL_INLINE_NODEBUG __m128i simd_load_broadcast_u64<16>(const void* src) noexcept { return simd_load_broadcast128_i64(src); } 3428 template<> BL_INLINE_NODEBUG __m128 simd_load_broadcast_f32<16>(const void* src) noexcept { return simd_load_broadcast128_f32(src); } 3429 template<> BL_INLINE_NODEBUG __m128d simd_load_broadcast_f64<16>(const void* src) noexcept { return simd_load_broadcast128_f64(src); } 3430 3431 template<> BL_INLINE_NODEBUG __m256i simd_load_broadcast_u32<32>(const void* src) noexcept { return simd_load_broadcast256_i32(src); } 3432 template<> BL_INLINE_NODEBUG __m256i simd_load_broadcast_u64<32>(const void* src) noexcept { return simd_load_broadcast256_i64(src); } 3433 template<> BL_INLINE_NODEBUG __m256 simd_load_broadcast_f32<32>(const void* src) noexcept { return simd_load_broadcast256_f32(src); } 3434 template<> BL_INLINE_NODEBUG __m256d simd_load_broadcast_f64<32>(const void* src) noexcept { return simd_load_broadcast256_f64(src); } 3435 3436 template<> BL_INLINE_NODEBUG __m256i simd_load_broadcast_4xi32<32>(const void* src) noexcept { return simd_load_broadcast256_4xi32(src); } 3437 template<> BL_INLINE_NODEBUG __m256i simd_load_broadcast_2xi64<32>(const void* src) noexcept { return simd_load_broadcast256_2xi64(src); } 3438 template<> BL_INLINE_NODEBUG __m256 simd_load_broadcast_f32x4<32>(const void* src) noexcept { return simd_load_broadcast256_f32x4(src); } 3439 template<> BL_INLINE_NODEBUG __m256d simd_load_broadcast_f64x2<32>(const void* src) noexcept { return simd_load_broadcast256_f64x2(src); } 3440 3441 BL_INLINE_NODEBUG void simd_store_8(void* dst, __m256i src) noexcept { simd_store_8(dst, _mm256_castsi256_si128(src)); } 3442 BL_INLINE_NODEBUG void simd_storea_16(void* dst, __m256i src) noexcept { simd_storea_16(dst, _mm256_castsi256_si128(src)); } 3443 BL_INLINE_NODEBUG void simd_storeu_16(void* dst, __m256i src) noexcept { simd_storeu_16(dst, _mm256_castsi256_si128(src)); } 3444 BL_INLINE_NODEBUG void simd_storea_32(void* dst, __m256i src) noexcept { simd_storea_32(dst, _mm256_castsi256_si128(src)); } 3445 BL_INLINE_NODEBUG void simd_storeu_32(void* dst, __m256i src) noexcept { simd_storeu_32(dst, _mm256_castsi256_si128(src)); } 3446 BL_INLINE_NODEBUG void simd_storea_64(void* dst, __m256i src) noexcept { simd_storea_64(dst, _mm256_castsi256_si128(src)); } 3447 BL_INLINE_NODEBUG void simd_storeu_64(void* dst, __m256i src) noexcept { simd_storeu_64(dst, _mm256_castsi256_si128(src)); } 3448 BL_INLINE_NODEBUG void simd_storeh_64(void* dst, __m256i src) noexcept { simd_storeh_64(dst, _mm256_castsi256_si128(src)); } 3449 BL_INLINE_NODEBUG void simd_storea_128(void* dst, __m256i src) noexcept { simd_storea_128(dst, _mm256_castsi256_si128(src)); } 3450 BL_INLINE_NODEBUG void simd_storeu_128(void* dst, __m256i src) noexcept { simd_storeu_128(dst, _mm256_castsi256_si128(src)); } 3451 BL_INLINE_NODEBUG void simd_storea_256(void* dst, __m256i src) noexcept { _mm256_store_si256(static_cast<__m256i*>(dst), src); } 3452 BL_INLINE_NODEBUG void simd_storeu_256(void* dst, __m256i src) noexcept { _mm256_storeu_si256(static_cast<__m256i*>(dst), src); } 3453 3454 BL_INLINE_NODEBUG void simd_storea_32(void* dst, __m256 src) noexcept { simd_storea_32(dst, _mm256_castps256_ps128(src)); } 3455 BL_INLINE_NODEBUG void simd_storeu_32(void* dst, __m256 src) noexcept { simd_storeu_32(dst, _mm256_castps256_ps128(src)); } 3456 BL_INLINE_NODEBUG void simd_storea_64(void* dst, __m256 src) noexcept { simd_storea_64(dst, _mm256_castps256_ps128(src)); } 3457 BL_INLINE_NODEBUG void simd_storeu_64(void* dst, __m256 src) noexcept { simd_storeu_64(dst, _mm256_castps256_ps128(src)); } 3458 BL_INLINE_NODEBUG void simd_storeh_64(void* dst, __m256 src) noexcept { simd_storeh_64(dst, _mm256_castps256_ps128(src)); } 3459 BL_INLINE_NODEBUG void simd_storea_128(void* dst, __m256 src) noexcept { simd_storea_128(dst, _mm256_castps256_ps128(src)); } 3460 BL_INLINE_NODEBUG void simd_storeu_128(void* dst, __m256 src) noexcept { simd_storeu_128(dst, _mm256_castps256_ps128(src)); } 3461 BL_INLINE_NODEBUG void simd_storea_256(void* dst, __m256 src) noexcept { _mm256_store_ps(static_cast<float*>(dst), src); } 3462 BL_INLINE_NODEBUG void simd_storeu_256(void* dst, __m256 src) noexcept { _mm256_storeu_ps(static_cast<float*>(dst), src); } 3463 3464 BL_INLINE_NODEBUG void simd_storea_64(void* dst, __m256d src) noexcept { simd_storea_64(dst, _mm256_castpd256_pd128(src)); } 3465 BL_INLINE_NODEBUG void simd_storeu_64(void* dst, __m256d src) noexcept { simd_storeu_64(dst, _mm256_castpd256_pd128(src)); } 3466 BL_INLINE_NODEBUG void simd_storeh_64(void* dst, __m256d src) noexcept { simd_storeh_64(dst, _mm256_castpd256_pd128(src)); } 3467 BL_INLINE_NODEBUG void simd_storea_128(void* dst, __m256d src) noexcept { simd_storea_128(dst, _mm256_castpd256_pd128(src)); } 3468 BL_INLINE_NODEBUG void simd_storeu_128(void* dst, __m256d src) noexcept { simd_storeu_128(dst, _mm256_castpd256_pd128(src)); } 3469 BL_INLINE_NODEBUG void simd_storea_256(void* dst, __m256d src) noexcept { _mm256_store_pd(static_cast<double*>(dst), src); } 3470 BL_INLINE_NODEBUG void simd_storeu_256(void* dst, __m256d src) noexcept { _mm256_storeu_pd(static_cast<double*>(dst), src); } 3471 3472 BL_INLINE_NODEBUG void simd_storea(void* dst, __m256i src) noexcept { _mm256_store_si256(static_cast<__m256i*>(dst), src); } 3473 BL_INLINE_NODEBUG void simd_storeu(void* dst, __m256i src) noexcept { _mm256_storeu_si256(static_cast<__m256i*>(dst), src); } 3474 BL_INLINE_NODEBUG void simd_storea(void* dst, __m256 src) noexcept { _mm256_store_ps(static_cast<float*>(dst), src); } 3475 BL_INLINE_NODEBUG void simd_storeu(void* dst, __m256 src) noexcept { _mm256_storeu_ps(static_cast<float*>(dst), src); } 3476 BL_INLINE_NODEBUG void simd_storea(void* dst, __m256d src) noexcept { _mm256_store_pd(static_cast<double*>(dst), src); } 3477 BL_INLINE_NODEBUG void simd_storeu(void* dst, __m256d src) noexcept { _mm256_storeu_pd(static_cast<double*>(dst), src); } 3478 #endif // BL_TARGET_OPT_AVX 3479 3480 #if defined(BL_TARGET_OPT_AVX512) 3481 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loada_512(const void* src) noexcept; 3482 template<typename SimdT> BL_INLINE_NODEBUG SimdT simd_loadu_512(const void* src) noexcept; 3483 3484 template<> BL_INLINE_NODEBUG __m512i simd_loada(const void* src) noexcept { return _mm512_load_si512(src); } 3485 template<> BL_INLINE_NODEBUG __m512i simd_loadu(const void* src) noexcept { return _mm512_loadu_si512(src); } 3486 template<> BL_INLINE_NODEBUG __m512 simd_loada(const void* src) noexcept { return _mm512_load_ps(src); } 3487 template<> BL_INLINE_NODEBUG __m512 simd_loadu(const void* src) noexcept { return _mm512_loadu_ps(src); } 3488 template<> BL_INLINE_NODEBUG __m512d simd_loada(const void* src) noexcept { return _mm512_load_pd(src); } 3489 template<> BL_INLINE_NODEBUG __m512d simd_loadu(const void* src) noexcept { return _mm512_loadu_pd(src); } 3490 3491 template<> BL_INLINE_NODEBUG __m512i simd_load_8(const void* src) noexcept { return _mm512_castsi128_si512(simd_load_8<__m128i>(src)); } 3492 template<> BL_INLINE_NODEBUG __m512i simd_loada_16(const void* src) noexcept { return _mm512_castsi128_si512(simd_loada_16<__m128i>(src)); } 3493 template<> BL_INLINE_NODEBUG __m512i simd_loadu_16(const void* src) noexcept { return _mm512_castsi128_si512(simd_loadu_16<__m128i>(src)); } 3494 template<> BL_INLINE_NODEBUG __m512i simd_loada_32(const void* src) noexcept { return _mm512_castsi128_si512(simd_loada_32<__m128i>(src)); } 3495 template<> BL_INLINE_NODEBUG __m512i simd_loadu_32(const void* src) noexcept { return _mm512_castsi128_si512(simd_loadu_32<__m128i>(src)); } 3496 template<> BL_INLINE_NODEBUG __m512i simd_loada_64(const void* src) noexcept { return _mm512_castsi128_si512(simd_loada_64<__m128i>(src)); } 3497 template<> BL_INLINE_NODEBUG __m512i simd_loadu_64(const void* src) noexcept { return _mm512_castsi128_si512(simd_loadu_64<__m128i>(src)); } 3498 template<> BL_INLINE_NODEBUG __m512i simd_loada_128(const void* src) noexcept { return _mm512_castsi128_si512(simd_loada_128<__m128i>(src)); } 3499 template<> BL_INLINE_NODEBUG __m512i simd_loadu_128(const void* src) noexcept { return _mm512_castsi128_si512(simd_loadu_128<__m128i>(src)); } 3500 template<> BL_INLINE_NODEBUG __m512i simd_loada_256(const void* src) noexcept { return _mm512_castsi256_si512(simd_loada_256<__m256i>(src)); } 3501 template<> BL_INLINE_NODEBUG __m512i simd_loadu_256(const void* src) noexcept { return _mm512_castsi256_si512(simd_loadu_256<__m256i>(src)); } 3502 template<> BL_INLINE_NODEBUG __m512i simd_loada_512(const void* src) noexcept { return _mm512_load_si512(src); } 3503 template<> BL_INLINE_NODEBUG __m512i simd_loadu_512(const void* src) noexcept { return _mm512_loadu_si512(src); } 3504 3505 template<> BL_INLINE_NODEBUG __m512 simd_loada_32(const void* src) noexcept { return _mm512_castps128_ps512(simd_loada_32<__m128>(src)); } 3506 template<> BL_INLINE_NODEBUG __m512 simd_loadu_32(const void* src) noexcept { return _mm512_castps128_ps512(simd_loadu_32<__m128>(src)); } 3507 template<> BL_INLINE_NODEBUG __m512 simd_loada_64(const void* src) noexcept { return _mm512_castps128_ps512(simd_loada_64<__m128>(src)); } 3508 template<> BL_INLINE_NODEBUG __m512 simd_loadu_64(const void* src) noexcept { return _mm512_castps128_ps512(simd_loadu_64<__m128>(src)); } 3509 template<> BL_INLINE_NODEBUG __m512 simd_loada_128(const void* src) noexcept { return _mm512_castps128_ps512(simd_loada_128<__m128>(src)); } 3510 template<> BL_INLINE_NODEBUG __m512 simd_loadu_128(const void* src) noexcept { return _mm512_castps128_ps512(simd_loadu_128<__m128>(src)); } 3511 template<> BL_INLINE_NODEBUG __m512 simd_loada_256(const void* src) noexcept { return _mm512_castps256_ps512(simd_loada_256<__m256>(src)); } 3512 template<> BL_INLINE_NODEBUG __m512 simd_loadu_256(const void* src) noexcept { return _mm512_castps256_ps512(simd_loadu_256<__m256>(src)); } 3513 template<> BL_INLINE_NODEBUG __m512 simd_loada_512(const void* src) noexcept { return _mm512_load_ps(src); } 3514 template<> BL_INLINE_NODEBUG __m512 simd_loadu_512(const void* src) noexcept { return _mm512_loadu_ps(src); } 3515 3516 template<> BL_INLINE_NODEBUG __m512d simd_loada_64(const void* src) noexcept { return _mm512_castpd128_pd512(simd_loada_64<__m128d>(src)); } 3517 template<> BL_INLINE_NODEBUG __m512d simd_loadu_64(const void* src) noexcept { return _mm512_castpd128_pd512(simd_loadu_64<__m128d>(src)); } 3518 template<> BL_INLINE_NODEBUG __m512d simd_loada_128(const void* src) noexcept { return _mm512_castpd128_pd512(simd_loada_128<__m128d>(src)); } 3519 template<> BL_INLINE_NODEBUG __m512d simd_loadu_128(const void* src) noexcept { return _mm512_castpd128_pd512(simd_loadu_128<__m128d>(src)); } 3520 template<> BL_INLINE_NODEBUG __m512d simd_loada_256(const void* src) noexcept { return _mm512_castpd256_pd512(simd_loada_256<__m256d>(src)); } 3521 template<> BL_INLINE_NODEBUG __m512d simd_loadu_256(const void* src) noexcept { return _mm512_castpd256_pd512(simd_loadu_256<__m256d>(src)); } 3522 template<> BL_INLINE_NODEBUG __m512d simd_loada_512(const void* src) noexcept { return _mm512_load_pd(src); } 3523 template<> BL_INLINE_NODEBUG __m512d simd_loadu_512(const void* src) noexcept { return _mm512_loadu_pd(src); } 3524 3525 // MSCV won't emit a single instruction if load_XX() is used to load from memory first. 3526 #if defined(_MSC_VER) && !defined(__clang__) 3527 BL_INLINE_NODEBUG __m512i simd_loada_128_i8_i32(const void* src) noexcept { return _mm512_cvtepi8_epi32(*static_cast<const __m128i*>(src)); } 3528 BL_INLINE_NODEBUG __m512i simd_loada_128_u8_u32(const void* src) noexcept { return _mm512_cvtepu8_epi32(*static_cast<const __m128i*>(src)); } 3529 BL_INLINE_NODEBUG __m512i simd_loadu_128_i8_i32(const void* src) noexcept { return _mm512_cvtepi8_epi32(*static_cast<const unaligned_m128i*>(src)); } 3530 BL_INLINE_NODEBUG __m512i simd_loadu_128_u8_u32(const void* src) noexcept { return _mm512_cvtepu8_epi32(*static_cast<const unaligned_m128i*>(src)); } 3531 BL_INLINE_NODEBUG __m512i simd_loadu_64_i8_i64(const void* src) noexcept { return _mm512_cvtepi8_epi64(*static_cast<const unaligned_m128i*>(src)); } 3532 BL_INLINE_NODEBUG __m512i simd_loadu_64_u8_u64(const void* src) noexcept { return _mm512_cvtepu8_epi64(*static_cast<const unaligned_m128i*>(src)); } 3533 #else 3534 BL_INLINE_NODEBUG __m512i simd_loada_128_i8_i32(const void* src) noexcept { return _mm512_cvtepi8_epi32(*static_cast<const __m128i*>(src)); } 3535 BL_INLINE_NODEBUG __m512i simd_loada_128_u8_u32(const void* src) noexcept { return _mm512_cvtepu8_epi32(*static_cast<const __m128i*>(src)); } 3536 BL_INLINE_NODEBUG __m512i simd_loadu_128_i8_i32(const void* src) noexcept { return _mm512_cvtepi8_epi32(simd_loadu_128<__m128i>(src)); } 3537 BL_INLINE_NODEBUG __m512i simd_loadu_128_u8_u32(const void* src) noexcept { return _mm512_cvtepu8_epi32(simd_loadu_128<__m128i>(src)); } 3538 BL_INLINE_NODEBUG __m512i simd_loadu_64_i8_i64(const void* src) noexcept { return _mm512_cvtepi8_epi64(simd_loadu_64<__m128i>(src)); } 3539 BL_INLINE_NODEBUG __m512i simd_loadu_64_u8_u64(const void* src) noexcept { return _mm512_cvtepu8_epi64(simd_loadu_64<__m128i>(src)); } 3540 #endif 3541 3542 BL_INLINE_NODEBUG __m512i simd_loada_64_i8_i64(const void* src) noexcept { return simd_loadu_64_i8_i64(src); } 3543 BL_INLINE_NODEBUG __m512i simd_loada_64_u8_u64(const void* src) noexcept { return simd_loadu_64_u8_u64(src); } 3544 3545 BL_INLINE_NODEBUG __m512i simd_loada_256_i8_i16(const void* src) noexcept { return _mm512_cvtepi8_epi16(*static_cast<const __m256i*>(src)); } 3546 BL_INLINE_NODEBUG __m512i simd_loadu_256_i8_i16(const void* src) noexcept { return _mm512_cvtepi8_epi16(*static_cast<const unaligned_m256i*>(src)); } 3547 BL_INLINE_NODEBUG __m512i simd_loada_256_u8_u16(const void* src) noexcept { return _mm512_cvtepu8_epi16(*static_cast<const __m256i*>(src)); } 3548 BL_INLINE_NODEBUG __m512i simd_loadu_256_u8_u16(const void* src) noexcept { return _mm512_cvtepu8_epi16(*static_cast<const unaligned_m256i*>(src)); } 3549 3550 BL_INLINE_NODEBUG __m512i simd_load_broadcast512_i32(const void* src) noexcept { return _mm512_broadcastd_epi32(simd_loadu_32<__m128i>(src)); } 3551 BL_INLINE_NODEBUG __m512i simd_load_broadcast512_i64(const void* src) noexcept { return _mm512_broadcastq_epi64(simd_loadu_64<__m128i>(src)); } 3552 BL_INLINE_NODEBUG __m512 simd_load_broadcast512_f32(const void* src) noexcept { return _mm512_broadcastss_ps(simd_loadu_32<__m128>(src)); } 3553 BL_INLINE_NODEBUG __m512d simd_load_broadcast512_f64(const void* src) noexcept { return _mm512_broadcastsd_pd(simd_loadu_64<__m128d>(src)); } 3554 3555 BL_INLINE_NODEBUG __m512i simd_load_broadcast512_4xi32(const void* src) noexcept { return _mm512_broadcast_i32x4(simd_loadu_128<__m128i>(src)); } 3556 BL_INLINE_NODEBUG __m512i simd_load_broadcast512_2xi64(const void* src) noexcept { return _mm512_broadcast_i64x2(simd_loadu_128<__m128i>(src)); } 3557 BL_INLINE_NODEBUG __m512 simd_load_broadcast512_f32x4(const void* src) noexcept { return _mm512_broadcast_f32x4(simd_loadu_128<__m128>(src)); } 3558 BL_INLINE_NODEBUG __m512d simd_load_broadcast512_f64x2(const void* src) noexcept { return _mm512_broadcast_f64x2(simd_loadu_128<__m128d>(src)); } 3559 3560 template<> BL_INLINE_NODEBUG __m512i simd_load_broadcast_u32<64>(const void* src) noexcept { return simd_load_broadcast512_i32(src); } 3561 template<> BL_INLINE_NODEBUG __m512i simd_load_broadcast_u64<64>(const void* src) noexcept { return simd_load_broadcast512_i64(src); } 3562 template<> BL_INLINE_NODEBUG __m512 simd_load_broadcast_f32<64>(const void* src) noexcept { return simd_load_broadcast512_f32(src); } 3563 template<> BL_INLINE_NODEBUG __m512d simd_load_broadcast_f64<64>(const void* src) noexcept { return simd_load_broadcast512_f64(src); } 3564 3565 template<> BL_INLINE_NODEBUG __m512i simd_load_broadcast_4xi32<64>(const void* src) noexcept { return simd_load_broadcast512_4xi32(src); } 3566 template<> BL_INLINE_NODEBUG __m512i simd_load_broadcast_2xi64<64>(const void* src) noexcept { return simd_load_broadcast512_2xi64(src); } 3567 template<> BL_INLINE_NODEBUG __m512 simd_load_broadcast_f32x4<64>(const void* src) noexcept { return simd_load_broadcast512_f32x4(src); } 3568 template<> BL_INLINE_NODEBUG __m512d simd_load_broadcast_f64x2<64>(const void* src) noexcept { return simd_load_broadcast512_f64x2(src); } 3569 3570 BL_INLINE_NODEBUG void simd_store_8(void* dst, __m512i src) noexcept { simd_store_8(dst, _mm512_castsi512_si128(src)); } 3571 BL_INLINE_NODEBUG void simd_storea_16(void* dst, __m512i src) noexcept { simd_storea_16(dst, _mm512_castsi512_si128(src)); } 3572 BL_INLINE_NODEBUG void simd_storeu_16(void* dst, __m512i src) noexcept { simd_storeu_16(dst, _mm512_castsi512_si128(src)); } 3573 BL_INLINE_NODEBUG void simd_storea_32(void* dst, __m512i src) noexcept { simd_storea_32(dst, _mm512_castsi512_si128(src)); } 3574 BL_INLINE_NODEBUG void simd_storeu_32(void* dst, __m512i src) noexcept { simd_storeu_32(dst, _mm512_castsi512_si128(src)); } 3575 BL_INLINE_NODEBUG void simd_storea_64(void* dst, __m512i src) noexcept { simd_storea_64(dst, _mm512_castsi512_si128(src)); } 3576 BL_INLINE_NODEBUG void simd_storeu_64(void* dst, __m512i src) noexcept { simd_storeu_64(dst, _mm512_castsi512_si128(src)); } 3577 BL_INLINE_NODEBUG void simd_storeh_64(void* dst, __m512i src) noexcept { simd_storeh_64(dst, _mm512_castsi512_si128(src)); } 3578 BL_INLINE_NODEBUG void simd_storea_128(void* dst, __m512i src) noexcept { simd_storea_128(dst, _mm512_castsi512_si128(src)); } 3579 BL_INLINE_NODEBUG void simd_storeu_128(void* dst, __m512i src) noexcept { simd_storeu_128(dst, _mm512_castsi512_si128(src)); } 3580 BL_INLINE_NODEBUG void simd_storea_256(void* dst, __m512i src) noexcept { simd_storea_256(dst, _mm512_castsi512_si256(src)); } 3581 BL_INLINE_NODEBUG void simd_storeu_256(void* dst, __m512i src) noexcept { simd_storeu_256(dst, _mm512_castsi512_si256(src)); } 3582 BL_INLINE_NODEBUG void simd_storea_512(void* dst, __m512i src) noexcept { _mm512_store_si512(static_cast<__m512i*>(dst), src); } 3583 BL_INLINE_NODEBUG void simd_storeu_512(void* dst, __m512i src) noexcept { _mm512_storeu_si512(static_cast<__m512i*>(dst), src); } 3584 3585 BL_INLINE_NODEBUG void simd_storea_32(void* dst, __m512 src) noexcept { simd_storea_32(dst, _mm512_castps512_ps128(src)); } 3586 BL_INLINE_NODEBUG void simd_storeu_32(void* dst, __m512 src) noexcept { simd_storeu_32(dst, _mm512_castps512_ps128(src)); } 3587 BL_INLINE_NODEBUG void simd_storea_64(void* dst, __m512 src) noexcept { simd_storea_64(dst, _mm512_castps512_ps128(src)); } 3588 BL_INLINE_NODEBUG void simd_storeu_64(void* dst, __m512 src) noexcept { simd_storeu_64(dst, _mm512_castps512_ps128(src)); } 3589 BL_INLINE_NODEBUG void simd_storeh_64(void* dst, __m512 src) noexcept { simd_storeh_64(dst, _mm512_castps512_ps128(src)); } 3590 BL_INLINE_NODEBUG void simd_storea_128(void* dst, __m512 src) noexcept { simd_storea_128(dst, _mm512_castps512_ps128(src)); } 3591 BL_INLINE_NODEBUG void simd_storeu_128(void* dst, __m512 src) noexcept { simd_storeu_128(dst, _mm512_castps512_ps128(src)); } 3592 BL_INLINE_NODEBUG void simd_storea_256(void* dst, __m512 src) noexcept { simd_storea_256(dst, _mm512_castps512_ps256(src)); } 3593 BL_INLINE_NODEBUG void simd_storeu_256(void* dst, __m512 src) noexcept { simd_storeu_256(dst, _mm512_castps512_ps256(src)); } 3594 BL_INLINE_NODEBUG void simd_storea_512(void* dst, __m512 src) noexcept { _mm512_store_ps(static_cast<float*>(dst), src); } 3595 BL_INLINE_NODEBUG void simd_storeu_512(void* dst, __m512 src) noexcept { _mm512_storeu_ps(static_cast<float*>(dst), src); } 3596 3597 BL_INLINE_NODEBUG void simd_storea_64(void* dst, __m512d src) noexcept { simd_storea_64(dst, _mm512_castpd512_pd128(src)); } 3598 BL_INLINE_NODEBUG void simd_storeu_64(void* dst, __m512d src) noexcept { simd_storeu_64(dst, _mm512_castpd512_pd128(src)); } 3599 BL_INLINE_NODEBUG void simd_storeh_64(void* dst, __m512d src) noexcept { simd_storeh_64(dst, _mm512_castpd512_pd128(src)); } 3600 BL_INLINE_NODEBUG void simd_storea_128(void* dst, __m512d src) noexcept { simd_storea_128(dst, _mm512_castpd512_pd128(src)); } 3601 BL_INLINE_NODEBUG void simd_storeu_128(void* dst, __m512d src) noexcept { simd_storeu_128(dst, _mm512_castpd512_pd128(src)); } 3602 BL_INLINE_NODEBUG void simd_storea_256(void* dst, __m512d src) noexcept { simd_storea_256(dst, _mm512_castpd512_pd256(src)); } 3603 BL_INLINE_NODEBUG void simd_storeu_256(void* dst, __m512d src) noexcept { simd_storeu_256(dst, _mm512_castpd512_pd256(src)); } 3604 BL_INLINE_NODEBUG void simd_storea_512(void* dst, __m512d src) noexcept { _mm512_store_pd(static_cast<double*>(dst), src); } 3605 BL_INLINE_NODEBUG void simd_storeu_512(void* dst, __m512d src) noexcept { _mm512_storeu_pd(static_cast<double*>(dst), src); } 3606 3607 BL_INLINE_NODEBUG void simd_storea(void* dst, __m512i src) noexcept { _mm512_store_si512(static_cast<__m512i*>(dst), src); } 3608 BL_INLINE_NODEBUG void simd_storeu(void* dst, __m512i src) noexcept { _mm512_storeu_si512(static_cast<__m512i*>(dst), src); } 3609 BL_INLINE_NODEBUG void simd_storea(void* dst, __m512 src) noexcept { _mm512_store_ps(static_cast<float*>(dst), src); } 3610 BL_INLINE_NODEBUG void simd_storeu(void* dst, __m512 src) noexcept { _mm512_storeu_ps(static_cast<float*>(dst), src); } 3611 BL_INLINE_NODEBUG void simd_storea(void* dst, __m512d src) noexcept { _mm512_store_pd(static_cast<double*>(dst), src); } 3612 BL_INLINE_NODEBUG void simd_storeu(void* dst, __m512d src) noexcept { _mm512_storeu_pd(static_cast<double*>(dst), src); } 3613 #endif // BL_TARGET_OPT_AVX512 3614 3615 } // {Internal} 3616 3617 // SIMD - Public - Make Zero & Ones & Undefined 3618 // ============================================ 3619 3620 template<typename V> 3621 BL_INLINE_NODEBUG V make_zero() noexcept { return V{I::simd_make_zero<typename V::SimdType>()}; } 3622 3623 template<typename V> 3624 BL_INLINE_NODEBUG V make_ones() noexcept { return V{I::simd_make_ones<typename V::SimdType>()}; } 3625 3626 template<typename V> 3627 BL_INLINE_NODEBUG V make_undefined() noexcept { return V{I::simd_make_undefined<typename V::SimdType>()}; } 3628 3629 // SIMD - Public - Make Vector (Any) 3630 // ================================= 3631 3632 template<typename V, typename... Args> BL_INLINE_NODEBUG V make_i8(Args&&... args) noexcept { return V{I::simd_make_i8<typename V::SimdType>(BLInternal::forward<Args>(args)...)}; } 3633 template<typename V, typename... Args> BL_INLINE_NODEBUG V make_i16(Args&&... args) noexcept { return V{I::simd_make_i16<typename V::SimdType>(BLInternal::forward<Args>(args)...)}; } 3634 template<typename V, typename... Args> BL_INLINE_NODEBUG V make_i32(Args&&... args) noexcept { return V{I::simd_make_i32<typename V::SimdType>(BLInternal::forward<Args>(args)...)}; } 3635 template<typename V, typename... Args> BL_INLINE_NODEBUG V make_i64(Args&&... args) noexcept { return V{I::simd_make_i64<typename V::SimdType>(BLInternal::forward<Args>(args)...)}; } 3636 template<typename V, typename... Args> BL_INLINE_NODEBUG V make_u8(Args&&... args) noexcept { return V{I::simd_make_u8<typename V::SimdType>(BLInternal::forward<Args>(args)...)}; } 3637 template<typename V, typename... Args> BL_INLINE_NODEBUG V make_u16(Args&&... args) noexcept { return V{I::simd_make_u16<typename V::SimdType>(BLInternal::forward<Args>(args)...)}; } 3638 template<typename V, typename... Args> BL_INLINE_NODEBUG V make_u32(Args&&... args) noexcept { return V{I::simd_make_u32<typename V::SimdType>(BLInternal::forward<Args>(args)...)}; } 3639 template<typename V, typename... Args> BL_INLINE_NODEBUG V make_u64(Args&&... args) noexcept { return V{I::simd_make_u64<typename V::SimdType>(BLInternal::forward<Args>(args)...)}; } 3640 template<typename V, typename... Args> BL_INLINE_NODEBUG V make_f32(Args&&... args) noexcept { return V{I::simd_make_f32<typename V::SimdType>(BLInternal::forward<Args>(args)...)}; } 3641 template<typename V, typename... Args> BL_INLINE_NODEBUG V make_f64(Args&&... args) noexcept { return V{I::simd_make_f64<typename V::SimdType>(BLInternal::forward<Args>(args)...)}; } 3642 3643 // SIMD - Public - Make Vector (128-bit) 3644 // ===================================== 3645 3646 template<typename V = Vec16xI8> 3647 BL_INLINE_NODEBUG V make128_i8(int8_t x0) noexcept { 3648 return from_simd<V>(I::simd_make128_u8(uint8_t(x0))); 3649 } 3650 3651 template<typename V = Vec16xI8> 3652 BL_INLINE_NODEBUG V make128_i8(int8_t x1, int8_t x0) noexcept { 3653 return from_simd<V>(I::simd_make128_u8(uint8_t(x1), uint8_t(x0))); 3654 } 3655 3656 template<typename V = Vec16xI8> 3657 BL_INLINE_NODEBUG V make128_i8(int8_t x3, int8_t x2, int8_t x1, int8_t x0) noexcept { 3658 return from_simd<V>(I::simd_make128_u8(uint8_t(x3), uint8_t(x2), uint8_t(x1), uint8_t(x0))); 3659 } 3660 3661 template<typename V = Vec16xI8> 3662 BL_INLINE_NODEBUG V make128_i8(int8_t x7, int8_t x6, int8_t x5, int8_t x4, 3663 int8_t x3, int8_t x2, int8_t x1, int8_t x0) noexcept { 3664 return from_simd<V>( 3665 I::simd_make128_u8( 3666 uint8_t(x7), uint8_t(x6), uint8_t(x5), uint8_t(x4), 3667 uint8_t(x3), uint8_t(x2), uint8_t(x1), uint8_t(x0))); 3668 } 3669 3670 template<typename V = Vec16xI8> 3671 BL_INLINE_NODEBUG V make128_i8(int8_t x15, int8_t x14, int8_t x13, int8_t x12, 3672 int8_t x11, int8_t x10, int8_t x09, int8_t x08, 3673 int8_t x07, int8_t x06, int8_t x05, int8_t x04, 3674 int8_t x03, int8_t x02, int8_t x01, int8_t x00) noexcept { 3675 return from_simd<V>( 3676 I::simd_make128_u8( 3677 uint8_t(x15), uint8_t(x14), uint8_t(x13), uint8_t(x12), 3678 uint8_t(x11), uint8_t(x10), uint8_t(x09), uint8_t(x08), 3679 uint8_t(x07), uint8_t(x06), uint8_t(x05), uint8_t(x04), 3680 uint8_t(x03), uint8_t(x02), uint8_t(x01), uint8_t(x00))); 3681 } 3682 3683 template<typename V = Vec16xU8> 3684 BL_INLINE_NODEBUG V make128_u8(uint8_t x0) noexcept { 3685 return from_simd<V>(I::simd_make128_u8(x0)); 3686 } 3687 3688 template<typename V = Vec16xU8> 3689 BL_INLINE_NODEBUG V make128_u8(uint8_t x1, uint8_t x0) noexcept { 3690 return from_simd<V>(I::simd_make128_u8(x1, x0)); 3691 } 3692 3693 template<typename V = Vec16xU8> 3694 BL_INLINE_NODEBUG V make128_u8(uint8_t x3, uint8_t x2, uint8_t x1, uint8_t x0) noexcept { 3695 return from_simd<V>(I::simd_make128_u8(x3, x2, x1, x0)); 3696 } 3697 3698 template<typename V = Vec16xU8> 3699 BL_INLINE_NODEBUG V make128_u8(uint8_t x7, uint8_t x6, uint8_t x5, uint8_t x4, 3700 uint8_t x3, uint8_t x2, uint8_t x1, uint8_t x0) noexcept { 3701 return from_simd<V>( 3702 I::simd_make128_u8( 3703 x7, x6, x5, x4, 3704 x3, x2, x1, x0)); 3705 } 3706 3707 template<typename V = Vec16xU8> 3708 BL_INLINE_NODEBUG V make128_u8(uint8_t x15, uint8_t x14, uint8_t x13, uint8_t x12, 3709 uint8_t x11, uint8_t x10, uint8_t x09, uint8_t x08, 3710 uint8_t x07, uint8_t x06, uint8_t x05, uint8_t x04, 3711 uint8_t x03, uint8_t x02, uint8_t x01, uint8_t x00) noexcept { 3712 return from_simd<V>( 3713 I::simd_make128_u8( 3714 x15, x14, x13, x12, 3715 x11, x10, x09, x08, 3716 x07, x06, x05, x04, 3717 x03, x02, x01, x00)); 3718 } 3719 3720 template<typename V = Vec8xI16> 3721 BL_INLINE_NODEBUG V make128_i16(int16_t x0) noexcept { 3722 return from_simd<V>(I::simd_make128_u16(uint16_t(x0))); 3723 } 3724 3725 template<typename V = Vec8xI16> 3726 BL_INLINE_NODEBUG V make128_i16(int16_t x1, int16_t x0) noexcept { 3727 return from_simd<V>(I::simd_make128_u16(uint16_t(x1), uint16_t(x0))); 3728 } 3729 3730 template<typename V = Vec8xI16> 3731 BL_INLINE_NODEBUG V make128_i16(int16_t x3, int16_t x2, int16_t x1, int16_t x0) noexcept { 3732 return from_simd<V>(I::simd_make128_u16(uint16_t(x3), uint16_t(x2), uint16_t(x1), uint16_t(x0))); 3733 } 3734 3735 template<typename V = Vec8xI16> 3736 BL_INLINE_NODEBUG V make128_i16(int16_t x7, int16_t x6, int16_t x5, int16_t x4, 3737 int16_t x3, int16_t x2, int16_t x1, int16_t x0) noexcept { 3738 return from_simd<V>( 3739 I::simd_make128_u16( 3740 uint16_t(x7), uint16_t(x6), uint16_t(x5), uint16_t(x4), 3741 uint16_t(x3), uint16_t(x2), uint16_t(x1), uint16_t(x0))); 3742 } 3743 3744 template<typename V = Vec8xU16> 3745 BL_INLINE_NODEBUG V make128_u16(uint16_t x0) noexcept { 3746 return from_simd<V>(I::simd_make128_u16(x0)); 3747 } 3748 3749 template<typename V = Vec8xU16> 3750 BL_INLINE_NODEBUG V make128_u16(uint16_t x1, uint16_t x0) noexcept { 3751 return from_simd<V>(I::simd_make128_u16(x1, x0)); 3752 } 3753 3754 template<typename V = Vec8xU16> 3755 BL_INLINE_NODEBUG V make128_u16(uint16_t x3, uint16_t x2, uint16_t x1, uint16_t x0) noexcept { 3756 return from_simd<V>(I::simd_make128_u16(x3, x2, x1, x0)); 3757 } 3758 3759 template<typename V = Vec8xU16> 3760 BL_INLINE_NODEBUG V make128_u16(uint16_t x7, uint16_t x6, uint16_t x5, uint16_t x4, 3761 uint16_t x3, uint16_t x2, uint16_t x1, uint16_t x0) noexcept { 3762 return from_simd<V>( 3763 I::simd_make128_u16( 3764 x7, x6, x5, x4, 3765 x3, x2, x1, x0)); 3766 } 3767 3768 template<typename V = Vec4xI32> 3769 BL_INLINE_NODEBUG V make128_i32(int32_t x0) noexcept { 3770 return from_simd<V>(I::simd_make128_u32(uint32_t(x0))); 3771 } 3772 3773 template<typename V = Vec4xI32> 3774 BL_INLINE_NODEBUG V make128_i32(int32_t x1, int32_t x0) noexcept { 3775 return from_simd<V>(I::simd_make128_u32(uint32_t(x1), uint32_t(x0))); 3776 } 3777 3778 template<typename V = Vec4xI32> 3779 BL_INLINE_NODEBUG V make128_i32(int32_t x3, int32_t x2, int32_t x1, int32_t x0) noexcept { 3780 return from_simd<V>(I::simd_make128_u32(uint32_t(x3), uint32_t(x2), uint32_t(x1), uint32_t(x0))); 3781 } 3782 3783 template<typename V = Vec4xU32> 3784 BL_INLINE_NODEBUG V make128_u32(uint32_t x0) noexcept { 3785 return from_simd<V>(I::simd_make128_u32(x0)); 3786 } 3787 3788 template<typename V = Vec4xU32> 3789 BL_INLINE_NODEBUG V make128_u32(uint32_t x1, uint32_t x0) noexcept { 3790 return from_simd<V>(I::simd_make128_u32(x1, x0)); 3791 } 3792 3793 template<typename V = Vec4xU32> 3794 BL_INLINE_NODEBUG V make128_u32(uint32_t x3, uint32_t x2, uint32_t x1, uint32_t x0) noexcept { 3795 return from_simd<V>(I::simd_make128_u32(x3, x2, x1, x0)); 3796 } 3797 3798 template<typename V = Vec2xI64> 3799 BL_INLINE_NODEBUG V make128_i64(int64_t x0) noexcept { 3800 return from_simd<V>(I::simd_make128_u64(uint64_t(x0))); 3801 } 3802 3803 template<typename V = Vec2xI64> 3804 BL_INLINE_NODEBUG V make128_i64(int64_t x1, int64_t x0) noexcept { 3805 return from_simd<V>(I::simd_make128_u64(uint64_t(x1), uint64_t(x0))); 3806 } 3807 3808 template<typename V = Vec2xU64> 3809 BL_INLINE_NODEBUG V make128_u64(uint64_t x0) noexcept { 3810 return from_simd<V>(I::simd_make128_u64(x0)); 3811 } 3812 3813 template<typename V = Vec2xU64> 3814 BL_INLINE_NODEBUG V make128_u64(uint64_t x1, uint64_t x0) noexcept { 3815 return from_simd<V>(I::simd_make128_u64(x1, x0)); 3816 } 3817 3818 template<typename V = Vec4xF32> 3819 BL_INLINE_NODEBUG V make128_f32(float x0) noexcept { 3820 return from_simd<V>(I::simd_make128_f32(x0)); 3821 } 3822 3823 template<typename V = Vec4xF32> 3824 BL_INLINE_NODEBUG V make128_f32(float x1, float x0) noexcept { 3825 return from_simd<V>(I::simd_make128_f32(x1, x0)); 3826 } 3827 3828 template<typename V = Vec4xF32> 3829 BL_INLINE_NODEBUG V make128_f32(float x3, float x2, float x1, float x0) noexcept { 3830 return from_simd<V>(I::simd_make128_f32(x3, x2, x1, x0)); 3831 } 3832 3833 template<typename V = Vec2xF64> 3834 BL_INLINE_NODEBUG V make128_f64(double x0) noexcept { 3835 return from_simd<V>(I::simd_make128_f64(x0)); 3836 } 3837 3838 template<typename V = Vec2xF64> 3839 BL_INLINE_NODEBUG V make128_f64(double x1, double x0) noexcept { 3840 return from_simd<V>(I::simd_make128_f64(x1, x0)); 3841 } 3842 3843 // SIMD - Public - Make Vector (256-bit) 3844 // ===================================== 3845 3846 #ifdef BL_TARGET_OPT_AVX 3847 template<typename V = Vec32xI8> 3848 BL_INLINE_NODEBUG V make256_i8(int8_t x0) noexcept { 3849 return from_simd<V>(I::simd_make256_u8(uint8_t(x0))); 3850 } 3851 3852 template<typename V = Vec32xI8> 3853 BL_INLINE_NODEBUG V make256_i8(int8_t x1, int8_t x0) noexcept { 3854 return from_simd<V>(I::simd_make256_u8(uint8_t(x1), uint8_t(x0))); 3855 } 3856 3857 template<typename V = Vec32xI8> 3858 BL_INLINE_NODEBUG V make256_i8(int8_t x3, int8_t x2, int8_t x1, int8_t x0) noexcept { 3859 return from_simd<V>(I::simd_make256_u8(uint8_t(x3), uint8_t(x2), uint8_t(x1), uint8_t(x0))); 3860 } 3861 3862 template<typename V = Vec32xI8> 3863 BL_INLINE_NODEBUG V make256_i8(int8_t x7, int8_t x6, int8_t x5, int8_t x4, 3864 int8_t x3, int8_t x2, int8_t x1, int8_t x0) noexcept { 3865 return from_simd<V>( 3866 I::simd_make256_u8( 3867 uint8_t(x7), uint8_t(x6), uint8_t(x5), uint8_t(x4), 3868 uint8_t(x3), uint8_t(x2), uint8_t(x1), uint8_t(x0))); 3869 } 3870 3871 template<typename V = Vec32xI8> 3872 BL_INLINE_NODEBUG V make256_i8(int8_t x15, int8_t x14, int8_t x13, int8_t x12, 3873 int8_t x11, int8_t x10, int8_t x09, int8_t x08, 3874 int8_t x07, int8_t x06, int8_t x05, int8_t x04, 3875 int8_t x03, int8_t x02, int8_t x01, int8_t x00) noexcept { 3876 return from_simd<V>( 3877 I::simd_make256_u8( 3878 uint8_t(x15), uint8_t(x14), uint8_t(x13), uint8_t(x12), 3879 uint8_t(x11), uint8_t(x10), uint8_t(x09), uint8_t(x08), 3880 uint8_t(x07), uint8_t(x06), uint8_t(x05), uint8_t(x04), 3881 uint8_t(x03), uint8_t(x02), uint8_t(x01), uint8_t(x00))); 3882 } 3883 3884 template<typename V = Vec32xI8> 3885 BL_INLINE_NODEBUG V make256_i8(int8_t x31, int8_t x30, int8_t x29, int8_t x28, 3886 int8_t x27, int8_t x26, int8_t x25, int8_t x24, 3887 int8_t x23, int8_t x22, int8_t x21, int8_t x20, 3888 int8_t x19, int8_t x18, int8_t x17, int8_t x16, 3889 int8_t x15, int8_t x14, int8_t x13, int8_t x12, 3890 int8_t x11, int8_t x10, int8_t x09, int8_t x08, 3891 int8_t x07, int8_t x06, int8_t x05, int8_t x04, 3892 int8_t x03, int8_t x02, int8_t x01, int8_t x00) noexcept { 3893 return from_simd<V>( 3894 I::simd_make256_u8( 3895 uint8_t(x31), uint8_t(x30), uint8_t(x29), uint8_t(x28), 3896 uint8_t(x27), uint8_t(x26), uint8_t(x25), uint8_t(x24), 3897 uint8_t(x23), uint8_t(x22), uint8_t(x21), uint8_t(x20), 3898 uint8_t(x19), uint8_t(x18), uint8_t(x17), uint8_t(x16), 3899 uint8_t(x15), uint8_t(x14), uint8_t(x13), uint8_t(x12), 3900 uint8_t(x11), uint8_t(x10), uint8_t(x09), uint8_t(x08), 3901 uint8_t(x07), uint8_t(x06), uint8_t(x05), uint8_t(x04), 3902 uint8_t(x03), uint8_t(x02), uint8_t(x01), uint8_t(x00))); 3903 } 3904 3905 template<typename V = Vec32xU8> 3906 BL_INLINE_NODEBUG V make256_u8(uint8_t x0) noexcept { 3907 return from_simd<V>(I::simd_make256_u8(x0)); 3908 } 3909 3910 template<typename V = Vec32xU8> 3911 BL_INLINE_NODEBUG V make256_u8(uint8_t x1, uint8_t x0) noexcept { 3912 return from_simd<V>(I::simd_make256_u8(x1, x0)); 3913 } 3914 3915 template<typename V = Vec32xU8> 3916 BL_INLINE_NODEBUG V make256_u8(uint8_t x3, uint8_t x2, uint8_t x1, uint8_t x0) noexcept { 3917 return from_simd<V>(I::simd_make256_u8(x3, x2, x1, x0)); 3918 } 3919 3920 template<typename V = Vec32xU8> 3921 BL_INLINE_NODEBUG V make256_u8(uint8_t x7, uint8_t x6, uint8_t x5, uint8_t x4, 3922 uint8_t x3, uint8_t x2, uint8_t x1, uint8_t x0) noexcept { 3923 return from_simd<V>( 3924 I::simd_make256_u8( 3925 x7, x6, x5, x4, 3926 x3, x2, x1, x0)); 3927 } 3928 3929 template<typename V = Vec32xU8> 3930 BL_INLINE_NODEBUG V make256_u8(uint8_t x15, uint8_t x14, uint8_t x13, uint8_t x12, 3931 uint8_t x11, uint8_t x10, uint8_t x09, uint8_t x08, 3932 uint8_t x07, uint8_t x06, uint8_t x05, uint8_t x04, 3933 uint8_t x03, uint8_t x02, uint8_t x01, uint8_t x00) noexcept { 3934 return from_simd<V>( 3935 I::simd_make256_u8( 3936 x15, x14, x13, x12, 3937 x11, x10, x09, x08, 3938 x07, x06, x05, x04, 3939 x03, x02, x01, x00)); 3940 } 3941 3942 template<typename V = Vec32xU8> 3943 BL_INLINE_NODEBUG V make256_u8(uint8_t x31, uint8_t x30, uint8_t x29, uint8_t x28, 3944 uint8_t x27, uint8_t x26, uint8_t x25, uint8_t x24, 3945 uint8_t x23, uint8_t x22, uint8_t x21, uint8_t x20, 3946 uint8_t x19, uint8_t x18, uint8_t x17, uint8_t x16, 3947 uint8_t x15, uint8_t x14, uint8_t x13, uint8_t x12, 3948 uint8_t x11, uint8_t x10, uint8_t x09, uint8_t x08, 3949 uint8_t x07, uint8_t x06, uint8_t x05, uint8_t x04, 3950 uint8_t x03, uint8_t x02, uint8_t x01, uint8_t x00) noexcept { 3951 return from_simd<V>( 3952 I::simd_make256_u8( 3953 x31, x30, x29, x28, 3954 x27, x26, x25, x24, 3955 x23, x22, x21, x20, 3956 x19, x18, x17, x16, 3957 x15, x14, x13, x12, 3958 x11, x10, x09, x08, 3959 x07, x06, x05, x04, 3960 x03, x02, x01, x00)); 3961 } 3962 3963 template<typename V = Vec16xI16> 3964 BL_INLINE_NODEBUG V make256_i16(int16_t x0) noexcept { 3965 return from_simd<V>(I::simd_make256_u16(uint16_t(x0))); 3966 } 3967 3968 template<typename V = Vec16xI16> 3969 BL_INLINE_NODEBUG V make256_i16(int16_t x1, int16_t x0) noexcept { 3970 return from_simd<V>(I::simd_make256_u16(uint16_t(x1), uint16_t(x0))); 3971 } 3972 3973 template<typename V = Vec16xI16> 3974 BL_INLINE_NODEBUG V make256_i16(int16_t x3, int16_t x2, int16_t x1, int16_t x0) noexcept { 3975 return from_simd<V>(I::simd_make256_u16(uint16_t(x3), uint16_t(x2), uint16_t(x1), uint16_t(x0))); 3976 } 3977 3978 template<typename V = Vec16xI16> 3979 BL_INLINE_NODEBUG V make256_i16(int16_t x7, int16_t x6, int16_t x5, int16_t x4, 3980 int16_t x3, int16_t x2, int16_t x1, int16_t x0) noexcept { 3981 return from_simd<V>( 3982 I::simd_make256_u16( 3983 uint16_t(x7), uint16_t(x6), uint16_t(x5), uint16_t(x4), 3984 uint16_t(x3), uint16_t(x2), uint16_t(x1), uint16_t(x0))); 3985 } 3986 3987 template<typename V = Vec16xI16> 3988 BL_INLINE_NODEBUG V make256_i16(int16_t x15, int16_t x14, int16_t x13, int16_t x12, 3989 int16_t x11, int16_t x10, int16_t x09, int16_t x08, 3990 int16_t x07, int16_t x06, int16_t x05, int16_t x04, 3991 int16_t x03, int16_t x02, int16_t x01, int16_t x00) noexcept { 3992 return from_simd<V>( 3993 I::simd_make256_u16( 3994 uint16_t(x15), uint16_t(x14), uint16_t(x13), uint16_t(x12), 3995 uint16_t(x11), uint16_t(x10), uint16_t(x09), uint16_t(x08), 3996 uint16_t(x07), uint16_t(x06), uint16_t(x05), uint16_t(x04), 3997 uint16_t(x03), uint16_t(x02), uint16_t(x01), uint16_t(x00))); 3998 } 3999 4000 template<typename V = Vec16xU16> 4001 BL_INLINE_NODEBUG V make256_u16(uint16_t x0) noexcept { 4002 return from_simd<V>(I::simd_make256_u16(x0)); 4003 } 4004 4005 template<typename V = Vec16xU16> 4006 BL_INLINE_NODEBUG V make256_u16(uint16_t x1, uint16_t x0) noexcept { 4007 return from_simd<V>(I::simd_make256_u16(x1, x0)); 4008 } 4009 4010 template<typename V = Vec16xU16> 4011 BL_INLINE_NODEBUG V make256_u16(uint16_t x3, uint16_t x2, uint16_t x1, uint16_t x0) noexcept { 4012 return from_simd<V>(I::simd_make256_u16(x3, x2, x1, x0)); 4013 } 4014 4015 template<typename V = Vec16xU16> 4016 BL_INLINE_NODEBUG V make256_u16(uint16_t x7, uint16_t x6, uint16_t x5, uint16_t x4, 4017 uint16_t x3, uint16_t x2, uint16_t x1, uint16_t x0) noexcept { 4018 return from_simd<V>( 4019 I::simd_make256_u16( 4020 x7, x6, x5, x4, 4021 x3, x2, x1, x0)); 4022 } 4023 4024 template<typename V = Vec16xU16> 4025 BL_INLINE_NODEBUG V make256_u16(uint16_t x15, uint16_t x14, uint16_t x13, uint16_t x12, 4026 uint16_t x11, uint16_t x10, uint16_t x09, uint16_t x08, 4027 uint16_t x07, uint16_t x06, uint16_t x05, uint16_t x04, 4028 uint16_t x03, uint16_t x02, uint16_t x01, uint16_t x00) noexcept { 4029 return from_simd<V>( 4030 I::simd_make256_u16( 4031 x15, x14, x13, x12, 4032 x11, x10, x09, x08, 4033 x07, x06, x05, x04, 4034 x03, x02, x01, x00)); 4035 } 4036 4037 template<typename V = Vec8xI32> 4038 BL_INLINE_NODEBUG V make256_i32(int32_t x0) noexcept { 4039 return from_simd<V>(I::simd_make256_u32(uint32_t(x0))); 4040 } 4041 4042 template<typename V = Vec8xI32> 4043 BL_INLINE_NODEBUG V make256_i32(int32_t x1, int32_t x0) noexcept { 4044 return from_simd<V>(I::simd_make256_u32(uint32_t(x1), uint32_t(x0))); 4045 } 4046 4047 template<typename V = Vec8xI32> 4048 BL_INLINE_NODEBUG V make256_i32(int32_t x3, int32_t x2, int32_t x1, int32_t x0) noexcept { 4049 return from_simd<V>(I::simd_make256_u32(uint32_t(x3), uint32_t(x2), uint32_t(x1), uint32_t(x0))); 4050 } 4051 4052 template<typename V = Vec8xI32> 4053 BL_INLINE_NODEBUG V make256_i32(int32_t x7, int32_t x6, int32_t x5, int32_t x4, 4054 int32_t x3, int32_t x2, int32_t x1, int32_t x0) noexcept { 4055 return from_simd<V>( 4056 I::simd_make256_u32( 4057 uint32_t(x7), uint32_t(x6), uint32_t(x5), uint32_t(x4), 4058 uint32_t(x3), uint32_t(x2), uint32_t(x1), uint32_t(x0))); 4059 } 4060 4061 template<typename V = Vec8xU32> 4062 BL_INLINE_NODEBUG V make256_u32(uint32_t x0) noexcept { 4063 return from_simd<V>(I::simd_make256_u32(x0)); 4064 } 4065 4066 template<typename V = Vec8xU32> 4067 BL_INLINE_NODEBUG V make256_u32(uint32_t x1, uint32_t x0) noexcept { 4068 return from_simd<V>(I::simd_make256_u32(x1, x0)); 4069 } 4070 4071 template<typename V = Vec8xU32> 4072 BL_INLINE_NODEBUG V make256_u32(uint32_t x3, uint32_t x2, uint32_t x1, uint32_t x0) noexcept { 4073 return from_simd<V>(I::simd_make256_u32(x3, x2, x1, x0)); 4074 } 4075 4076 template<typename V = Vec8xU32> 4077 BL_INLINE_NODEBUG V make256_u32(uint32_t x7, uint32_t x6, uint32_t x5, uint32_t x4, 4078 uint32_t x3, uint32_t x2, uint32_t x1, uint32_t x0) noexcept { 4079 return from_simd<V>(I::simd_make256_u32(x7, x6, x5, x4, x3, x2, x1, x0)); 4080 } 4081 4082 template<typename V = Vec4xI64> 4083 BL_INLINE_NODEBUG V make256_i64(int64_t x0) noexcept { 4084 return from_simd<V>(I::simd_make256_u64(uint64_t(x0))); 4085 } 4086 4087 template<typename V = Vec4xI64> 4088 BL_INLINE_NODEBUG V make256_i64(int64_t x1, int64_t x0) noexcept { 4089 return from_simd<V>(I::simd_make256_u64(uint64_t(x1), uint64_t(x0))); 4090 } 4091 4092 template<typename V = Vec4xI64> 4093 BL_INLINE_NODEBUG V make256_i64(int64_t x3, int64_t x2, int64_t x1, int64_t x0) noexcept { 4094 return from_simd<V>(I::simd_make256_u64(uint64_t(x3), uint64_t(x2), uint64_t(x1), uint64_t(x0))); 4095 } 4096 4097 template<typename V = Vec4xU64> 4098 BL_INLINE_NODEBUG V make256_u64(uint64_t x0) noexcept { 4099 return from_simd<V>(I::simd_make256_u64(x0)); 4100 } 4101 4102 template<typename V = Vec4xU64> 4103 BL_INLINE_NODEBUG V make256_u64(uint64_t x1, uint64_t x0) noexcept { 4104 return from_simd<V>(I::simd_make256_u64(x1, x0)); 4105 } 4106 4107 template<typename V = Vec4xU64> 4108 BL_INLINE_NODEBUG V make256_u64(uint64_t x3, uint64_t x2, uint64_t x1, uint64_t x0) noexcept { 4109 return from_simd<V>(I::simd_make256_u64(x3, x2, x1, x0)); 4110 } 4111 4112 template<typename V = Vec8xF32> 4113 BL_INLINE_NODEBUG V make256_f32(float x0) noexcept { 4114 return from_simd<V>(I::simd_make256_f32(x0)); 4115 } 4116 4117 template<typename V = Vec8xF32> 4118 BL_INLINE_NODEBUG V make256_f32(float x1, float x0) noexcept { 4119 return from_simd<V>(I::simd_make256_f32(x1, x0)); 4120 } 4121 4122 template<typename V = Vec8xF32> 4123 BL_INLINE_NODEBUG V make256_f32(float x3, float x2, float x1, float x0) noexcept { 4124 return from_simd<V>(I::simd_make256_f32(x3, x2, x1, x0)); 4125 } 4126 4127 template<typename V = Vec8xF32> 4128 BL_INLINE_NODEBUG V make256_f32(float x7, float x6, float x5, float x4, 4129 float x3, float x2, float x1, float x0) noexcept { 4130 return from_simd<V>(I::simd_make256_f32(x7, x6, x5, x4, x3, x2, x1, x0)); 4131 } 4132 4133 template<typename V = Vec4xF64> 4134 BL_INLINE_NODEBUG V make256_f64(double x0) noexcept { 4135 return from_simd<V>(I::simd_make256_f64(x0)); 4136 } 4137 4138 template<typename V = Vec4xF64> 4139 BL_INLINE_NODEBUG V make256_f64(double x1, double x0) noexcept { 4140 return from_simd<V>(I::simd_make256_f64(x1, x0)); 4141 } 4142 4143 template<typename V = Vec4xF64> 4144 BL_INLINE_NODEBUG V make256_f64(double x3, double x2, double x1, double x0) noexcept { 4145 return from_simd<V>(I::simd_make256_f64(x3, x2, x1, x0)); 4146 } 4147 #endif // BL_TARGET_OPT_AVX 4148 4149 // SIMD - Public - Make Vector (512-bit) 4150 // ===================================== 4151 4152 #ifdef BL_TARGET_OPT_AVX512 4153 template<typename V = Vec64xI8> 4154 BL_INLINE_NODEBUG V make512_i8(int8_t x0) noexcept { 4155 return from_simd<V>(I::simd_make512_u8(uint8_t(x0))); 4156 } 4157 4158 template<typename V = Vec64xI8> 4159 BL_INLINE_NODEBUG V make512_i8(int8_t x1, int8_t x0) noexcept { 4160 return from_simd<V>(I::simd_make512_u8(uint8_t(x1), uint8_t(x0))); 4161 } 4162 4163 template<typename V = Vec64xI8> 4164 BL_INLINE_NODEBUG V make512_i8(int8_t x3, int8_t x2, int8_t x1, int8_t x0) noexcept { 4165 return from_simd<V>(I::simd_make512_u8(uint8_t(x3), uint8_t(x2), uint8_t(x1), uint8_t(x0))); 4166 } 4167 4168 template<typename V = Vec64xI8> 4169 BL_INLINE_NODEBUG V make512_i8(int8_t x7, int8_t x6, int8_t x5, int8_t x4, 4170 int8_t x3, int8_t x2, int8_t x1, int8_t x0) noexcept { 4171 return from_simd<V>( 4172 I::simd_make512_u8( 4173 uint8_t(x7), uint8_t(x6), uint8_t(x5), uint8_t(x4), 4174 uint8_t(x3), uint8_t(x2), uint8_t(x1), uint8_t(x0))); 4175 } 4176 4177 template<typename V = Vec64xI8> 4178 BL_INLINE_NODEBUG V make512_i8(int8_t x15, int8_t x14, int8_t x13, int8_t x12, 4179 int8_t x11, int8_t x10, int8_t x09, int8_t x08, 4180 int8_t x07, int8_t x06, int8_t x05, int8_t x04, 4181 int8_t x03, int8_t x02, int8_t x01, int8_t x00) noexcept { 4182 return from_simd<V>( 4183 I::simd_make512_u8( 4184 uint8_t(x15), uint8_t(x14), uint8_t(x13), uint8_t(x12), 4185 uint8_t(x11), uint8_t(x10), uint8_t(x09), uint8_t(x08), 4186 uint8_t(x07), uint8_t(x06), uint8_t(x05), uint8_t(x04), 4187 uint8_t(x03), uint8_t(x02), uint8_t(x01), uint8_t(x00))); 4188 } 4189 4190 template<typename V = Vec64xI8> 4191 BL_INLINE_NODEBUG V make512_i8(int8_t x31, int8_t x30, int8_t x29, int8_t x28, 4192 int8_t x27, int8_t x26, int8_t x25, int8_t x24, 4193 int8_t x23, int8_t x22, int8_t x21, int8_t x20, 4194 int8_t x19, int8_t x18, int8_t x17, int8_t x16, 4195 int8_t x15, int8_t x14, int8_t x13, int8_t x12, 4196 int8_t x11, int8_t x10, int8_t x09, int8_t x08, 4197 int8_t x07, int8_t x06, int8_t x05, int8_t x04, 4198 int8_t x03, int8_t x02, int8_t x01, int8_t x00) noexcept { 4199 return from_simd<V>( 4200 I::simd_make512_u8( 4201 uint8_t(x31), uint8_t(x30), uint8_t(x29), uint8_t(x28), 4202 uint8_t(x27), uint8_t(x26), uint8_t(x25), uint8_t(x24), 4203 uint8_t(x23), uint8_t(x22), uint8_t(x21), uint8_t(x20), 4204 uint8_t(x19), uint8_t(x18), uint8_t(x17), uint8_t(x16), 4205 uint8_t(x15), uint8_t(x14), uint8_t(x13), uint8_t(x12), 4206 uint8_t(x11), uint8_t(x10), uint8_t(x09), uint8_t(x08), 4207 uint8_t(x07), uint8_t(x06), uint8_t(x05), uint8_t(x04), 4208 uint8_t(x03), uint8_t(x02), uint8_t(x01), uint8_t(x00))); 4209 } 4210 4211 template<typename V = Vec64xI8> 4212 BL_INLINE_NODEBUG V make512_i8(int8_t x63, int8_t x62, int8_t x61, int8_t x60, 4213 int8_t x59, int8_t x58, int8_t x57, int8_t x56, 4214 int8_t x55, int8_t x54, int8_t x53, int8_t x52, 4215 int8_t x51, int8_t x50, int8_t x49, int8_t x48, 4216 int8_t x47, int8_t x46, int8_t x45, int8_t x44, 4217 int8_t x43, int8_t x42, int8_t x41, int8_t x40, 4218 int8_t x39, int8_t x38, int8_t x37, int8_t x36, 4219 int8_t x35, int8_t x34, int8_t x33, int8_t x32, 4220 int8_t x31, int8_t x30, int8_t x29, int8_t x28, 4221 int8_t x27, int8_t x26, int8_t x25, int8_t x24, 4222 int8_t x23, int8_t x22, int8_t x21, int8_t x20, 4223 int8_t x19, int8_t x18, int8_t x17, int8_t x16, 4224 int8_t x15, int8_t x14, int8_t x13, int8_t x12, 4225 int8_t x11, int8_t x10, int8_t x09, int8_t x08, 4226 int8_t x07, int8_t x06, int8_t x05, int8_t x04, 4227 int8_t x03, int8_t x02, int8_t x01, int8_t x00) noexcept { 4228 return from_simd<V>( 4229 I::simd_make512_u8( 4230 uint8_t(x63), uint8_t(x62), uint8_t(x61), uint8_t(x60), 4231 uint8_t(x59), uint8_t(x58), uint8_t(x57), uint8_t(x56), 4232 uint8_t(x55), uint8_t(x54), uint8_t(x53), uint8_t(x52), 4233 uint8_t(x51), uint8_t(x50), uint8_t(x49), uint8_t(x48), 4234 uint8_t(x47), uint8_t(x46), uint8_t(x45), uint8_t(x44), 4235 uint8_t(x43), uint8_t(x42), uint8_t(x41), uint8_t(x40), 4236 uint8_t(x39), uint8_t(x38), uint8_t(x37), uint8_t(x36), 4237 uint8_t(x35), uint8_t(x34), uint8_t(x33), uint8_t(x32), 4238 uint8_t(x31), uint8_t(x30), uint8_t(x29), uint8_t(x28), 4239 uint8_t(x27), uint8_t(x26), uint8_t(x25), uint8_t(x24), 4240 uint8_t(x23), uint8_t(x22), uint8_t(x21), uint8_t(x20), 4241 uint8_t(x19), uint8_t(x18), uint8_t(x17), uint8_t(x16), 4242 uint8_t(x15), uint8_t(x14), uint8_t(x13), uint8_t(x12), 4243 uint8_t(x11), uint8_t(x10), uint8_t(x09), uint8_t(x08), 4244 uint8_t(x07), uint8_t(x06), uint8_t(x05), uint8_t(x04), 4245 uint8_t(x03), uint8_t(x02), uint8_t(x01), uint8_t(x00))); 4246 } 4247 4248 template<typename V = Vec64xU8> 4249 BL_INLINE_NODEBUG V make512_u8(uint8_t x0) noexcept { 4250 return from_simd<V>(I::simd_make512_u8(x0)); 4251 } 4252 4253 template<typename V = Vec64xU8> 4254 BL_INLINE_NODEBUG V make512_u8(uint8_t x1, uint8_t x0) noexcept { 4255 return from_simd<V>(I::simd_make512_u8(x1, x0)); 4256 } 4257 4258 template<typename V = Vec64xU8> 4259 BL_INLINE_NODEBUG V make512_u8(uint8_t x3, uint8_t x2, uint8_t x1, uint8_t x0) noexcept { 4260 return from_simd<V>(I::simd_make512_u8(x3, x2, x1, x0)); 4261 } 4262 4263 template<typename V = Vec64xU8> 4264 BL_INLINE_NODEBUG V make512_u8(uint8_t x7, uint8_t x6, uint8_t x5, uint8_t x4, 4265 uint8_t x3, uint8_t x2, uint8_t x1, uint8_t x0) noexcept { 4266 return from_simd<V>( 4267 I::simd_make512_u8( 4268 x7, x6, x5, x4, 4269 x3, x2, x1, x0)); 4270 } 4271 4272 template<typename V = Vec64xU8> 4273 BL_INLINE_NODEBUG V make512_u8(uint8_t x15, uint8_t x14, uint8_t x13, uint8_t x12, 4274 uint8_t x11, uint8_t x10, uint8_t x09, uint8_t x08, 4275 uint8_t x07, uint8_t x06, uint8_t x05, uint8_t x04, 4276 uint8_t x03, uint8_t x02, uint8_t x01, uint8_t x00) noexcept { 4277 return from_simd<V>( 4278 I::simd_make512_u8( 4279 x15, x14, x13, x12, 4280 x11, x10, x09, x08, 4281 x07, x06, x05, x04, 4282 x03, x02, x01, x00)); 4283 } 4284 4285 template<typename V = Vec64xU8> 4286 BL_INLINE_NODEBUG V make512_u8(uint8_t x31, uint8_t x30, uint8_t x29, uint8_t x28, 4287 uint8_t x27, uint8_t x26, uint8_t x25, uint8_t x24, 4288 uint8_t x23, uint8_t x22, uint8_t x21, uint8_t x20, 4289 uint8_t x19, uint8_t x18, uint8_t x17, uint8_t x16, 4290 uint8_t x15, uint8_t x14, uint8_t x13, uint8_t x12, 4291 uint8_t x11, uint8_t x10, uint8_t x09, uint8_t x08, 4292 uint8_t x07, uint8_t x06, uint8_t x05, uint8_t x04, 4293 uint8_t x03, uint8_t x02, uint8_t x01, uint8_t x00) noexcept { 4294 return from_simd<V>( 4295 I::simd_make512_u8( 4296 x31, x30, x29, x28, 4297 x27, x26, x25, x24, 4298 x23, x22, x21, x20, 4299 x19, x18, x17, x16, 4300 x15, x14, x13, x12, 4301 x11, x10, x09, x08, 4302 x07, x06, x05, x04, 4303 x03, x02, x01, x00)); 4304 } 4305 4306 template<typename V = Vec64xU8> 4307 BL_INLINE_NODEBUG V make512_u8(uint8_t x63, uint8_t x62, uint8_t x61, uint8_t x60, 4308 uint8_t x59, uint8_t x58, uint8_t x57, uint8_t x56, 4309 uint8_t x55, uint8_t x54, uint8_t x53, uint8_t x52, 4310 uint8_t x51, uint8_t x50, uint8_t x49, uint8_t x48, 4311 uint8_t x47, uint8_t x46, uint8_t x45, uint8_t x44, 4312 uint8_t x43, uint8_t x42, uint8_t x41, uint8_t x40, 4313 uint8_t x39, uint8_t x38, uint8_t x37, uint8_t x36, 4314 uint8_t x35, uint8_t x34, uint8_t x33, uint8_t x32, 4315 uint8_t x31, uint8_t x30, uint8_t x29, uint8_t x28, 4316 uint8_t x27, uint8_t x26, uint8_t x25, uint8_t x24, 4317 uint8_t x23, uint8_t x22, uint8_t x21, uint8_t x20, 4318 uint8_t x19, uint8_t x18, uint8_t x17, uint8_t x16, 4319 uint8_t x15, uint8_t x14, uint8_t x13, uint8_t x12, 4320 uint8_t x11, uint8_t x10, uint8_t x09, uint8_t x08, 4321 uint8_t x07, uint8_t x06, uint8_t x05, uint8_t x04, 4322 uint8_t x03, uint8_t x02, uint8_t x01, uint8_t x00) noexcept { 4323 return from_simd<V>( 4324 I::simd_make512_u8( 4325 x63, x62, x61, x60, 4326 x59, x58, x57, x56, 4327 x55, x54, x53, x52, 4328 x51, x50, x49, x48, 4329 x47, x46, x45, x44, 4330 x43, x42, x41, x40, 4331 x39, x38, x37, x36, 4332 x35, x34, x33, x32, 4333 x31, x30, x29, x28, 4334 x27, x26, x25, x24, 4335 x23, x22, x21, x20, 4336 x19, x18, x17, x16, 4337 x15, x14, x13, x12, 4338 x11, x10, x09, x08, 4339 x07, x06, x05, x04, 4340 x03, x02, x01, x00)); 4341 } 4342 4343 template<typename V = Vec32xI16> 4344 BL_INLINE_NODEBUG V make512_i16(int16_t x0) noexcept { 4345 return from_simd<V>(I::simd_make512_u16(uint16_t(x0))); 4346 } 4347 4348 template<typename V = Vec32xI16> 4349 BL_INLINE_NODEBUG V make512_i16(int16_t x1, int16_t x0) noexcept { 4350 return from_simd<V>(I::simd_make512_u16(uint16_t(x1), uint16_t(x0))); 4351 } 4352 4353 template<typename V = Vec32xI16> 4354 BL_INLINE_NODEBUG V make512_i16(int16_t x3, int16_t x2, int16_t x1, int16_t x0) noexcept { 4355 return from_simd<V>(I::simd_make512_u16(uint16_t(x3), uint16_t(x2), uint16_t(x1), uint16_t(x0))); 4356 } 4357 4358 template<typename V = Vec32xI16> 4359 BL_INLINE_NODEBUG V make512_i16(int16_t x7, int16_t x6, int16_t x5, int16_t x4, 4360 int16_t x3, int16_t x2, int16_t x1, int16_t x0) noexcept { 4361 return from_simd<V>( 4362 I::simd_make512_u16( 4363 uint16_t(x7), uint16_t(x6), uint16_t(x5), uint16_t(x4), 4364 uint16_t(x3), uint16_t(x2), uint16_t(x1), uint16_t(x0))); 4365 } 4366 4367 template<typename V = Vec32xI16> 4368 BL_INLINE_NODEBUG V make512_i16(int16_t x15, int16_t x14, int16_t x13, int16_t x12, 4369 int16_t x11, int16_t x10, int16_t x09, int16_t x08, 4370 int16_t x07, int16_t x06, int16_t x05, int16_t x04, 4371 int16_t x03, int16_t x02, int16_t x01, int16_t x00) noexcept { 4372 return from_simd<V>( 4373 I::simd_make512_u16( 4374 uint16_t(x15), uint16_t(x14), uint16_t(x13), uint16_t(x12), 4375 uint16_t(x11), uint16_t(x10), uint16_t(x09), uint16_t(x08), 4376 uint16_t(x07), uint16_t(x06), uint16_t(x05), uint16_t(x04), 4377 uint16_t(x03), uint16_t(x02), uint16_t(x01), uint16_t(x00))); 4378 } 4379 4380 template<typename V = Vec32xI16> 4381 BL_INLINE_NODEBUG V make512_i16(int16_t x31, int16_t x30, int16_t x29, int16_t x28, 4382 int16_t x27, int16_t x26, int16_t x25, int16_t x24, 4383 int16_t x23, int16_t x22, int16_t x21, int16_t x20, 4384 int16_t x19, int16_t x18, int16_t x17, int16_t x16, 4385 int16_t x15, int16_t x14, int16_t x13, int16_t x12, 4386 int16_t x11, int16_t x10, int16_t x09, int16_t x08, 4387 int16_t x07, int16_t x06, int16_t x05, int16_t x04, 4388 int16_t x03, int16_t x02, int16_t x01, int16_t x00) noexcept { 4389 return from_simd<V>( 4390 I::simd_make512_u16( 4391 uint16_t(x31), uint16_t(x30), uint16_t(x29), uint16_t(x28), 4392 uint16_t(x27), uint16_t(x26), uint16_t(x25), uint16_t(x24), 4393 uint16_t(x23), uint16_t(x22), uint16_t(x21), uint16_t(x20), 4394 uint16_t(x19), uint16_t(x18), uint16_t(x17), uint16_t(x16), 4395 uint16_t(x15), uint16_t(x14), uint16_t(x13), uint16_t(x12), 4396 uint16_t(x11), uint16_t(x10), uint16_t(x09), uint16_t(x08), 4397 uint16_t(x07), uint16_t(x06), uint16_t(x05), uint16_t(x04), 4398 uint16_t(x03), uint16_t(x02), uint16_t(x01), uint16_t(x00))); 4399 } 4400 4401 template<typename V = Vec32xU16> 4402 BL_INLINE_NODEBUG V make512_u16(uint16_t x0) noexcept { 4403 return from_simd<V>(I::simd_make512_u16(x0)); 4404 } 4405 4406 template<typename V = Vec32xU16> 4407 BL_INLINE_NODEBUG V make512_u16(uint16_t x1, uint16_t x0) noexcept { 4408 return from_simd<V>(I::simd_make512_u16(x1, x0)); 4409 } 4410 4411 template<typename V = Vec32xU16> 4412 BL_INLINE_NODEBUG V make512_u16(uint16_t x3, uint16_t x2, uint16_t x1, uint16_t x0) noexcept { 4413 return from_simd<V>(I::simd_make512_u16(x3, x2, x1, x0)); 4414 } 4415 4416 template<typename V = Vec32xU16> 4417 BL_INLINE_NODEBUG V make512_u16(uint16_t x7, uint16_t x6, uint16_t x5, uint16_t x4, 4418 uint16_t x3, uint16_t x2, uint16_t x1, uint16_t x0) noexcept { 4419 return from_simd<V>( 4420 I::simd_make512_u16( 4421 x7, x6, x5, x4, 4422 x3, x2, x1, x0)); 4423 } 4424 4425 template<typename V = Vec32xU16> 4426 BL_INLINE_NODEBUG V make512_u16(uint16_t x15, uint16_t x14, uint16_t x13, uint16_t x12, 4427 uint16_t x11, uint16_t x10, uint16_t x09, uint16_t x08, 4428 uint16_t x07, uint16_t x06, uint16_t x05, uint16_t x04, 4429 uint16_t x03, uint16_t x02, uint16_t x01, uint16_t x00) noexcept { 4430 return from_simd<V>( 4431 I::simd_make512_u16( 4432 x15, x14, x13, x12, 4433 x11, x10, x09, x08, 4434 x07, x06, x05, x04, 4435 x03, x02, x01, x00)); 4436 } 4437 4438 template<typename V = Vec32xU16> 4439 BL_INLINE_NODEBUG V make512_u16(uint16_t x31, uint16_t x30, uint16_t x29, uint16_t x28, 4440 uint16_t x27, uint16_t x26, uint16_t x25, uint16_t x24, 4441 uint16_t x23, uint16_t x22, uint16_t x21, uint16_t x20, 4442 uint16_t x19, uint16_t x18, uint16_t x17, uint16_t x16, 4443 uint16_t x15, uint16_t x14, uint16_t x13, uint16_t x12, 4444 uint16_t x11, uint16_t x10, uint16_t x09, uint16_t x08, 4445 uint16_t x07, uint16_t x06, uint16_t x05, uint16_t x04, 4446 uint16_t x03, uint16_t x02, uint16_t x01, uint16_t x00) noexcept { 4447 return from_simd<V>( 4448 I::simd_make512_u16( 4449 x31, x30, x29, x28, 4450 x27, x26, x25, x24, 4451 x23, x22, x21, x20, 4452 x19, x18, x17, x16, 4453 x15, x14, x13, x12, 4454 x11, x10, x09, x08, 4455 x07, x06, x05, x04, 4456 x03, x02, x01, x00)); 4457 } 4458 4459 template<typename V = Vec16xI32> 4460 BL_INLINE_NODEBUG V make512_i32(int32_t x0) noexcept { 4461 return from_simd<V>(I::simd_make512_u32(uint32_t(x0))); 4462 } 4463 4464 template<typename V = Vec16xI32> 4465 BL_INLINE_NODEBUG V make512_i32(int32_t x1, int32_t x0) noexcept { 4466 return from_simd<V>(I::simd_make512_u32(uint32_t(x1), uint32_t(x0))); 4467 } 4468 4469 template<typename V = Vec16xI32> 4470 BL_INLINE_NODEBUG V make512_i32(int32_t x3, int32_t x2, int32_t x1, int32_t x0) noexcept { 4471 return from_simd<V>(I::simd_make512_u32(uint32_t(x3), uint32_t(x2), uint32_t(x1), uint32_t(x0))); 4472 } 4473 4474 template<typename V = Vec16xI32> 4475 BL_INLINE_NODEBUG V make512_i32(int32_t x7, int32_t x6, int32_t x5, int32_t x4, 4476 int32_t x3, int32_t x2, int32_t x1, int32_t x0) noexcept { 4477 return from_simd<V>( 4478 I::simd_make512_u32( 4479 uint32_t(x7), uint32_t(x6), uint32_t(x5), uint32_t(x4), 4480 uint32_t(x3), uint32_t(x2), uint32_t(x1), uint32_t(x0))); 4481 } 4482 4483 template<typename V = Vec16xI32> 4484 BL_INLINE_NODEBUG V make512_i32(int32_t x15, int32_t x14, int32_t x13, int32_t x12, 4485 int32_t x11, int32_t x10, int32_t x09, int32_t x08, 4486 int32_t x07, int32_t x06, int32_t x05, int32_t x04, 4487 int32_t x03, int32_t x02, int32_t x01, int32_t x00) noexcept { 4488 return from_simd<V>( 4489 I::simd_make512_u32( 4490 uint32_t(x15), uint32_t(x14), uint32_t(x13), uint32_t(x12), 4491 uint32_t(x11), uint32_t(x10), uint32_t(x09), uint32_t(x08), 4492 uint32_t(x07), uint32_t(x06), uint32_t(x05), uint32_t(x04), 4493 uint32_t(x03), uint32_t(x02), uint32_t(x01), uint32_t(x00))); 4494 } 4495 4496 template<typename V = Vec16xU32> 4497 BL_INLINE_NODEBUG V make512_u32(uint32_t x0) noexcept { 4498 return from_simd<V>(I::simd_make512_u32(x0)); 4499 } 4500 4501 template<typename V = Vec16xU32> 4502 BL_INLINE_NODEBUG V make512_u32(uint32_t x1, uint32_t x0) noexcept { 4503 return from_simd<V>(I::simd_make512_u32(x1, x0)); 4504 } 4505 4506 template<typename V = Vec16xU32> 4507 BL_INLINE_NODEBUG V make512_u32(uint32_t x3, uint32_t x2, uint32_t x1, uint32_t x0) noexcept { 4508 return from_simd<V>(I::simd_make512_u32(x3, x2, x1, x0)); 4509 } 4510 4511 template<typename V = Vec16xU32> 4512 BL_INLINE_NODEBUG V make512_u32(uint32_t x7, uint32_t x6, uint32_t x5, uint32_t x4, 4513 uint32_t x3, uint32_t x2, uint32_t x1, uint32_t x0) noexcept { 4514 return from_simd<V>(I::simd_make512_u32(x7, x6, x5, x4, x3, x2, x1, x0)); 4515 } 4516 4517 template<typename V = Vec16xU32> 4518 BL_INLINE_NODEBUG V make512_u32(uint32_t x15, uint32_t x14, uint32_t x13, uint32_t x12, 4519 uint32_t x11, uint32_t x10, uint32_t x09, uint32_t x08, 4520 uint32_t x07, uint32_t x06, uint32_t x05, uint32_t x04, 4521 uint32_t x03, uint32_t x02, uint32_t x01, uint32_t x00) noexcept { 4522 return from_simd<V>( 4523 I::simd_make512_u32( 4524 x15, x14, x13, x12, 4525 x11, x10, x09, x08, 4526 x07, x06, x05, x04, 4527 x03, x02, x01, x00)); 4528 } 4529 4530 template<typename V = Vec8xI64> 4531 BL_INLINE_NODEBUG V make512_i64(int64_t x0) noexcept { 4532 return from_simd<V>(I::simd_make512_u64(uint64_t(x0))); 4533 } 4534 4535 template<typename V = Vec8xI64> 4536 BL_INLINE_NODEBUG V make512_i64(int64_t x1, int64_t x0) noexcept { 4537 return from_simd<V>(I::simd_make512_u64(uint64_t(x1), uint64_t(x0))); 4538 } 4539 4540 template<typename V = Vec8xI64> 4541 BL_INLINE_NODEBUG V make512_i64(int64_t x3, int64_t x2, int64_t x1, int64_t x0) noexcept { 4542 return from_simd<V>(I::simd_make512_u64(uint64_t(x3), uint64_t(x2), uint64_t(x1), uint64_t(x0))); 4543 } 4544 4545 template<typename V = Vec8xI64> 4546 BL_INLINE_NODEBUG V make512_i64(int64_t x7, int64_t x6, int64_t x5, int64_t x4, 4547 int64_t x3, int64_t x2, int64_t x1, int64_t x0) noexcept { 4548 return from_simd<V>( 4549 I::simd_make512_u64( 4550 uint64_t(x7), uint64_t(x6), uint64_t(x5), uint64_t(x4), 4551 uint64_t(x3), uint64_t(x2), uint64_t(x1), uint64_t(x0))); 4552 } 4553 4554 template<typename V = Vec8xU64> 4555 BL_INLINE_NODEBUG V make512_u64(uint64_t x0) noexcept { 4556 return from_simd<V>(I::simd_make512_u64(x0)); 4557 } 4558 4559 template<typename V = Vec8xU64> 4560 BL_INLINE_NODEBUG V make512_u64(uint64_t x1, uint64_t x0) noexcept { 4561 return from_simd<V>(I::simd_make512_u64(x1, x0)); 4562 } 4563 4564 template<typename V = Vec8xU64> 4565 BL_INLINE_NODEBUG V make512_u64(uint64_t x3, uint64_t x2, uint64_t x1, uint64_t x0) noexcept { 4566 return from_simd<V>(I::simd_make512_u64(x3, x2, x1, x0)); 4567 } 4568 4569 template<typename V = Vec8xU64> 4570 BL_INLINE_NODEBUG V make512_u64(uint64_t x7, uint64_t x6, uint64_t x5, uint64_t x4, 4571 uint64_t x3, uint64_t x2, uint64_t x1, uint64_t x0) noexcept { 4572 return from_simd<V>( 4573 I::simd_make512_u64( 4574 x7, x6, x5, x4, 4575 x3, x2, x1, x0)); 4576 } 4577 4578 template<typename V = Vec16xF32> 4579 BL_INLINE_NODEBUG V make512_f32(float x0) noexcept { 4580 return from_simd<V>(I::simd_make512_f32(x0)); 4581 } 4582 4583 template<typename V = Vec16xF32> 4584 BL_INLINE_NODEBUG V make512_f32(float x1, float x0) noexcept { 4585 return from_simd<V>(I::simd_make512_f32(x1, x0)); 4586 } 4587 4588 template<typename V = Vec16xF32> 4589 BL_INLINE_NODEBUG V make512_f32(float x3, float x2, float x1, float x0) noexcept { 4590 return from_simd<V>(I::simd_make512_f32(x3, x2, x1, x0)); 4591 } 4592 4593 template<typename V = Vec16xF32> 4594 BL_INLINE_NODEBUG V make512_f32(float x7, float x6, float x5, float x4, 4595 float x3, float x2, float x1, float x0) noexcept { 4596 return from_simd<V>(I::simd_make512_f32(x7, x6, x5, x4, x3, x2, x1, x0)); 4597 } 4598 4599 template<typename V = Vec16xF32> 4600 BL_INLINE_NODEBUG V make512_f32(float x15, float x14, float x13, float x12, 4601 float x11, float x10, float x09, float x08, 4602 float x07, float x06, float x05, float x04, 4603 float x03, float x02, float x01, float x00) noexcept { 4604 return from_simd<V>( 4605 I::simd_make512_f32( 4606 x15, x14, x13, x12, 4607 x11, x10, x09, x08, 4608 x07, x06, x05, x04, 4609 x03, x02, x01, x00)); 4610 } 4611 4612 template<typename V = Vec8xF64> 4613 BL_INLINE_NODEBUG V make512_f64(double x0) noexcept { 4614 return from_simd<V>(I::simd_make512_f64(x0)); 4615 } 4616 4617 template<typename V = Vec8xF64> 4618 BL_INLINE_NODEBUG V make512_f64(double x1, double x0) noexcept { 4619 return from_simd<V>(I::simd_make512_f64(x1, x0)); 4620 } 4621 4622 template<typename V = Vec8xF64> 4623 BL_INLINE_NODEBUG V make512_f64(double x3, double x2, double x1, double x0) noexcept { 4624 return from_simd<V>(I::simd_make512_f64(x3, x2, x1, x0)); 4625 } 4626 4627 template<typename V = Vec8xF64> 4628 BL_INLINE_NODEBUG V make512_f32(double x7, double x6, double x5, double x4, 4629 double x3, double x2, double x1, double x0) noexcept { 4630 return from_simd<V>(I::simd_make512_f64(x7, x6, x5, x4, x3, x2, x1, x0)); 4631 } 4632 #endif // BL_TARGET_OPT_AVX512 4633 4634 // SIMD - Public - Cast Vector <-> Scalar 4635 // ====================================== 4636 4637 template<typename V = Vec4xI32> BL_INLINE_NODEBUG V cast_from_i32(int32_t val) noexcept { return from_simd<V>(I::simd_cast_from_u32(uint32_t(val))); } 4638 template<typename V = Vec4xU32> BL_INLINE_NODEBUG V cast_from_u32(uint32_t val) noexcept { return from_simd<V>(I::simd_cast_from_u32(val)); } 4639 template<typename V = Vec2xI64> BL_INLINE_NODEBUG V cast_from_i64(int64_t val) noexcept { return from_simd<V>(I::simd_cast_from_u64(uint64_t(val))); } 4640 template<typename V = Vec2xU64> BL_INLINE_NODEBUG V cast_from_u64(uint64_t val) noexcept { return from_simd<V>(I::simd_cast_from_u64(val)); } 4641 template<typename V = Vec4xF32> BL_INLINE_NODEBUG V cast_from_f32(float val) noexcept { return from_simd<V>(I::simd_cast_from_f32(val)); } 4642 template<typename V = Vec2xF64> BL_INLINE_NODEBUG V cast_from_f64(double val) noexcept { return from_simd<V>(I::simd_cast_from_f64(val)); } 4643 4644 template<typename V> BL_INLINE_NODEBUG int32_t cast_to_i32(const V& src) noexcept { return int32_t(I::simd_cast_to_u32(simd_cast<__m128i>(src.v))); } 4645 template<typename V> BL_INLINE_NODEBUG uint32_t cast_to_u32(const V& src) noexcept { return I::simd_cast_to_u32(simd_cast<__m128i>(src.v)); } 4646 template<typename V> BL_INLINE_NODEBUG int64_t cast_to_i64(const V& src) noexcept { return int64_t(I::simd_cast_to_u64(simd_cast<__m128i>(src.v))); } 4647 template<typename V> BL_INLINE_NODEBUG uint64_t cast_to_u64(const V& src) noexcept { return I::simd_cast_to_u64(simd_cast<__m128i>(src.v)); } 4648 template<typename V> BL_INLINE_NODEBUG float cast_to_f32(const V& src) noexcept { return I::simd_cast_to_f32(simd_cast<__m128>(src.v)); } 4649 template<typename V> BL_INLINE_NODEBUG double cast_to_f64(const V& src) noexcept { return I::simd_cast_to_f64(simd_cast<__m128d>(src.v)); } 4650 4651 // SIMD - Public - Convert Vector <-> Vector 4652 // ========================================= 4653 4654 template<typename V> BL_INLINE_NODEBUG Vec<V::kW, float> cvt_i32_f32(const V& a) noexcept { return Vec<V::kW, float>{I::simd_cvt_i32_f32(simd_as_i(a.v))}; } 4655 template<typename V> BL_INLINE_NODEBUG Vec<V::kW, int32_t> cvt_f32_i32(const V& a) noexcept { return Vec<V::kW, int32_t>{I::simd_cvt_f32_i32(simd_as_f(a.v))}; } 4656 template<typename V> BL_INLINE_NODEBUG Vec<V::kW, int32_t> cvtt_f32_i32(const V& a) noexcept { return Vec<V::kW, int32_t>{I::simd_cvtt_f32_i32(simd_as_f(a.v))}; } 4657 4658 // SIMD - Public - Convert Vector <-> Scalar 4659 // ========================================= 4660 4661 BL_INLINE_NODEBUG Vec4xF32 cvt_f32_from_scalar_i32(int32_t val) noexcept { return Vec4xF32{I::simd_cvt_f32_from_scalar_i32(val)}; } 4662 BL_INLINE_NODEBUG Vec2xF64 cvt_f64_from_scalar_i32(int32_t val) noexcept { return Vec2xF64{I::simd_cvt_f64_from_scalar_i32(val)}; } 4663 4664 template<typename V> BL_INLINE_NODEBUG int32_t cvt_f32_to_scalar_i32(const V& src) noexcept { return I::simd_cvt_f32_to_scalar_i32(simd_cast<__m128>(src.v)); } 4665 template<typename V> BL_INLINE_NODEBUG int32_t cvt_f64_to_scalar_i32(const V& src) noexcept { return I::simd_cvt_f64_to_scalar_i32(simd_cast<__m128d>(src.v)); } 4666 template<typename V> BL_INLINE_NODEBUG int32_t cvtt_f32_to_scalar_i32(const V& src) noexcept { return I::simd_cvtt_f32_to_scalar_i32(simd_cast<__m128>(src.v)); } 4667 template<typename V> BL_INLINE_NODEBUG int32_t cvtt_f64_to_scalar_i32(const V& src) noexcept { return I::simd_cvtt_f64_to_scalar_i32(simd_cast<__m128d>(src.v)); } 4668 4669 #if BL_TARGET_ARCH_BITS >= 64 4670 BL_INLINE_NODEBUG Vec4xF32 cvt_f32_from_scalar_i64(int64_t val) noexcept { return Vec4xF32{I::simd_cvt_f32_from_scalar_i64(val)}; } 4671 BL_INLINE_NODEBUG Vec2xF64 cvt_f64_from_scalar_i64(int64_t val) noexcept { return Vec2xF64{I::simd_cvt_f64_from_scalar_i64(val)}; } 4672 4673 template<typename V> BL_INLINE_NODEBUG int64_t cvt_f32_to_scalar_i64(const V& src) noexcept { return I::simd_cvt_f32_to_scalar_i64(simd_cast<__m128>(src.v)); } 4674 template<typename V> BL_INLINE_NODEBUG int64_t cvt_f64_to_scalar_i64(const V& src) noexcept { return I::simd_cvt_f64_to_scalar_i64(simd_cast<__m128d>(src.v)); } 4675 template<typename V> BL_INLINE_NODEBUG int64_t cvtt_f32_to_scalar_i64(const V& src) noexcept { return I::simd_cvtt_f32_to_scalar_i64(simd_cast<__m128>(src.v)); } 4676 template<typename V> BL_INLINE_NODEBUG int64_t cvtt_f64_to_scalar_i64(const V& src) noexcept { return I::simd_cvtt_f64_to_scalar_i64(simd_cast<__m128d>(src.v)); } 4677 #endif 4678 4679 // SIMD - Public - Extract MSB 4680 // =========================== 4681 4682 template<typename T> BL_INLINE_NODEBUG uint32_t extract_sign_bits_i8(const Vec<16, T>& a) noexcept { 4683 return I::simd_extract_sign_bits_i8(simd_as_i(a.v)); 4684 } 4685 4686 template<typename T> BL_INLINE_NODEBUG uint32_t extract_sign_bits_i8(const Vec<16, T>& a, const Vec<16, T>& b) noexcept { 4687 return extract_sign_bits_i8(a) | (extract_sign_bits_i8(b) << 16); 4688 } 4689 4690 template<typename T> BL_INLINE_NODEBUG uint64_t extract_sign_bits_i8(const Vec<16, T>& a, const Vec<16, T>& b, const Vec<16, T>& c, const Vec<16, T>& d) noexcept { 4691 uint32_t i0 = extract_sign_bits_i8(a) | (extract_sign_bits_i8(b) << 16); 4692 uint32_t i1 = extract_sign_bits_i8(c) | (extract_sign_bits_i8(d) << 16); 4693 4694 return uint64_t(i0) | (uint64_t(i1) << 32); 4695 } 4696 4697 template<typename T> BL_INLINE_NODEBUG uint32_t extract_mask_bits_i8(const Vec<16, T>& a) noexcept { 4698 return extract_sign_bits_i8(a); 4699 } 4700 4701 template<typename T> BL_INLINE_NODEBUG uint32_t extract_mask_bits_i8(const Vec<16, T>& a, const Vec<16, T>& b) noexcept { 4702 return extract_sign_bits_i8(a, b); 4703 } 4704 4705 template<typename T> BL_INLINE_NODEBUG uint64_t extract_mask_bits_i8(const Vec<16, T>& a, const Vec<16, T>& b, const Vec<16, T>& c, const Vec<16, T>& d) noexcept { 4706 return extract_sign_bits_i8(a, b, c, d); 4707 } 4708 4709 #if defined(BL_TARGET_OPT_AVX2) 4710 template<typename T> BL_INLINE_NODEBUG uint32_t extract_sign_bits_i8(const Vec<32, T>& a) noexcept { 4711 return I::simd_extract_sign_bits_i8(simd_as_i(a.v)); 4712 } 4713 4714 template<typename T> BL_INLINE_NODEBUG uint64_t extract_sign_bits_i8(const Vec<32, T>& a, const Vec<32, T>& b) noexcept { 4715 return uint64_t(extract_sign_bits_i8(a)) | (uint64_t(extract_sign_bits_i8(b)) << 32); 4716 } 4717 4718 template<typename T> BL_INLINE_NODEBUG uint32_t extract_mask_bits_i8(const Vec<32, T>& a) noexcept { 4719 return extract_sign_bits_i8(a); 4720 } 4721 4722 template<typename T> BL_INLINE_NODEBUG uint64_t extract_mask_bits_i8(const Vec<32, T>& a, const Vec<32, T>& b) noexcept { 4723 return extract_sign_bits_i8(a, b); 4724 } 4725 #endif // BL_TARGET_OPT_AVX2 4726 4727 #if defined(BL_TARGET_OPT_AVX512) 4728 // NOTE: 64-byte vectors require 64-bit integer to store all most significant bits. 4729 template<typename T> BL_INLINE_NODEBUG uint64_t extract_sign_bits_i8(const Vec<64, T>& a) noexcept { 4730 return I::simd_extract_sign_bits_i8(simd_as_i(a.v)); 4731 } 4732 4733 template<typename T> BL_INLINE_NODEBUG uint64_t extract_mask_bits_i8(const Vec<64, T>& a) noexcept { 4734 return extract_sign_bits_i8(a); 4735 } 4736 #endif // BL_TARGET_OPT_AVX512 4737 4738 template<typename V> BL_INLINE_NODEBUG uint32_t extract_sign_bits_i32(const V& a) noexcept { return I::simd_extract_sign_bits_i32(simd_as_i(a.v)); } 4739 template<typename V> BL_INLINE_NODEBUG uint32_t extract_sign_bits_i64(const V& a) noexcept { return I::simd_extract_sign_bits_i64(simd_as_i(a.v)); } 4740 4741 template<typename V> BL_INLINE_NODEBUG uint32_t extract_mask_bits_i32(const V& a) noexcept { return extract_sign_bits_i32(a); } 4742 template<typename V> BL_INLINE_NODEBUG uint32_t extract_mask_bits_i64(const V& a) noexcept { return extract_sign_bits_i64(a); } 4743 4744 // SIMD - Public - Load & Store Operations (128-bit) 4745 // ================================================= 4746 4747 template<typename V> BL_INLINE_NODEBUG V load_8(const void* src) noexcept { return V{I::simd_load_8<typename V::SimdType>(src)}; } 4748 template<typename V> BL_INLINE_NODEBUG V loada_16(const void* src) noexcept { return V{I::simd_loada_16<typename V::SimdType>(src)}; } 4749 template<typename V> BL_INLINE_NODEBUG V loadu_16(const void* src) noexcept { return V{I::simd_loadu_16<typename V::SimdType>(src)}; } 4750 template<typename V> BL_INLINE_NODEBUG V loada_32(const void* src) noexcept { return V{I::simd_loada_32<typename V::SimdType>(src)}; } 4751 template<typename V> BL_INLINE_NODEBUG V loadu_32(const void* src) noexcept { return V{I::simd_loadu_32<typename V::SimdType>(src)}; } 4752 template<typename V> BL_INLINE_NODEBUG V loada_64(const void* src) noexcept { return V{I::simd_loada_64<typename V::SimdType>(src)}; } 4753 template<typename V> BL_INLINE_NODEBUG V loadu_64(const void* src) noexcept { return V{I::simd_loadu_64<typename V::SimdType>(src)}; } 4754 template<typename V> BL_INLINE_NODEBUG V loada_128(const void* src) noexcept { return V{I::simd_loada_128<typename V::SimdType>(src)}; } 4755 template<typename V> BL_INLINE_NODEBUG V loadu_128(const void* src) noexcept { return V{I::simd_loadu_128<typename V::SimdType>(src)}; } 4756 4757 template<typename T> BL_INLINE_NODEBUG Vec<16, T> loadl_64(const Vec<16, T>& dst, const void* src) noexcept { return Vec<16, T>{I::simd_loadl_64(dst.v, src)}; } 4758 template<typename T> BL_INLINE_NODEBUG Vec<16, T> loadh_64(const Vec<16, T>& dst, const void* src) noexcept { return Vec<16, T>{I::simd_loadh_64(dst.v, src)}; } 4759 4760 template<typename V> BL_INLINE_NODEBUG void store_8(void* dst, const V& src) noexcept { I::simd_store_8(dst, src.v); } 4761 template<typename V> BL_INLINE_NODEBUG void storea_16(void* dst, const V& src) noexcept { I::simd_storea_16(dst, src.v); } 4762 template<typename V> BL_INLINE_NODEBUG void storeu_16(void* dst, const V& src) noexcept { I::simd_storeu_16(dst, src.v); } 4763 template<typename V> BL_INLINE_NODEBUG void storea_32(void* dst, const V& src) noexcept { I::simd_storea_32(dst, src.v); } 4764 template<typename V> BL_INLINE_NODEBUG void storeu_32(void* dst, const V& src) noexcept { I::simd_storeu_32(dst, src.v); } 4765 template<typename V> BL_INLINE_NODEBUG void storea_64(void* dst, const V& src) noexcept { I::simd_storea_64(dst, src.v); } 4766 template<typename V> BL_INLINE_NODEBUG void storeu_64(void* dst, const V& src) noexcept { I::simd_storeu_64(dst, src.v); } 4767 template<typename V> BL_INLINE_NODEBUG void storeh_64(void* dst, const V& src) noexcept { I::simd_storeh_64(dst, src.v); } 4768 template<typename V> BL_INLINE_NODEBUG void storea_128(void* dst, const V& src) noexcept { I::simd_storea_128(dst, src.v); } 4769 template<typename V> BL_INLINE_NODEBUG void storeu_128(void* dst, const V& src) noexcept { I::simd_storeu_128(dst, src.v); } 4770 4771 #if defined(BL_TARGET_OPT_SSE4_1) 4772 template<typename V> BL_INLINE_NODEBUG V loada_16_i8_i64(const void* src) noexcept { return from_simd<V>(I::simd_loada_16_i8_i64(src)); } 4773 template<typename V> BL_INLINE_NODEBUG V loadu_16_i8_i64(const void* src) noexcept { return from_simd<V>(I::simd_loadu_16_i8_i64(src)); } 4774 template<typename V> BL_INLINE_NODEBUG V loada_16_u8_u64(const void* src) noexcept { return from_simd<V>(I::simd_loada_16_u8_u64(src)); } 4775 template<typename V> BL_INLINE_NODEBUG V loadu_16_u8_u64(const void* src) noexcept { return from_simd<V>(I::simd_loadu_16_u8_u64(src)); } 4776 #endif // BL_TARGET_OPT_SSE4_1 4777 4778 template<typename V> BL_INLINE_NODEBUG V loada_32_i8_i32(const void* src) noexcept { return from_simd<V>(I::simd_loada_32_i8_i32(src)); } 4779 template<typename V> BL_INLINE_NODEBUG V loadu_32_i8_i32(const void* src) noexcept { return from_simd<V>(I::simd_loadu_32_i8_i32(src)); } 4780 template<typename V> BL_INLINE_NODEBUG V loada_32_u8_u32(const void* src) noexcept { return from_simd<V>(I::simd_loada_32_u8_u32(src)); } 4781 template<typename V> BL_INLINE_NODEBUG V loadu_32_u8_u32(const void* src) noexcept { return from_simd<V>(I::simd_loadu_32_u8_u32(src)); } 4782 template<typename V> BL_INLINE_NODEBUG V loada_64_i8_i16(const void* src) noexcept { return from_simd<V>(I::simd_loada_64_i8_i16(src)); } 4783 template<typename V> BL_INLINE_NODEBUG V loadu_64_i8_i16(const void* src) noexcept { return from_simd<V>(I::simd_loadu_64_i8_i16(src)); } 4784 template<typename V> BL_INLINE_NODEBUG V loada_64_u8_u16(const void* src) noexcept { return from_simd<V>(I::simd_loada_64_u8_u16(src)); } 4785 template<typename V> BL_INLINE_NODEBUG V loadu_64_u8_u16(const void* src) noexcept { return from_simd<V>(I::simd_loadu_64_u8_u16(src)); } 4786 4787 // SIMD - Public - Load & Store Operations (256-bit) 4788 // ================================================= 4789 4790 #if defined(BL_TARGET_OPT_AVX) 4791 template<typename V> BL_INLINE_NODEBUG V loada_256(const void* src) noexcept { return V{I::simd_loada_256<typename V::SimdType>(src)}; } 4792 template<typename V> BL_INLINE_NODEBUG V loadu_256(const void* src) noexcept { return V{I::simd_loadu_256<typename V::SimdType>(src)}; } 4793 template<typename V> BL_INLINE_NODEBUG void storea_256(void* dst, const V& src) noexcept { I::simd_storea_256(dst, src.v); } 4794 template<typename V> BL_INLINE_NODEBUG void storeu_256(void* dst, const V& src) noexcept { I::simd_storeu_256(dst, src.v); } 4795 4796 template<typename V> BL_INLINE_NODEBUG V load_broadcast_u32(const void* src) noexcept { return from_simd<V>(I::simd_load_broadcast_u32<V::kW>(src)); } 4797 template<typename V> BL_INLINE_NODEBUG V load_broadcast_u64(const void* src) noexcept { return from_simd<V>(I::simd_load_broadcast_u64<V::kW>(src)); } 4798 template<typename V> BL_INLINE_NODEBUG V load_broadcast_f32(const void* src) noexcept { return from_simd<V>(I::simd_load_broadcast_f32<V::kW>(src)); } 4799 template<typename V> BL_INLINE_NODEBUG V load_broadcast_f64(const void* src) noexcept { return from_simd<V>(I::simd_load_broadcast_f64<V::kW>(src)); } 4800 4801 template<typename V> BL_INLINE_NODEBUG V load_broadcast_4xi32(const void* src) noexcept { return from_simd<V>(I::simd_load_broadcast_4xi32<V::kW>(src)); } 4802 template<typename V> BL_INLINE_NODEBUG V load_broadcast_2xi64(const void* src) noexcept { return from_simd<V>(I::simd_load_broadcast_2xi64<V::kW>(src)); } 4803 template<typename V> BL_INLINE_NODEBUG V load_broadcast_f32x4(const void* src) noexcept { return from_simd<V>(I::simd_load_broadcast_f32x4<V::kW>(src)); } 4804 template<typename V> BL_INLINE_NODEBUG V load_broadcast_f64x2(const void* src) noexcept { return from_simd<V>(I::simd_load_broadcast_f64x2<V::kW>(src)); } 4805 #endif // BL_TARGET_OPT_AVX 4806 4807 #if defined(BL_TARGET_OPT_AVX2) 4808 template<typename V> BL_INLINE_NODEBUG V loada_32_i8_i64(const void* src) noexcept { return from_simd<V>(I::simd_loada_32_i8_i64(src)); } 4809 template<typename V> BL_INLINE_NODEBUG V loadu_32_i8_i64(const void* src) noexcept { return from_simd<V>(I::simd_loadu_32_i8_i64(src)); } 4810 template<typename V> BL_INLINE_NODEBUG V loada_32_u8_u64(const void* src) noexcept { return from_simd<V>(I::simd_loada_32_u8_u64(src)); } 4811 template<typename V> BL_INLINE_NODEBUG V loadu_32_u8_u64(const void* src) noexcept { return from_simd<V>(I::simd_loadu_32_u8_u64(src)); } 4812 template<typename V> BL_INLINE_NODEBUG V loada_64_i8_i32(const void* src) noexcept { return from_simd<V>(I::simd_loada_64_i8_i32(src)); } 4813 template<typename V> BL_INLINE_NODEBUG V loadu_64_i8_i32(const void* src) noexcept { return from_simd<V>(I::simd_loadu_64_i8_i32(src)); } 4814 template<typename V> BL_INLINE_NODEBUG V loada_64_u8_u32(const void* src) noexcept { return from_simd<V>(I::simd_loada_64_u8_u32(src)); } 4815 template<typename V> BL_INLINE_NODEBUG V loadu_64_u8_u32(const void* src) noexcept { return from_simd<V>(I::simd_loadu_64_u8_u32(src)); } 4816 template<typename V> BL_INLINE_NODEBUG V loada_128_i8_i16(const void* src) noexcept { return from_simd<V>(I::simd_loada_128_i8_i16(src)); } 4817 template<typename V> BL_INLINE_NODEBUG V loadu_128_i8_i16(const void* src) noexcept { return from_simd<V>(I::simd_loadu_128_i8_i16(src)); } 4818 template<typename V> BL_INLINE_NODEBUG V loada_128_u8_u16(const void* src) noexcept { return from_simd<V>(I::simd_loada_128_u8_u16(src)); } 4819 template<typename V> BL_INLINE_NODEBUG V loadu_128_u8_u16(const void* src) noexcept { return from_simd<V>(I::simd_loadu_128_u8_u16(src)); } 4820 #endif // BL_TARGET_OPT_AVX2 4821 4822 // SIMD - Public - Load & Store Operations (512-bit) 4823 // ================================================= 4824 4825 #if defined(BL_TARGET_OPT_AVX512) 4826 template<typename V> BL_INLINE_NODEBUG V loada_512(const void* src) noexcept { return V{I::simd_loada_512<typename V::SimdType>(src)}; } 4827 template<typename V> BL_INLINE_NODEBUG V loadu_512(const void* src) noexcept { return V{I::simd_loadu_512<typename V::SimdType>(src)}; } 4828 template<typename V> BL_INLINE_NODEBUG void storea_512(void* dst, const V& src) noexcept { I::simd_storea_512(dst, src.v); } 4829 template<typename V> BL_INLINE_NODEBUG void storeu_512(void* dst, const V& src) noexcept { I::simd_storeu_512(dst, src.v); } 4830 4831 template<typename V> BL_INLINE_NODEBUG V loada_64_i8_i64(const void* src) noexcept { return from_simd<V>(I::simd_loada_64_i8_i64(src)); } 4832 template<typename V> BL_INLINE_NODEBUG V loadu_64_i8_i64(const void* src) noexcept { return from_simd<V>(I::simd_loadu_64_i8_i64(src)); } 4833 template<typename V> BL_INLINE_NODEBUG V loada_64_u8_u64(const void* src) noexcept { return from_simd<V>(I::simd_loada_64_u8_u64(src)); } 4834 template<typename V> BL_INLINE_NODEBUG V loadu_64_u8_u64(const void* src) noexcept { return from_simd<V>(I::simd_loadu_64_u8_u64(src)); } 4835 template<typename V> BL_INLINE_NODEBUG V loada_128_i8_i32(const void* src) noexcept { return from_simd<V>(I::simd_loada_128_i8_i32(src)); } 4836 template<typename V> BL_INLINE_NODEBUG V loadu_128_i8_i32(const void* src) noexcept { return from_simd<V>(I::simd_loadu_128_i8_i32(src)); } 4837 template<typename V> BL_INLINE_NODEBUG V loada_128_u8_u32(const void* src) noexcept { return from_simd<V>(I::simd_loada_128_u8_u32(src)); } 4838 template<typename V> BL_INLINE_NODEBUG V loadu_128_u8_u32(const void* src) noexcept { return from_simd<V>(I::simd_loadu_128_u8_u32(src)); } 4839 template<typename V> BL_INLINE_NODEBUG V loada_256_i8_i16(const void* src) noexcept { return from_simd<V>(I::simd_loada_256_i8_i16(src)); } 4840 template<typename V> BL_INLINE_NODEBUG V loadu_256_i8_i16(const void* src) noexcept { return from_simd<V>(I::simd_loadu_256_i8_i16(src)); } 4841 template<typename V> BL_INLINE_NODEBUG V loada_256_u8_u16(const void* src) noexcept { return from_simd<V>(I::simd_loada_256_u8_u16(src)); } 4842 template<typename V> BL_INLINE_NODEBUG V loadu_256_u8_u16(const void* src) noexcept { return from_simd<V>(I::simd_loadu_256_u8_u16(src)); } 4843 #endif // BL_TARGET_OPT_AVX512 4844 4845 // SIMD - Public - Load & Store Operations (Native) 4846 // ================================================ 4847 4848 template<typename V> BL_INLINE_NODEBUG V loada(const void* src) noexcept { return V{I::simd_loada<typename V::SimdType>(src)}; } 4849 template<typename V> BL_INLINE_NODEBUG V loadu(const void* src) noexcept { return V{I::simd_loadu<typename V::SimdType>(src)}; } 4850 4851 template<typename V> BL_INLINE_NODEBUG void storea(void* dst, const V& src) noexcept { I::simd_storea(dst, src.v); } 4852 template<typename V> BL_INLINE_NODEBUG void storeu(void* dst, const V& src) noexcept { I::simd_storeu(dst, src.v); } 4853 4854 // SIMD - Public - Shuffle & Permute 4855 // ================================= 4856 4857 #if defined(BL_SIMD_FEATURE_SWIZZLEV_U8) 4858 template<typename V, typename W> 4859 BL_INLINE_NODEBUG V swizzlev_u8(const V& a, const W& b) noexcept { return V{I::simd_swizzlev_u8(a.v, b.v)}; } 4860 #endif // BL_SIMD_FEATURE_SWIZZLEV_U8 4861 4862 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A, typename V> 4863 BL_INLINE_NODEBUG V swizzle_u16(const V& a) noexcept { return V{I::simd_swizzle_u16<D, C, B, A>(a.v)}; } 4864 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A, typename V> 4865 BL_INLINE_NODEBUG V swizzle_lo_u16(const V& a) noexcept { return V{I::simd_swizzle_lo_u16<D, C, B, A>(a.v)}; } 4866 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A, typename V> 4867 BL_INLINE_NODEBUG V swizzle_hi_u16(const V& a) noexcept { return V{I::simd_swizzle_hi_u16<D, C, B, A>(a.v)}; } 4868 4869 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A, typename V> 4870 BL_INLINE_NODEBUG V swizzle_f32(const V& a) noexcept { return V{I::simd_swizzle_f32<D, C, B, A>(a.v)}; } 4871 template<uint8_t B, uint8_t A, typename V> 4872 BL_INLINE_NODEBUG V swizzle_f64(const V& a) noexcept { return V{I::simd_swizzle_f64<B, A>(a.v)}; } 4873 4874 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A, typename V> 4875 BL_INLINE_NODEBUG V swizzle_u32(const V& a) noexcept { return V{I::simd_swizzle_u32<D, C, B, A>(a.v)}; } 4876 template<uint8_t B, uint8_t A, typename V> 4877 BL_INLINE_NODEBUG V swizzle_u64(const V& a) noexcept { return V{I::simd_swizzle_u64<B, A>(a.v)}; } 4878 4879 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A, typename V> 4880 BL_INLINE_NODEBUG V shuffle_f32(const V& lo, const V& hi) noexcept { return V{I::simd_shuffle_f32<D, C, B, A>(lo.v, hi.v)}; } 4881 template<uint8_t B, uint8_t A, typename V> 4882 BL_INLINE_NODEBUG V shuffle_f64(const V& lo, const V& hi) noexcept { return V{I::simd_shuffle_f64<B, A>(lo.v, hi.v)}; } 4883 4884 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A, typename V> 4885 BL_INLINE_NODEBUG V shuffle_u32(const V& lo, const V& hi) noexcept { return V{I::simd_shuffle_u32<D, C, B, A>(lo.v, hi.v)}; } 4886 template<uint8_t B, uint8_t A, typename V> 4887 BL_INLINE_NODEBUG V shuffle_u64(const V& lo, const V& hi) noexcept { return V{I::simd_shuffle_u64<B, A>(lo.v, hi.v)}; } 4888 4889 #if defined(BL_TARGET_OPT_AVX2) 4890 template<uint8_t D, uint8_t C, uint8_t B, uint8_t A, typename V> 4891 BL_INLINE_NODEBUG V permute_i64(const V& a) noexcept { return from_simd<V>(I::simd_permute_u64<D, C, B, A>(simd_as_i(a.v))); } 4892 4893 template<uint8_t B, uint8_t A, typename V> 4894 BL_INLINE_NODEBUG V permute_i128(const V& a) noexcept { return from_simd<V>(I::simd_permute_u128<B, A>(simd_cast<__m256i>(a.v))); } 4895 template<uint8_t B, uint8_t A, typename V> 4896 BL_INLINE_NODEBUG V permute_i128(const V& a, const V& b) noexcept { return from_simd<V>(I::simd_permute_u128<B, A>(simd_cast<__m256i>(a.v), simd_cast<__m256i>(b.v))); } 4897 4898 template<typename V> 4899 BL_INLINE_NODEBUG V interleave_i128(const V& a, const V& b) noexcept { return from_simd<V>(I::simd_interleave_u128(simd_cast<__m128i>(a.v), simd_cast<__m128i>(b.v))); } 4900 template<typename V, typename W> 4901 BL_INLINE_NODEBUG V interleave_i128(const W& a, const W& b) noexcept { return from_simd<V>(I::simd_interleave_u128(simd_cast<__m128i>(a.v), simd_cast<__m128i>(b.v))); } 4902 #endif 4903 4904 template<typename V, typename W> BL_INLINE_NODEBUG V broadcast_u8(const W& a) noexcept { return V{I::simd_broadcast_u8<V::kW>(simd_cast<__m128i>(a.v))}; } 4905 template<typename V, typename W> BL_INLINE_NODEBUG V broadcast_u16(const W& a) noexcept { return V{I::simd_broadcast_u16<V::kW>(simd_cast<__m128i>(a.v))}; } 4906 template<typename V, typename W> BL_INLINE_NODEBUG V broadcast_u32(const W& a) noexcept { return V{I::simd_broadcast_u32<V::kW>(simd_cast<__m128i>(a.v))}; } 4907 template<typename V, typename W> BL_INLINE_NODEBUG V broadcast_u64(const W& a) noexcept { return V{I::simd_broadcast_u64<V::kW>(simd_cast<__m128i>(a.v))}; } 4908 template<typename V, typename W> BL_INLINE_NODEBUG V broadcast_f32(const W& a) noexcept { return V{I::simd_broadcast_f32<V::kW>(simd_cast<__m128>(a.v))}; } 4909 template<typename V, typename W> BL_INLINE_NODEBUG V broadcast_f64(const W& a) noexcept { return V{I::simd_broadcast_f64<V::kW>(simd_cast<__m128d>(a.v))}; } 4910 4911 template<typename V> BL_INLINE_NODEBUG V dup_lo_u32(const V& a) noexcept { return V{I::simd_dup_lo_u32(a.v)}; } 4912 template<typename V> BL_INLINE_NODEBUG V dup_hi_u32(const V& a) noexcept { return V{I::simd_dup_hi_u32(a.v)}; } 4913 template<typename V> BL_INLINE_NODEBUG V dup_lo_u64(const V& a) noexcept { return V{I::simd_dup_lo_u64(a.v)}; } 4914 template<typename V> BL_INLINE_NODEBUG V dup_hi_u64(const V& a) noexcept { return V{I::simd_dup_hi_u64(a.v)}; } 4915 4916 template<typename V> BL_INLINE_NODEBUG V dup_lo_f32(const V& a) noexcept { return V{I::simd_dup_lo_f32(a.v)}; } 4917 template<typename V> BL_INLINE_NODEBUG V dup_hi_f32(const V& a) noexcept { return V{I::simd_dup_hi_f32(a.v)}; } 4918 template<typename V> BL_INLINE_NODEBUG V dup_lo_f32x2(const V& a) noexcept { return V{I::simd_dup_lo_f32x2(a.v)}; } 4919 template<typename V> BL_INLINE_NODEBUG V dup_hi_f32x2(const V& a) noexcept { return V{I::simd_dup_hi_f32x2(a.v)}; } 4920 template<typename V> BL_INLINE_NODEBUG V dup_lo_f64(const V& a) noexcept { return V{I::simd_dup_lo_f64(a.v)}; } 4921 template<typename V> BL_INLINE_NODEBUG V dup_hi_f64(const V& a) noexcept { return V{I::simd_dup_hi_f64(a.v)}; } 4922 4923 template<typename V> BL_INLINE_NODEBUG V swap_u32(const V& a) noexcept { return V{I::simd_swap_u32(a.v)}; } 4924 template<typename V> BL_INLINE_NODEBUG V swap_u64(const V& a) noexcept { return V{I::simd_swap_u64(a.v)}; } 4925 template<typename V> BL_INLINE_NODEBUG V swap_f32(const V& a) noexcept { return V{I::simd_swap_f32(a.v)}; } 4926 template<typename V> BL_INLINE_NODEBUG V swap_f64(const V& a) noexcept { return V{I::simd_swap_f64(a.v)}; } 4927 4928 template<typename V> BL_INLINE_NODEBUG V interleave_lo_u8(const V& a, const V& b) noexcept { return V{I::simd_interleave_lo_u8(a.v, b.v)}; } 4929 template<typename V> BL_INLINE_NODEBUG V interleave_hi_u8(const V& a, const V& b) noexcept { return V{I::simd_interleave_hi_u8(a.v, b.v)}; } 4930 template<typename V> BL_INLINE_NODEBUG V interleave_lo_u16(const V& a, const V& b) noexcept { return V{I::simd_interleave_lo_u16(a.v, b.v)}; } 4931 template<typename V> BL_INLINE_NODEBUG V interleave_hi_u16(const V& a, const V& b) noexcept { return V{I::simd_interleave_hi_u16(a.v, b.v)}; } 4932 template<typename V> BL_INLINE_NODEBUG V interleave_lo_u32(const V& a, const V& b) noexcept { return V{I::simd_interleave_lo_u32(a.v, b.v)}; } 4933 template<typename V> BL_INLINE_NODEBUG V interleave_hi_u32(const V& a, const V& b) noexcept { return V{I::simd_interleave_hi_u32(a.v, b.v)}; } 4934 template<typename V> BL_INLINE_NODEBUG V interleave_lo_u64(const V& a, const V& b) noexcept { return V{I::simd_interleave_lo_u64(a.v, b.v)}; } 4935 template<typename V> BL_INLINE_NODEBUG V interleave_hi_u64(const V& a, const V& b) noexcept { return V{I::simd_interleave_hi_u64(a.v, b.v)}; } 4936 4937 template<typename V> BL_INLINE_NODEBUG V interleave_lo_f32(const V& a, const V& b) noexcept { return V{I::simd_interleave_lo_f32(a.v, b.v)}; } 4938 template<typename V> BL_INLINE_NODEBUG V interleave_hi_f32(const V& a, const V& b) noexcept { return V{I::simd_interleave_hi_f32(a.v, b.v)}; } 4939 template<typename V> BL_INLINE_NODEBUG V interleave_lo_f64(const V& a, const V& b) noexcept { return V{I::simd_interleave_lo_f64(a.v, b.v)}; } 4940 template<typename V> BL_INLINE_NODEBUG V interleave_hi_f64(const V& a, const V& b) noexcept { return V{I::simd_interleave_hi_f64(a.v, b.v)}; } 4941 4942 template<int kNumBytes, typename V> 4943 BL_INLINE_NODEBUG V alignr_u128(const V& a, const V& b) noexcept { return V{I::simd_alignr_u128<kNumBytes>(a.v, b.v)}; } 4944 4945 // SIMD - Public - Integer Packing & Unpacking 4946 // =========================================== 4947 4948 template<typename V> BL_INLINE_NODEBUG V packs_128_i16_i8(const V& a) noexcept { return V{I::simd_packs_128_i16_i8(a.v)}; } 4949 template<typename V> BL_INLINE_NODEBUG V packs_128_i16_u8(const V& a) noexcept { return V{I::simd_packs_128_i16_u8(a.v)}; } 4950 template<typename V> BL_INLINE_NODEBUG V packz_128_u16_u8(const V& a) noexcept { return V{I::simd_packz_128_u16_u8(a.v)}; } 4951 template<typename V> BL_INLINE_NODEBUG V packs_128_i32_i8(const V& a) noexcept { return V{I::simd_packs_128_i32_i8(a.v)}; } 4952 template<typename V> BL_INLINE_NODEBUG V packs_128_i32_u8(const V& a) noexcept { return V{I::simd_packs_128_i32_u8(a.v)}; } 4953 template<typename V> BL_INLINE_NODEBUG V packs_128_i32_i16(const V& a) noexcept { return V{I::simd_packs_128_i32_i16(a.v)}; } 4954 template<typename V> BL_INLINE_NODEBUG V packs_128_i32_u16(const V& a) noexcept { return V{I::simd_packs_128_i32_u16(a.v)}; } 4955 template<typename V> BL_INLINE_NODEBUG V packz_128_u32_u8(const V& a) noexcept { return V{I::simd_packz_128_u32_u8(a.v)}; } 4956 template<typename V> BL_INLINE_NODEBUG V packz_128_u32_u16(const V& a) noexcept { return V{I::simd_packz_128_u32_u16(a.v)}; } 4957 4958 template<typename V> BL_INLINE_NODEBUG V packs_128_i16_i8(const V& a, const V& b) noexcept { return V{I::simd_packs_128_i16_i8(a.v, b.v)}; } 4959 template<typename V> BL_INLINE_NODEBUG V packs_128_i16_u8(const V& a, const V& b) noexcept { return V{I::simd_packs_128_i16_u8(a.v, b.v)}; } 4960 template<typename V> BL_INLINE_NODEBUG V packz_128_u16_u8(const V& a, const V& b) noexcept { return V{I::simd_packz_128_u16_u8(a.v, b.v)}; } 4961 template<typename V> BL_INLINE_NODEBUG V packs_128_i32_i8(const V& a, const V& b) noexcept { return V{I::simd_packs_128_i32_i8(a.v, b.v)}; } 4962 template<typename V> BL_INLINE_NODEBUG V packs_128_i32_u8(const V& a, const V& b) noexcept { return V{I::simd_packs_128_i32_u8(a.v, b.v)}; } 4963 template<typename V> BL_INLINE_NODEBUG V packs_128_i32_i16(const V& a, const V& b) noexcept { return V{I::simd_packs_128_i32_i16(a.v, b.v)}; } 4964 template<typename V> BL_INLINE_NODEBUG V packs_128_i32_u16(const V& a, const V& b) noexcept { return V{I::simd_packs_128_i32_u16(a.v, b.v)}; } 4965 template<typename V> BL_INLINE_NODEBUG V packz_128_u32_u8(const V& a, const V& b) noexcept { return V{I::simd_packz_128_u32_u8(a.v, b.v)}; } 4966 template<typename V> BL_INLINE_NODEBUG V packz_128_u32_u16(const V& a, const V& b) noexcept { return V{I::simd_packz_128_u32_u16(a.v, b.v)}; } 4967 4968 template<typename V> BL_INLINE_NODEBUG V packs_128_i32_i8(const V& a, const V& b, const V& c, const V& d) noexcept { return V{I::simd_packs_128_i32_i8(a.v, b.v, c.v, d.v)}; } 4969 template<typename V> BL_INLINE_NODEBUG V packs_128_i32_u8(const V& a, const V& b, const V& c, const V& d) noexcept { return V{I::simd_packs_128_i32_u8(a.v, b.v, c.v, d.v)}; } 4970 template<typename V> BL_INLINE_NODEBUG V packz_128_u32_u8(const V& a, const V& b, const V& c, const V& d) noexcept { return V{I::simd_packz_128_u32_u8(a.v, b.v, c.v, d.v)}; } 4971 4972 template<typename V> BL_INLINE_NODEBUG V unpack_lo64_i8_i16(const V& a) noexcept { return V{I::simd_unpack_lo64_i8_i16(a.v)}; } 4973 template<typename V> BL_INLINE_NODEBUG V unpack_lo64_u8_u16(const V& a) noexcept { return V{I::simd_unpack_lo64_u8_u16(a.v)}; } 4974 template<typename V> BL_INLINE_NODEBUG V unpack_lo64_i16_i32(const V& a) noexcept { return V{I::simd_unpack_lo64_i16_i32(a.v)}; } 4975 template<typename V> BL_INLINE_NODEBUG V unpack_lo64_u16_u32(const V& a) noexcept { return V{I::simd_unpack_lo64_u16_u32(a.v)}; } 4976 template<typename V> BL_INLINE_NODEBUG V unpack_lo64_i32_i64(const V& a) noexcept { return V{I::simd_unpack_lo64_i32_i64(a.v)}; } 4977 template<typename V> BL_INLINE_NODEBUG V unpack_lo64_u32_u64(const V& a) noexcept { return V{I::simd_unpack_lo64_u32_u64(a.v)}; } 4978 template<typename V> BL_INLINE_NODEBUG V unpack_lo32_i8_i32(const V& a) noexcept { return V{I::simd_unpack_lo32_i8_i32(a.v)}; } 4979 template<typename V> BL_INLINE_NODEBUG V unpack_lo32_u8_u32(const V& a) noexcept { return V{I::simd_unpack_lo32_u8_u32(a.v)}; } 4980 4981 template<typename V> BL_INLINE_NODEBUG V unpack_hi64_i8_i16(const V& a) noexcept { return V{I::simd_unpack_hi64_i8_i16(a.v)}; } 4982 template<typename V> BL_INLINE_NODEBUG V unpack_hi64_u8_u16(const V& a) noexcept { return V{I::simd_unpack_hi64_u8_u16(a.v)}; } 4983 template<typename V> BL_INLINE_NODEBUG V unpack_hi64_i16_i32(const V& a) noexcept { return V{I::simd_unpack_hi64_i16_i32(a.v)}; } 4984 template<typename V> BL_INLINE_NODEBUG V unpack_hi64_u16_u32(const V& a) noexcept { return V{I::simd_unpack_hi64_u16_u32(a.v)}; } 4985 template<typename V> BL_INLINE_NODEBUG V unpack_hi64_i32_i64(const V& a) noexcept { return V{I::simd_unpack_hi64_i32_i64(a.v)}; } 4986 template<typename V> BL_INLINE_NODEBUG V unpack_hi64_u32_u64(const V& a) noexcept { return V{I::simd_unpack_hi64_u32_u64(a.v)}; } 4987 4988 #if defined(BL_SIMD_FEATURE_MOVW) 4989 template<typename V> BL_INLINE_NODEBUG V movw_i8_i16(const V& a) noexcept { return V{I::simd_movw_i8_i16(a.v)}; } 4990 template<typename V> BL_INLINE_NODEBUG V movw_i8_i32(const V& a) noexcept { return V{I::simd_movw_i8_i32(a.v)}; } 4991 template<typename V> BL_INLINE_NODEBUG V movw_i8_i64(const V& a) noexcept { return V{I::simd_movw_i8_i64(a.v)}; } 4992 template<typename V> BL_INLINE_NODEBUG V movw_i16_i32(const V& a) noexcept { return V{I::simd_movw_i16_i32(a.v)}; } 4993 template<typename V> BL_INLINE_NODEBUG V movw_i16_i64(const V& a) noexcept { return V{I::simd_movw_i16_i64(a.v)}; } 4994 template<typename V> BL_INLINE_NODEBUG V movw_i32_i64(const V& a) noexcept { return V{I::simd_movw_i32_i64(a.v)}; } 4995 4996 template<typename V> BL_INLINE_NODEBUG V movw_u8_u16(const V& a) noexcept { return V{I::simd_movw_u8_u16(a.v)}; } 4997 template<typename V> BL_INLINE_NODEBUG V movw_u8_u32(const V& a) noexcept { return V{I::simd_movw_u8_u32(a.v)}; } 4998 template<typename V> BL_INLINE_NODEBUG V movw_u8_u64(const V& a) noexcept { return V{I::simd_movw_u8_u64(a.v)}; } 4999 template<typename V> BL_INLINE_NODEBUG V movw_u16_u32(const V& a) noexcept { return V{I::simd_movw_u16_u32(a.v)}; } 5000 template<typename V> BL_INLINE_NODEBUG V movw_u16_u64(const V& a) noexcept { return V{I::simd_movw_u16_u64(a.v)}; } 5001 template<typename V> BL_INLINE_NODEBUG V movw_u32_u64(const V& a) noexcept { return V{I::simd_movw_u32_u64(a.v)}; } 5002 #endif // BL_SIMD_FEATURE_MOVW 5003 5004 // SIMD - Public - Arithmetic & Logical Operations 5005 // =============================================== 5006 5007 #if defined(BL_TARGET_OPT_AVX512) 5008 template<uint8_t kPred, size_t W, typename T> 5009 BL_INLINE_NODEBUG Vec<W, T> simd_ternlog(const Vec<W, T>& a, const Vec<W, T>& b, const Vec<W, T>& c) noexcept { return Vec<W, T>{I::simd_ternlog(a.v, b.v, c.v)}; } 5010 #endif // BL_TARGET_OPT_AVX512 5011 5012 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> not_(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_not(a.v)); } 5013 5014 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> and_(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return from_simd<Vec<W, T>>(I::simd_and(a.v, b.v)); } 5015 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> andnot(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return from_simd<Vec<W, T>>(I::simd_andnot(a.v, b.v)); } 5016 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> or_(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return from_simd<Vec<W, T>>(I::simd_or(a.v, b.v)); } 5017 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> xor_(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return from_simd<Vec<W, T>>(I::simd_xor(a.v, b.v)); } 5018 5019 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> and_(const Vec<W, T>& a, const Vec<W, T>& b, const Vec<W, T>& c) noexcept { return and_(and_(a, b), c); } 5020 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> or_(const Vec<W, T>& a, const Vec<W, T>& b, const Vec<W, T>& c) noexcept { return or_(or_(a, b), c); } 5021 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> xor_(const Vec<W, T>& a, const Vec<W, T>& b, const Vec<W, T>& c) noexcept { return xor_(xor_(a, b), c); } 5022 5023 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> blendv_bits(const Vec<W, T>& a, const Vec<W, T>& b, const Vec<W, T>& msk) noexcept { return from_simd<Vec<W, T>>(I::simd_blendv_bits(a.v, b.v, msk.v)); } 5024 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> blendv_u8(const Vec<W, T>& a, const Vec<W, T>& b, const Vec<W, T>& msk) noexcept { return from_simd<Vec<W, T>>(I::simd_blendv_u8(a.v, b.v, msk.v)); } 5025 5026 #if defined(BL_SIMD_FEATURE_BLEND_IMM) 5027 template<uint32_t H, uint32_t G, uint32_t F, uint32_t E, uint32_t D, uint32_t C, uint32_t B, uint32_t A, typename V> 5028 BL_INLINE_NODEBUG V blend_i16(const V& a, const V& b) noexcept { return V{I::simd_blend_i16<H, G, F, E, D, C, B, A>(a.v, b.v)}; } 5029 5030 template<uint32_t D, uint32_t C, uint32_t B, uint32_t A, typename V> 5031 BL_INLINE_NODEBUG V blend_i32(const V& a, const V& b) noexcept { return V{I::simd_blend_i32<D, C, B, A>(a.v, b.v)}; } 5032 5033 template<uint32_t B, uint32_t A, typename V> 5034 BL_INLINE_NODEBUG V blend_i64(const V& a, const V& b) noexcept { return V{I::simd_blend_i64<B, A>(a.v, b.v)}; } 5035 #endif // BL_SIMD_FEATURE_BLEND_IMM 5036 5037 BL_INLINE_NODEBUG Vec4xF32 add_f32x1(const Vec4xF32& a, const Vec4xF32& b) noexcept { return Vec4xF32{_mm_add_ss(a.v, b.v)}; } 5038 BL_INLINE_NODEBUG Vec2xF64 add_f64x1(const Vec2xF64& a, const Vec2xF64& b) noexcept { return Vec2xF64{_mm_add_sd(a.v, b.v)}; } 5039 BL_INLINE_NODEBUG Vec4xF32 sub_f32x1(const Vec4xF32& a, const Vec4xF32& b) noexcept { return Vec4xF32{_mm_sub_ss(a.v, b.v)}; } 5040 BL_INLINE_NODEBUG Vec2xF64 sub_f64x1(const Vec2xF64& a, const Vec2xF64& b) noexcept { return Vec2xF64{_mm_sub_sd(a.v, b.v)}; } 5041 BL_INLINE_NODEBUG Vec4xF32 mul_f32x1(const Vec4xF32& a, const Vec4xF32& b) noexcept { return Vec4xF32{_mm_mul_ss(a.v, b.v)}; } 5042 BL_INLINE_NODEBUG Vec2xF64 mul_f64x1(const Vec2xF64& a, const Vec2xF64& b) noexcept { return Vec2xF64{_mm_mul_sd(a.v, b.v)}; } 5043 BL_INLINE_NODEBUG Vec4xF32 div_f32x1(const Vec4xF32& a, const Vec4xF32& b) noexcept { return Vec4xF32{_mm_div_ss(a.v, b.v)}; } 5044 BL_INLINE_NODEBUG Vec2xF64 div_f64x1(const Vec2xF64& a, const Vec2xF64& b) noexcept { return Vec2xF64{_mm_div_sd(a.v, b.v)}; } 5045 BL_INLINE_NODEBUG Vec4xF32 min_f32x1(const Vec4xF32& a, const Vec4xF32& b) noexcept { return Vec4xF32{_mm_min_ss(a.v, b.v)}; } 5046 BL_INLINE_NODEBUG Vec2xF64 min_f64x1(const Vec2xF64& a, const Vec2xF64& b) noexcept { return Vec2xF64{_mm_min_sd(a.v, b.v)}; } 5047 BL_INLINE_NODEBUG Vec4xF32 max_f32x1(const Vec4xF32& a, const Vec4xF32& b) noexcept { return Vec4xF32{_mm_max_ss(a.v, b.v)}; } 5048 BL_INLINE_NODEBUG Vec2xF64 max_f64x1(const Vec2xF64& a, const Vec2xF64& b) noexcept { return Vec2xF64{_mm_max_sd(a.v, b.v)}; } 5049 5050 BL_INLINE_NODEBUG Vec4xF32 sqrt_f32x1(const Vec4xF32& a) noexcept { return Vec4xF32{_mm_sqrt_ss(a.v)}; } 5051 BL_INLINE_NODEBUG Vec2xF64 sqrt_f64x1(const Vec2xF64& a) noexcept { return Vec2xF64{_mm_sqrt_sd(a.v, a.v)}; } 5052 5053 BL_INLINE_NODEBUG Vec4xF32 cmp_eq_f32x1(const Vec4xF32& a, const Vec4xF32& b) noexcept { return Vec4xF32{_mm_cmpeq_ss(a.v, b.v)}; } 5054 BL_INLINE_NODEBUG Vec2xF64 cmp_eq_f64x1(const Vec2xF64& a, const Vec2xF64& b) noexcept { return Vec2xF64{_mm_cmpeq_sd(a.v, b.v)}; } 5055 BL_INLINE_NODEBUG Vec4xF32 cmp_ne_f32x1(const Vec4xF32& a, const Vec4xF32& b) noexcept { return Vec4xF32{_mm_cmpneq_ss(a.v, b.v)}; } 5056 BL_INLINE_NODEBUG Vec2xF64 cmp_ne_f64x1(const Vec2xF64& a, const Vec2xF64& b) noexcept { return Vec2xF64{_mm_cmpneq_sd(a.v, b.v)}; } 5057 BL_INLINE_NODEBUG Vec4xF32 cmp_ge_f32x1(const Vec4xF32& a, const Vec4xF32& b) noexcept { return Vec4xF32{_mm_cmpge_ss(a.v, b.v)}; } 5058 BL_INLINE_NODEBUG Vec2xF64 cmp_ge_f64x1(const Vec2xF64& a, const Vec2xF64& b) noexcept { return Vec2xF64{_mm_cmpge_sd(a.v, b.v)}; } 5059 BL_INLINE_NODEBUG Vec4xF32 cmp_gt_f32x1(const Vec4xF32& a, const Vec4xF32& b) noexcept { return Vec4xF32{_mm_cmpgt_ss(a.v, b.v)}; } 5060 BL_INLINE_NODEBUG Vec2xF64 cmp_gt_f64x1(const Vec2xF64& a, const Vec2xF64& b) noexcept { return Vec2xF64{_mm_cmpgt_sd(a.v, b.v)}; } 5061 BL_INLINE_NODEBUG Vec4xF32 cmp_le_f32x1(const Vec4xF32& a, const Vec4xF32& b) noexcept { return Vec4xF32{_mm_cmple_ss(a.v, b.v)}; } 5062 BL_INLINE_NODEBUG Vec2xF64 cmp_le_f64x1(const Vec2xF64& a, const Vec2xF64& b) noexcept { return Vec2xF64{_mm_cmple_sd(a.v, b.v)}; } 5063 BL_INLINE_NODEBUG Vec4xF32 cmp_lt_f32x1(const Vec4xF32& a, const Vec4xF32& b) noexcept { return Vec4xF32{_mm_cmplt_ss(a.v, b.v)}; } 5064 BL_INLINE_NODEBUG Vec2xF64 cmp_lt_f64x1(const Vec2xF64& a, const Vec2xF64& b) noexcept { return Vec2xF64{_mm_cmplt_sd(a.v, b.v)}; } 5065 5066 template<typename V> BL_INLINE_NODEBUG V add_f32(const V& a, const V& b) noexcept { return from_simd<V>(I::simd_add_f32(simd_as_f(a.v), simd_as_f(b.v))); } 5067 template<typename V> BL_INLINE_NODEBUG V add_f64(const V& a, const V& b) noexcept { return from_simd<V>(I::simd_add_f64(simd_as_d(a.v), simd_as_d(b.v))); } 5068 template<typename V> BL_INLINE_NODEBUG V sub_f32(const V& a, const V& b) noexcept { return from_simd<V>(I::simd_sub_f32(simd_as_f(a.v), simd_as_f(b.v))); } 5069 template<typename V> BL_INLINE_NODEBUG V sub_f64(const V& a, const V& b) noexcept { return from_simd<V>(I::simd_sub_f64(simd_as_d(a.v), simd_as_d(b.v))); } 5070 template<typename V> BL_INLINE_NODEBUG V mul_f32(const V& a, const V& b) noexcept { return from_simd<V>(I::simd_mul_f32(simd_as_f(a.v), simd_as_f(b.v))); } 5071 template<typename V> BL_INLINE_NODEBUG V mul_f64(const V& a, const V& b) noexcept { return from_simd<V>(I::simd_mul_f64(simd_as_d(a.v), simd_as_d(b.v))); } 5072 template<typename V> BL_INLINE_NODEBUG V div_f32(const V& a, const V& b) noexcept { return from_simd<V>(I::simd_div_f32(simd_as_f(a.v), simd_as_f(b.v))); } 5073 template<typename V> BL_INLINE_NODEBUG V div_f64(const V& a, const V& b) noexcept { return from_simd<V>(I::simd_div_f64(simd_as_d(a.v), simd_as_d(b.v))); } 5074 template<typename V> BL_INLINE_NODEBUG V min_f32(const V& a, const V& b) noexcept { return from_simd<V>(I::simd_min_f32(simd_as_f(a.v), simd_as_f(b.v))); } 5075 template<typename V> BL_INLINE_NODEBUG V min_f64(const V& a, const V& b) noexcept { return from_simd<V>(I::simd_min_f64(simd_as_d(a.v), simd_as_d(b.v))); } 5076 template<typename V> BL_INLINE_NODEBUG V max_f32(const V& a, const V& b) noexcept { return from_simd<V>(I::simd_max_f32(simd_as_f(a.v), simd_as_f(b.v))); } 5077 template<typename V> BL_INLINE_NODEBUG V max_f64(const V& a, const V& b) noexcept { return from_simd<V>(I::simd_max_f64(simd_as_d(a.v), simd_as_d(b.v))); } 5078 5079 template<typename V> BL_INLINE_NODEBUG V cmp_eq_f32(const V& a, const V& b) noexcept { return V{I::simd_cmp_eq_f32(a.v, b.v)}; } 5080 template<typename V> BL_INLINE_NODEBUG V cmp_eq_f64(const V& a, const V& b) noexcept { return V{I::simd_cmp_eq_f64(a.v, b.v)}; } 5081 template<typename V> BL_INLINE_NODEBUG V cmp_ne_f32(const V& a, const V& b) noexcept { return V{I::simd_cmp_ne_f32(a.v, b.v)}; } 5082 template<typename V> BL_INLINE_NODEBUG V cmp_ne_f64(const V& a, const V& b) noexcept { return V{I::simd_cmp_ne_f64(a.v, b.v)}; } 5083 template<typename V> BL_INLINE_NODEBUG V cmp_ge_f32(const V& a, const V& b) noexcept { return V{I::simd_cmp_ge_f32(a.v, b.v)}; } 5084 template<typename V> BL_INLINE_NODEBUG V cmp_ge_f64(const V& a, const V& b) noexcept { return V{I::simd_cmp_ge_f64(a.v, b.v)}; } 5085 template<typename V> BL_INLINE_NODEBUG V cmp_gt_f32(const V& a, const V& b) noexcept { return V{I::simd_cmp_gt_f32(a.v, b.v)}; } 5086 template<typename V> BL_INLINE_NODEBUG V cmp_gt_f64(const V& a, const V& b) noexcept { return V{I::simd_cmp_gt_f64(a.v, b.v)}; } 5087 template<typename V> BL_INLINE_NODEBUG V cmp_le_f32(const V& a, const V& b) noexcept { return V{I::simd_cmp_le_f32(a.v, b.v)}; } 5088 template<typename V> BL_INLINE_NODEBUG V cmp_le_f64(const V& a, const V& b) noexcept { return V{I::simd_cmp_le_f64(a.v, b.v)}; } 5089 template<typename V> BL_INLINE_NODEBUG V cmp_lt_f32(const V& a, const V& b) noexcept { return V{I::simd_cmp_lt_f32(a.v, b.v)}; } 5090 template<typename V> BL_INLINE_NODEBUG V cmp_lt_f64(const V& a, const V& b) noexcept { return V{I::simd_cmp_lt_f64(a.v, b.v)}; } 5091 5092 template<typename V> BL_INLINE_NODEBUG V abs_f32(const V& a) noexcept { return from_simd<V>(I::simd_abs_f32(simd_as_f(a.v))); } 5093 template<typename V> BL_INLINE_NODEBUG V abs_f64(const V& a) noexcept { return from_simd<V>(I::simd_abs_f64(simd_as_d(a.v))); } 5094 5095 template<typename V> BL_INLINE_NODEBUG V sqrt_f32(const V& a) noexcept { return from_simd<V>(I::simd_sqrt_f32(simd_as_f(a.v))); } 5096 template<typename V> BL_INLINE_NODEBUG V sqrt_f64(const V& a) noexcept { return from_simd<V>(I::simd_sqrt_f64(simd_as_d(a.v))); } 5097 5098 template<size_t W> BL_INLINE_NODEBUG Vec<W, float > add(const Vec<W, float >& a, const Vec<W, float >& b) noexcept { return Vec<W, float >{I::simd_add_f32(a.v, b.v)}; } 5099 template<size_t W> BL_INLINE_NODEBUG Vec<W, double> add(const Vec<W, double>& a, const Vec<W, double>& b) noexcept { return Vec<W, double>{I::simd_add_f64(a.v, b.v)}; } 5100 template<size_t W> BL_INLINE_NODEBUG Vec<W, float > sub(const Vec<W, float >& a, const Vec<W, float >& b) noexcept { return Vec<W, float >{I::simd_sub_f32(a.v, b.v)}; } 5101 template<size_t W> BL_INLINE_NODEBUG Vec<W, double> sub(const Vec<W, double>& a, const Vec<W, double>& b) noexcept { return Vec<W, double>{I::simd_sub_f64(a.v, b.v)}; } 5102 template<size_t W> BL_INLINE_NODEBUG Vec<W, float > mul(const Vec<W, float >& a, const Vec<W, float >& b) noexcept { return Vec<W, float >{I::simd_mul_f32(a.v, b.v)}; } 5103 template<size_t W> BL_INLINE_NODEBUG Vec<W, double> mul(const Vec<W, double>& a, const Vec<W, double>& b) noexcept { return Vec<W, double>{I::simd_mul_f64(a.v, b.v)}; } 5104 template<size_t W> BL_INLINE_NODEBUG Vec<W, float > div(const Vec<W, float >& a, const Vec<W, float >& b) noexcept { return Vec<W, float >{I::simd_div_f32(a.v, b.v)}; } 5105 template<size_t W> BL_INLINE_NODEBUG Vec<W, double> div(const Vec<W, double>& a, const Vec<W, double>& b) noexcept { return Vec<W, double>{I::simd_div_f64(a.v, b.v)}; } 5106 template<size_t W> BL_INLINE_NODEBUG Vec<W, float > min(const Vec<W, float >& a, const Vec<W, float >& b) noexcept { return Vec<W, float >{I::simd_min_f32(a.v, b.v)}; } 5107 template<size_t W> BL_INLINE_NODEBUG Vec<W, double> min(const Vec<W, double>& a, const Vec<W, double>& b) noexcept { return Vec<W, double>{I::simd_min_f64(a.v, b.v)}; } 5108 template<size_t W> BL_INLINE_NODEBUG Vec<W, float > max(const Vec<W, float >& a, const Vec<W, float >& b) noexcept { return Vec<W, float >{I::simd_max_f32(a.v, b.v)}; } 5109 template<size_t W> BL_INLINE_NODEBUG Vec<W, double> max(const Vec<W, double>& a, const Vec<W, double>& b) noexcept { return Vec<W, double>{I::simd_max_f64(a.v, b.v)}; } 5110 5111 template<size_t W> BL_INLINE_NODEBUG Vec<W, float > cmp_eq(const Vec<W, float >& a, const Vec<W, float >& b) noexcept { return Vec<W, float >{I::simd_cmp_eq_f32(a.v, b.v)}; } 5112 template<size_t W> BL_INLINE_NODEBUG Vec<W, double> cmp_eq(const Vec<W, double>& a, const Vec<W, double>& b) noexcept { return Vec<W, double>{I::simd_cmp_eq_f64(a.v, b.v)}; } 5113 template<size_t W> BL_INLINE_NODEBUG Vec<W, float > cmp_ne(const Vec<W, float >& a, const Vec<W, float >& b) noexcept { return Vec<W, float >{I::simd_cmp_ne_f32(a.v, b.v)}; } 5114 template<size_t W> BL_INLINE_NODEBUG Vec<W, double> cmp_ne(const Vec<W, double>& a, const Vec<W, double>& b) noexcept { return Vec<W, double>{I::simd_cmp_ne_f64(a.v, b.v)}; } 5115 template<size_t W> BL_INLINE_NODEBUG Vec<W, float > cmp_ge(const Vec<W, float >& a, const Vec<W, float >& b) noexcept { return Vec<W, float >{I::simd_cmp_ge_f32(a.v, b.v)}; } 5116 template<size_t W> BL_INLINE_NODEBUG Vec<W, double> cmp_ge(const Vec<W, double>& a, const Vec<W, double>& b) noexcept { return Vec<W, double>{I::simd_cmp_ge_f64(a.v, b.v)}; } 5117 template<size_t W> BL_INLINE_NODEBUG Vec<W, float > cmp_gt(const Vec<W, float >& a, const Vec<W, float >& b) noexcept { return Vec<W, float >{I::simd_cmp_gt_f32(a.v, b.v)}; } 5118 template<size_t W> BL_INLINE_NODEBUG Vec<W, double> cmp_gt(const Vec<W, double>& a, const Vec<W, double>& b) noexcept { return Vec<W, double>{I::simd_cmp_gt_f64(a.v, b.v)}; } 5119 template<size_t W> BL_INLINE_NODEBUG Vec<W, float > cmp_le(const Vec<W, float >& a, const Vec<W, float >& b) noexcept { return Vec<W, float >{I::simd_cmp_le_f32(a.v, b.v)}; } 5120 template<size_t W> BL_INLINE_NODEBUG Vec<W, double> cmp_le(const Vec<W, double>& a, const Vec<W, double>& b) noexcept { return Vec<W, double>{I::simd_cmp_le_f64(a.v, b.v)}; } 5121 template<size_t W> BL_INLINE_NODEBUG Vec<W, float > cmp_lt(const Vec<W, float >& a, const Vec<W, float >& b) noexcept { return Vec<W, float >{I::simd_cmp_lt_f32(a.v, b.v)}; } 5122 template<size_t W> BL_INLINE_NODEBUG Vec<W, double> cmp_lt(const Vec<W, double>& a, const Vec<W, double>& b) noexcept { return Vec<W, double>{I::simd_cmp_lt_f64(a.v, b.v)}; } 5123 5124 template<size_t W> BL_INLINE_NODEBUG Vec<W, float > abs(const Vec<W, float >& a) noexcept { return Vec<W, float >{I::simd_abs_f32(a.v)}; } 5125 template<size_t W> BL_INLINE_NODEBUG Vec<W, double> abs(const Vec<W, double>& a) noexcept { return Vec<W, double>{I::simd_abs_f64(a.v)}; } 5126 5127 template<size_t W> BL_INLINE_NODEBUG Vec<W, float > sqrt(const Vec<W, float >& a) noexcept { return Vec<W, float >{I::simd_sqrt_f32(a.v)}; } 5128 template<size_t W> BL_INLINE_NODEBUG Vec<W, double> sqrt(const Vec<W, double>& a) noexcept { return Vec<W, double>{I::simd_sqrt_f64(a.v)}; } 5129 5130 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> abs_i8(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_abs_i8(simd_as_i(a.v))); } 5131 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> abs_i16(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_abs_i16(simd_as_i(a.v))); } 5132 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> abs_i32(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_abs_i32(simd_as_i(a.v))); } 5133 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> abs_i64(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_abs_i64(simd_as_i(a.v))); } 5134 5135 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> abs(const Vec<W, int8_t>& a) noexcept { return Vec<W, int8_t>{I::simd_abs_i8(simd_as_i(a.v))}; } 5136 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> abs(const Vec<W, int16_t>& a) noexcept { return Vec<W, int16_t>{I::simd_abs_i16(simd_as_i(a.v))}; } 5137 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> abs(const Vec<W, int32_t>& a) noexcept { return Vec<W, int32_t>{I::simd_abs_i32(simd_as_i(a.v))}; } 5138 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> abs(const Vec<W, int64_t>& a) noexcept { return Vec<W, int64_t>{I::simd_abs_i64(simd_as_i(a.v))}; } 5139 5140 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> add_i8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_add_i8(a.v, b.v)}; } 5141 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> add_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_add_i16(a.v, b.v)}; } 5142 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> add_i32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_add_i32(a.v, b.v)}; } 5143 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> add_i64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_add_i64(a.v, b.v)}; } 5144 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> add_u8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_add_u8(a.v, b.v)}; } 5145 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> add_u16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_add_u16(a.v, b.v)}; } 5146 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> add_u32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_add_u32(a.v, b.v)}; } 5147 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> add_u64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_add_u64(a.v, b.v)}; } 5148 5149 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> adds_i8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_adds_i8(a.v, b.v)}; } 5150 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> adds_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_adds_i16(a.v, b.v)}; } 5151 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> adds_u8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_adds_u8(a.v, b.v)}; } 5152 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> adds_u16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_adds_u16(a.v, b.v)}; } 5153 5154 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> add(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_add_i8(a.v, b.v)}; } 5155 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> add(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_add_i16(a.v, b.v)}; } 5156 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> add(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_add_i32(a.v, b.v)}; } 5157 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> add(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_add_i64(a.v, b.v)}; } 5158 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> add(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_add_u8(a.v, b.v)}; } 5159 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> add(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_add_u16(a.v, b.v)}; } 5160 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> add(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_add_u32(a.v, b.v)}; } 5161 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> add(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_add_u64(a.v, b.v)}; } 5162 5163 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> adds(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_adds_i8(a.v, b.v)}; } 5164 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> adds(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_adds_i16(a.v, b.v)}; } 5165 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> adds(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_adds_u8(a.v, b.v)}; } 5166 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> adds(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_adds_u16(a.v, b.v)}; } 5167 5168 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> sub_i8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_sub_i8(a.v, b.v)}; } 5169 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> sub_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_sub_i16(a.v, b.v)}; } 5170 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> sub_i32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_sub_i32(a.v, b.v)}; } 5171 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> sub_i64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_sub_i64(a.v, b.v)}; } 5172 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> sub_u8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_sub_u8(a.v, b.v)}; } 5173 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> sub_u16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_sub_u16(a.v, b.v)}; } 5174 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> sub_u32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_sub_u32(a.v, b.v)}; } 5175 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> sub_u64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_sub_u64(a.v, b.v)}; } 5176 5177 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> subs_i8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_subs_i8(a.v, b.v)}; } 5178 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> subs_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_subs_i16(a.v, b.v)}; } 5179 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> subs_u8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_subs_u8(a.v, b.v)}; } 5180 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> subs_u16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_subs_u16(a.v, b.v)}; } 5181 5182 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> sub(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_sub_i8(a.v, b.v)}; } 5183 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> sub(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_sub_i16(a.v, b.v)}; } 5184 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> sub(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_sub_i32(a.v, b.v)}; } 5185 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> sub(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_sub_i64(a.v, b.v)}; } 5186 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> sub(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_sub_u8(a.v, b.v)}; } 5187 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> sub(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_sub_u16(a.v, b.v)}; } 5188 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> sub(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_sub_u32(a.v, b.v)}; } 5189 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> sub(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_sub_u64(a.v, b.v)}; } 5190 5191 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> subs(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_subs_i8(a.v, b.v)}; } 5192 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> subs(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_subs_i16(a.v, b.v)}; } 5193 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> subs(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_subs_u8(a.v, b.v)}; } 5194 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> subs(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_subs_u16(a.v, b.v)}; } 5195 5196 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> mul_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_mul_i16(a.v, b.v)}; } 5197 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> mul_i32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_mul_i32(a.v, b.v)}; } 5198 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> mul_i64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_mul_i64(a.v, b.v)}; } 5199 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> mul_u16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_mul_u16(a.v, b.v)}; } 5200 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> mul_u32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_mul_u32(a.v, b.v)}; } 5201 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> mul_u64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_mul_u64(a.v, b.v)}; } 5202 5203 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> mulh_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_mulh_i16(a.v, b.v)}; } 5204 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> mulh_u16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_mulh_u16(a.v, b.v)}; } 5205 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> mulw_u32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_mulw_u32(a.v, b.v)}; } 5206 5207 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> maddw_i16_i32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_maddw_i16_i32(a.v, b.v)}; } 5208 5209 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> mul(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_mul_i16(a.v, b.v)}; } 5210 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> mul(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_mul_i32(a.v, b.v)}; } 5211 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> mul(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_mul_i64(a.v, b.v)}; } 5212 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> mul(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_mul_u16(a.v, b.v)}; } 5213 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> mul(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_mul_u32(a.v, b.v)}; } 5214 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> mul(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_mul_u64(a.v, b.v)}; } 5215 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> mulh(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_mulh_i16(a.v, b.v)}; } 5216 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> mulh(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_mulh_u16(a.v, b.v)}; } 5217 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> mulw(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_mulw_u32(a.v, b.v)}; } 5218 5219 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_eq_i8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_eq_i8(a.v, b.v)}; } 5220 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_eq_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_eq_i16(a.v, b.v)}; } 5221 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_eq_i32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_eq_i32(a.v, b.v)}; } 5222 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_eq_i64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_eq_i64(a.v, b.v)}; } 5223 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_eq_u8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_eq_i8(a.v, b.v)}; } 5224 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_eq_u16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_eq_i16(a.v, b.v)}; } 5225 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_eq_u32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_eq_i32(a.v, b.v)}; } 5226 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_eq_u64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_eq_i64(a.v, b.v)}; } 5227 5228 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> cmp_eq(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_cmp_eq_i8(a.v, b.v)}; } 5229 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> cmp_eq(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_cmp_eq_i16(a.v, b.v)}; } 5230 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> cmp_eq(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_cmp_eq_i32(a.v, b.v)}; } 5231 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> cmp_eq(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_cmp_eq_i64(a.v, b.v)}; } 5232 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> cmp_eq(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_cmp_eq_i8(a.v, b.v)}; } 5233 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> cmp_eq(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_cmp_eq_i16(a.v, b.v)}; } 5234 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> cmp_eq(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_cmp_eq_i32(a.v, b.v)}; } 5235 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> cmp_eq(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_cmp_eq_i64(a.v, b.v)}; } 5236 5237 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ne_i8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ne_i8(a.v, b.v)}; } 5238 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ne_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ne_i16(a.v, b.v)}; } 5239 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ne_i32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ne_i32(a.v, b.v)}; } 5240 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ne_i64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ne_i64(a.v, b.v)}; } 5241 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ne_u8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ne_i8(a.v, b.v)}; } 5242 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ne_u16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ne_i16(a.v, b.v)}; } 5243 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ne_u32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ne_i32(a.v, b.v)}; } 5244 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ne_u64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ne_i64(a.v, b.v)}; } 5245 5246 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> cmp_ne(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_cmp_ne_i8(a.v, b.v)}; } 5247 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> cmp_ne(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_cmp_ne_i16(a.v, b.v)}; } 5248 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> cmp_ne(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_cmp_ne_i32(a.v, b.v)}; } 5249 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> cmp_ne(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_cmp_ne_i64(a.v, b.v)}; } 5250 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> cmp_ne(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_cmp_ne_i8(a.v, b.v)}; } 5251 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> cmp_ne(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_cmp_ne_i16(a.v, b.v)}; } 5252 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> cmp_ne(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_cmp_ne_i32(a.v, b.v)}; } 5253 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> cmp_ne(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_cmp_ne_i64(a.v, b.v)}; } 5254 5255 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_gt_i8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_gt_i8(a.v, b.v)}; } 5256 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_gt_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_gt_i16(a.v, b.v)}; } 5257 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_gt_i32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_gt_i32(a.v, b.v)}; } 5258 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_gt_i64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_gt_i64(a.v, b.v)}; } 5259 5260 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_gt_u8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_gt_u8(a.v, b.v)}; } 5261 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_gt_u16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_gt_u16(a.v, b.v)}; } 5262 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_gt_u32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_gt_u32(a.v, b.v)}; } 5263 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_gt_u64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_gt_u64(a.v, b.v)}; } 5264 5265 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> cmp_gt(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_cmp_gt_i8(a.v, b.v)}; } 5266 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> cmp_gt(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_cmp_gt_i16(a.v, b.v)}; } 5267 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> cmp_gt(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_cmp_gt_i32(a.v, b.v)}; } 5268 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> cmp_gt(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_cmp_gt_i64(a.v, b.v)}; } 5269 5270 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> cmp_gt(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_cmp_gt_u8(a.v, b.v)}; } 5271 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> cmp_gt(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_cmp_gt_u16(a.v, b.v)}; } 5272 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> cmp_gt(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_cmp_gt_u32(a.v, b.v)}; } 5273 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> cmp_gt(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_cmp_gt_u64(a.v, b.v)}; } 5274 5275 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ge_i8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ge_i8(a.v, b.v)}; } 5276 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ge_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ge_i16(a.v, b.v)}; } 5277 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ge_i32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ge_i32(a.v, b.v)}; } 5278 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ge_i64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ge_i64(a.v, b.v)}; } 5279 5280 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ge_u8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ge_u8(a.v, b.v)}; } 5281 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ge_u16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ge_u16(a.v, b.v)}; } 5282 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ge_u32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ge_u32(a.v, b.v)}; } 5283 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_ge_u64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_ge_u64(a.v, b.v)}; } 5284 5285 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> cmp_ge(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_cmp_ge_i8(a.v, b.v)}; } 5286 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> cmp_ge(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_cmp_ge_i16(a.v, b.v)}; } 5287 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> cmp_ge(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_cmp_ge_i32(a.v, b.v)}; } 5288 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> cmp_ge(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_cmp_ge_i64(a.v, b.v)}; } 5289 5290 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> cmp_ge(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_cmp_ge_u8(a.v, b.v)}; } 5291 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> cmp_ge(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_cmp_ge_u16(a.v, b.v)}; } 5292 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> cmp_ge(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_cmp_ge_u32(a.v, b.v)}; } 5293 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> cmp_ge(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_cmp_ge_u64(a.v, b.v)}; } 5294 5295 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_lt_i8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_lt_i8(a.v, b.v)}; } 5296 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_lt_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_lt_i16(a.v, b.v)}; } 5297 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_lt_i32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_lt_i32(a.v, b.v)}; } 5298 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_lt_i64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_lt_i64(a.v, b.v)}; } 5299 5300 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_lt_u8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_lt_u8(a.v, b.v)}; } 5301 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_lt_u16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_lt_u16(a.v, b.v)}; } 5302 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_lt_u32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_lt_u32(a.v, b.v)}; } 5303 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_lt_u64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_lt_u64(a.v, b.v)}; } 5304 5305 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> cmp_lt(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_cmp_lt_i8(a.v, b.v)}; } 5306 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> cmp_lt(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_cmp_lt_i16(a.v, b.v)}; } 5307 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> cmp_lt(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_cmp_lt_i32(a.v, b.v)}; } 5308 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> cmp_lt(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_cmp_lt_i64(a.v, b.v)}; } 5309 5310 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> cmp_lt(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_cmp_lt_u8(a.v, b.v)}; } 5311 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> cmp_lt(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_cmp_lt_u16(a.v, b.v)}; } 5312 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> cmp_lt(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_cmp_lt_u32(a.v, b.v)}; } 5313 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> cmp_lt(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_cmp_lt_u64(a.v, b.v)}; } 5314 5315 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_le_i8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_le_i8(a.v, b.v)}; } 5316 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_le_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_le_i16(a.v, b.v)}; } 5317 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_le_i32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_le_i32(a.v, b.v)}; } 5318 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_le_i64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_le_i64(a.v, b.v)}; } 5319 5320 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_le_u8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_le_u8(a.v, b.v)}; } 5321 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_le_u16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_le_u16(a.v, b.v)}; } 5322 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_le_u32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_le_u32(a.v, b.v)}; } 5323 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> cmp_le_u64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_cmp_le_u64(a.v, b.v)}; } 5324 5325 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> cmp_le(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_cmp_le_i8(a.v, b.v)}; } 5326 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> cmp_le(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_cmp_le_i16(a.v, b.v)}; } 5327 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> cmp_le(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_cmp_le_i32(a.v, b.v)}; } 5328 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> cmp_le(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_cmp_le_i64(a.v, b.v)}; } 5329 5330 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> cmp_le(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_cmp_le_u8(a.v, b.v)}; } 5331 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> cmp_le(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_cmp_le_u16(a.v, b.v)}; } 5332 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> cmp_le(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_cmp_le_u32(a.v, b.v)}; } 5333 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> cmp_le(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_cmp_le_u64(a.v, b.v)}; } 5334 5335 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> min_i8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_min_i8(a.v, b.v)}; } 5336 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> min_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_min_i16(a.v, b.v)}; } 5337 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> min_i32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_min_i32(a.v, b.v)}; } 5338 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> min_i64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_min_i64(a.v, b.v)}; } 5339 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> min_u8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_min_u8(a.v, b.v)}; } 5340 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> min_u16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_min_u16(a.v, b.v)}; } 5341 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> min_u32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_min_u32(a.v, b.v)}; } 5342 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> min_u64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_min_u64(a.v, b.v)}; } 5343 5344 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> max_i8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_max_i8(a.v, b.v)}; } 5345 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> max_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_max_i16(a.v, b.v)}; } 5346 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> max_i32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_max_i32(a.v, b.v)}; } 5347 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> max_i64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_max_i64(a.v, b.v)}; } 5348 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> max_u8(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_max_u8(a.v, b.v)}; } 5349 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> max_u16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_max_u16(a.v, b.v)}; } 5350 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> max_u32(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_max_u32(a.v, b.v)}; } 5351 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> max_u64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return Vec<W, T>{I::simd_max_u64(a.v, b.v)}; } 5352 5353 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> min(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_min_i8(a.v, b.v)}; } 5354 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> min(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_min_i16(a.v, b.v)}; } 5355 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> min(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_min_i32(a.v, b.v)}; } 5356 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> min(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_min_i64(a.v, b.v)}; } 5357 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> min(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_min_u8(a.v, b.v)}; } 5358 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> min(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_min_u16(a.v, b.v)}; } 5359 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> min(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_min_u32(a.v, b.v)}; } 5360 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> min(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_min_u64(a.v, b.v)}; } 5361 5362 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> max(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_max_i8(a.v, b.v)}; } 5363 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> max(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_max_i16(a.v, b.v)}; } 5364 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> max(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_max_i32(a.v, b.v)}; } 5365 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> max(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_max_i64(a.v, b.v)}; } 5366 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> max(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_max_u8(a.v, b.v)}; } 5367 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> max(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_max_u16(a.v, b.v)}; } 5368 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> max(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_max_u32(a.v, b.v)}; } 5369 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> max(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_max_u64(a.v, b.v)}; } 5370 5371 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> smin(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_min_i8(a.v, b.v)}; } 5372 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> smin(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_min_i16(a.v, b.v)}; } 5373 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> smin(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_min_i32(a.v, b.v)}; } 5374 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> smin(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_min_i64(a.v, b.v)}; } 5375 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> smin(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_min_i8(a.v, b.v)}; } 5376 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> smin(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_min_i16(a.v, b.v)}; } 5377 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> smin(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_min_i32(a.v, b.v)}; } 5378 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> smin(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_min_i64(a.v, b.v)}; } 5379 5380 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> smax(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_max_i8(a.v, b.v)}; } 5381 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> smax(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_max_i16(a.v, b.v)}; } 5382 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> smax(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_max_i32(a.v, b.v)}; } 5383 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> smax(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_max_i64(a.v, b.v)}; } 5384 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> smax(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_max_i8(a.v, b.v)}; } 5385 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> smax(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_max_i16(a.v, b.v)}; } 5386 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> smax(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_max_i32(a.v, b.v)}; } 5387 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> smax(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_max_i64(a.v, b.v)}; } 5388 5389 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> umin(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_min_u8(a.v, b.v)}; } 5390 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> umin(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_min_u16(a.v, b.v)}; } 5391 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> umin(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_min_u32(a.v, b.v)}; } 5392 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> umin(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_min_u64(a.v, b.v)}; } 5393 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> umin(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_min_u8(a.v, b.v)}; } 5394 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> umin(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_min_u16(a.v, b.v)}; } 5395 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> umin(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_min_u32(a.v, b.v)}; } 5396 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> umin(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_min_u64(a.v, b.v)}; } 5397 5398 template<size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> umax(const Vec<W, int8_t>& a, const Vec<W, int8_t>& b) noexcept { return Vec<W, int8_t>{I::simd_max_u8(a.v, b.v)}; } 5399 template<size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> umax(const Vec<W, int16_t>& a, const Vec<W, int16_t>& b) noexcept { return Vec<W, int16_t>{I::simd_max_u16(a.v, b.v)}; } 5400 template<size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> umax(const Vec<W, int32_t>& a, const Vec<W, int32_t>& b) noexcept { return Vec<W, int32_t>{I::simd_max_u32(a.v, b.v)}; } 5401 template<size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> umax(const Vec<W, int64_t>& a, const Vec<W, int64_t>& b) noexcept { return Vec<W, int64_t>{I::simd_max_u64(a.v, b.v)}; } 5402 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> umax(const Vec<W, uint8_t>& a, const Vec<W, uint8_t>& b) noexcept { return Vec<W, uint8_t>{I::simd_max_u8(a.v, b.v)}; } 5403 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> umax(const Vec<W, uint16_t>& a, const Vec<W, uint16_t>& b) noexcept { return Vec<W, uint16_t>{I::simd_max_u16(a.v, b.v)}; } 5404 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> umax(const Vec<W, uint32_t>& a, const Vec<W, uint32_t>& b) noexcept { return Vec<W, uint32_t>{I::simd_max_u32(a.v, b.v)}; } 5405 template<size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> umax(const Vec<W, uint64_t>& a, const Vec<W, uint64_t>& b) noexcept { return Vec<W, uint64_t>{I::simd_max_u64(a.v, b.v)}; } 5406 5407 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> slli_i8(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_slli_i8<kN>(simd_as_i(a.v))); } 5408 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> slli_i16(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_slli_i16<kN>(simd_as_i(a.v))); } 5409 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> slli_i32(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_slli_i32<kN>(simd_as_i(a.v))); } 5410 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> slli_i64(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_slli_i64<kN>(simd_as_i(a.v))); } 5411 5412 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> slli_u8(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_slli_i8<kN>(simd_as_i(a.v))); } 5413 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> slli_u16(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_slli_i16<kN>(simd_as_i(a.v))); } 5414 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> slli_u32(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_slli_i32<kN>(simd_as_i(a.v))); } 5415 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> slli_u64(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_slli_i64<kN>(simd_as_i(a.v))); } 5416 5417 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> srli_u8(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_srli_u8<kN>(simd_as_i(a.v))); } 5418 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> srli_u16(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_srli_u16<kN>(simd_as_i(a.v))); } 5419 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> srli_u32(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_srli_u32<kN>(simd_as_i(a.v))); } 5420 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> srli_u64(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_srli_u64<kN>(simd_as_i(a.v))); } 5421 5422 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> srai_i8(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_srai_i8<kN>(simd_as_i(a.v))); } 5423 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> srai_i16(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_srai_i16<kN>(simd_as_i(a.v))); } 5424 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> srai_i32(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_srai_i32<kN>(simd_as_i(a.v))); } 5425 template<uint32_t kN, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> srai_i64(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_srai_i64<kN>(simd_as_i(a.v))); } 5426 5427 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> slli(const Vec<W, int8_t>& a) noexcept { return from_simd<Vec<W, int8_t>>(I::simd_slli_i8<kN>(simd_as_i(a.v))); } 5428 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> slli(const Vec<W, int16_t>& a) noexcept { return from_simd<Vec<W, int16_t>>(I::simd_slli_i16<kN>(simd_as_i(a.v))); } 5429 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> slli(const Vec<W, int32_t>& a) noexcept { return from_simd<Vec<W, int32_t>>(I::simd_slli_i32<kN>(simd_as_i(a.v))); } 5430 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> slli(const Vec<W, int64_t>& a) noexcept { return from_simd<Vec<W, int64_t>>(I::simd_slli_i64<kN>(simd_as_i(a.v))); } 5431 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> slli(const Vec<W, uint8_t>& a) noexcept { return from_simd<Vec<W, uint8_t>>(I::simd_slli_i8<kN>(simd_as_i(a.v))); } 5432 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> slli(const Vec<W, uint16_t>& a) noexcept { return from_simd<Vec<W, uint16_t>>(I::simd_slli_i16<kN>(simd_as_i(a.v))); } 5433 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> slli(const Vec<W, uint32_t>& a) noexcept { return from_simd<Vec<W, uint32_t>>(I::simd_slli_i32<kN>(simd_as_i(a.v))); } 5434 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> slli(const Vec<W, uint64_t>& a) noexcept { return from_simd<Vec<W, uint64_t>>(I::simd_slli_i64<kN>(simd_as_i(a.v))); } 5435 5436 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> srli(const Vec<W, int8_t>& a) noexcept { return from_simd<Vec<W, int8_t>>(I::simd_srli_u8<kN>(simd_as_i(a.v))); } 5437 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> srli(const Vec<W, int16_t>& a) noexcept { return from_simd<Vec<W, int16_t>>(I::simd_srli_u16<kN>(simd_as_i(a.v))); } 5438 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> srli(const Vec<W, int32_t>& a) noexcept { return from_simd<Vec<W, int32_t>>(I::simd_srli_u32<kN>(simd_as_i(a.v))); } 5439 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> srli(const Vec<W, int64_t>& a) noexcept { return from_simd<Vec<W, int64_t>>(I::simd_srli_u64<kN>(simd_as_i(a.v))); } 5440 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> srli(const Vec<W, uint8_t>& a) noexcept { return from_simd<Vec<W, uint8_t>>(I::simd_srli_u8<kN>(simd_as_i(a.v))); } 5441 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> srli(const Vec<W, uint16_t>& a) noexcept { return from_simd<Vec<W, uint16_t>>(I::simd_srli_u16<kN>(simd_as_i(a.v))); } 5442 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> srli(const Vec<W, uint32_t>& a) noexcept { return from_simd<Vec<W, uint32_t>>(I::simd_srli_u32<kN>(simd_as_i(a.v))); } 5443 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> srli(const Vec<W, uint64_t>& a) noexcept { return from_simd<Vec<W, uint64_t>>(I::simd_srli_u64<kN>(simd_as_i(a.v))); } 5444 5445 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, int8_t> srai(const Vec<W, int8_t>& a) noexcept { return from_simd<Vec<W, int8_t>>(I::simd_srai_i8<kN>(simd_as_i(a.v))); } 5446 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, int16_t> srai(const Vec<W, int16_t>& a) noexcept { return from_simd<Vec<W, int16_t>>(I::simd_srai_i16<kN>(simd_as_i(a.v))); } 5447 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, int32_t> srai(const Vec<W, int32_t>& a) noexcept { return from_simd<Vec<W, int32_t>>(I::simd_srai_i32<kN>(simd_as_i(a.v))); } 5448 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, int64_t> srai(const Vec<W, int64_t>& a) noexcept { return from_simd<Vec<W, int64_t>>(I::simd_srai_i64<kN>(simd_as_i(a.v))); } 5449 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, uint8_t> srai(const Vec<W, uint8_t>& a) noexcept { return from_simd<Vec<W, uint8_t>>(I::simd_srai_i8<kN>(simd_as_i(a.v))); } 5450 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, uint16_t> srai(const Vec<W, uint16_t>& a) noexcept { return from_simd<Vec<W, uint16_t>>(I::simd_srai_i16<kN>(simd_as_i(a.v))); } 5451 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, uint32_t> srai(const Vec<W, uint32_t>& a) noexcept { return from_simd<Vec<W, uint32_t>>(I::simd_srai_i32<kN>(simd_as_i(a.v))); } 5452 template<uint32_t kN, size_t W> BL_INLINE_NODEBUG Vec<W, uint64_t> srai(const Vec<W, uint64_t>& a) noexcept { return from_simd<Vec<W, uint64_t>>(I::simd_srai_i64<kN>(simd_as_i(a.v))); } 5453 5454 template<uint32_t kNumBytes, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> sllb_u128(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_sllb_u128<kNumBytes>(simd_as_i(a.v))); } 5455 template<uint32_t kNumBytes, size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> srlb_u128(const Vec<W, T>& a) noexcept { return from_simd<Vec<W, T>>(I::simd_srlb_u128<kNumBytes>(simd_as_i(a.v))); } 5456 5457 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> sad_u8_u64(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return from_simd<Vec<W, T>>(I::simd_sad_u8_u64(simd_as_i(a.v), simd_as_i(b.v))); } 5458 5459 #if defined(BL_TARGET_OPT_SSSE3) 5460 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> maddws_u8xi8_i16(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return from_simd<Vec<W, T>>(I::simd_maddws_u8xi8_i16(simd_as_i(a.v), simd_as_i(b.v))); } 5461 #endif // BL_TARGET_OPT_SSSE3 5462 5463 #if defined(BL_TARGET_OPT_SSE4_2) 5464 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> clmul_u128_ll(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return from_simd<Vec<W, T>>(I::simd_clmul_u128_ll(simd_as_i(a.v), simd_as_i(b.v))); } 5465 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> clmul_u128_lh(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return from_simd<Vec<W, T>>(I::simd_clmul_u128_lh(simd_as_i(a.v), simd_as_i(b.v))); } 5466 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> clmul_u128_hl(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return from_simd<Vec<W, T>>(I::simd_clmul_u128_hl(simd_as_i(a.v), simd_as_i(b.v))); } 5467 template<size_t W, typename T> BL_INLINE_NODEBUG Vec<W, T> clmul_u128_hh(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return from_simd<Vec<W, T>>(I::simd_clmul_u128_hh(simd_as_i(a.v), simd_as_i(b.v))); } 5468 #endif // BL_TARGET_OPT_SSE4_2 5469 5470 // SIMD - Public - Overloaded Operators 5471 // ==================================== 5472 5473 // Overloaded operators won't compile if they are behind an anonymous namespace. 5474 } // {anonymous} 5475 5476 template<size_t W, typename T> static BL_INLINE_NODEBUG Vec<W, T> operator&(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return and_(a, b); } 5477 template<size_t W, typename T> static BL_INLINE_NODEBUG Vec<W, T> operator|(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return or_(a, b); } 5478 template<size_t W, typename T> static BL_INLINE_NODEBUG Vec<W, T> operator^(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return xor_(a, b); } 5479 template<size_t W, typename T> static BL_INLINE_NODEBUG Vec<W, T> operator+(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return add(a, b); } 5480 template<size_t W, typename T> static BL_INLINE_NODEBUG Vec<W, T> operator-(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return sub(a, b); } 5481 template<size_t W, typename T> static BL_INLINE_NODEBUG Vec<W, T> operator*(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return mul(a, b); } 5482 template<size_t W, typename T> static BL_INLINE_NODEBUG Vec<W, T> operator/(const Vec<W, T>& a, const Vec<W, T>& b) noexcept { return div(a, b); } 5483 5484 template<size_t W, typename T> static BL_INLINE_NODEBUG Vec<W, T>& operator&=(Vec<W, T>& a, const Vec<W, T>& b) noexcept { a = and_(a, b); return a; } 5485 template<size_t W, typename T> static BL_INLINE_NODEBUG Vec<W, T>& operator|=(Vec<W, T>& a, const Vec<W, T>& b) noexcept { a = or_(a, b); return a; } 5486 template<size_t W, typename T> static BL_INLINE_NODEBUG Vec<W, T>& operator^=(Vec<W, T>& a, const Vec<W, T>& b) noexcept { a = xor_(a, b); return a; } 5487 template<size_t W, typename T> static BL_INLINE_NODEBUG Vec<W, T>& operator+=(Vec<W, T>& a, const Vec<W, T>& b) noexcept { a = add(a, b); return a; } 5488 template<size_t W, typename T> static BL_INLINE_NODEBUG Vec<W, T>& operator-=(Vec<W, T>& a, const Vec<W, T>& b) noexcept { a = sub(a, b); return a; } 5489 template<size_t W, typename T> static BL_INLINE_NODEBUG Vec<W, T>& operator*=(Vec<W, T>& a, const Vec<W, T>& b) noexcept { a = mul(a, b); return a; } 5490 template<size_t W, typename T> static BL_INLINE_NODEBUG Vec<W, T>& operator/=(Vec<W, T>& a, const Vec<W, T>& b) noexcept { a = div(a, b); return a; } 5491 5492 template<size_t W, typename T, uint32_t kN> static BL_INLINE_NODEBUG Vec<W, T> operator<<(const Vec<W, T>& a, Shift<kN>) noexcept { return slli<kN>(a); } 5493 template<size_t W, typename T, uint32_t kN> static BL_INLINE_NODEBUG Vec<W, T> operator>>(const Vec<W, T>& a, Shift<kN>) noexcept { return std::is_unsigned_v<T> ? srli<kN>(a) : srai<kN>(a); } 5494 5495 template<size_t W, typename T, uint32_t kN> static BL_INLINE_NODEBUG Vec<W, T>& operator<<=(Vec<W, T>& a, Shift<kN> b) noexcept { a = a << b; return a; } 5496 template<size_t W, typename T, uint32_t kN> static BL_INLINE_NODEBUG Vec<W, T>& operator>>=(Vec<W, T>& a, Shift<kN> b) noexcept { a = a >> b; return a; } 5497 5498 namespace { 5499 5500 // SIMD - Public - Workarounds 5501 // =========================== 5502 5503 // TODO: These need a proper abstraction in Internal namespace. 5504 5505 BL_INLINE_NODEBUG Vec2xF64 cvt_2xi32_f64(const Vec4xI32& a) noexcept { return Vec2xF64{_mm_cvtepi32_pd(a.v)}; } 5506 BL_INLINE_NODEBUG Vec4xI32 cvtt_f64_i32(const Vec2xF64& a) noexcept { return Vec4xI32{_mm_cvttpd_epi32(a.v)}; } 5507 5508 #if defined(BL_TARGET_OPT_AVX2) 5509 BL_INLINE_NODEBUG Vec4xI32 cvtt_f64_i32(const Vec4xF64& a) noexcept { return Vec4xI32{_mm256_cvttpd_epi32(a.v)}; } 5510 #endif // BL_TARGET_OPT_AVX2 5511 5512 namespace Internal { 5513 5514 template<typename T> 5515 BL_INLINE_NODEBUG T simd_broadcast_u128(const __m128i& a) noexcept; 5516 5517 template<> BL_INLINE_NODEBUG __m128i simd_broadcast_u128(const __m128i& a) noexcept { return a; } 5518 5519 #if defined(BL_TARGET_OPT_AVX2) 5520 template<> BL_INLINE_NODEBUG __m256i simd_broadcast_u128(const __m128i& a) noexcept { return _mm256_broadcastsi128_si256(a); } 5521 #endif // BL_TARGET_OPT_AVX2 5522 5523 #if defined(BL_TARGET_OPT_AVX512) 5524 template<> BL_INLINE_NODEBUG __m512i simd_broadcast_u128(const __m128i& a) noexcept { return _mm512_broadcast_i32x4(a); } 5525 #endif // BL_TARGET_OPT_AVX512 5526 5527 } 5528 5529 #if defined(BL_TARGET_OPT_AVX) 5530 template<typename Dst, typename Src> 5531 BL_INLINE_NODEBUG Dst make256_128(const Src& hi, const Src& lo) noexcept { 5532 return from_simd<Dst>(I::simd_interleave_u128(simd_cast<__m128i>(lo.v), simd_cast<__m128i>(hi.v))); 5533 } 5534 5535 template<typename Dst, typename Src> 5536 BL_INLINE_NODEBUG Dst broadcast_i128(const Src& a) noexcept { 5537 return from_simd<Dst>(I::simd_broadcast_u128<typename Dst::SimdType>(simd_cast<__m128i>(a.v))); 5538 } 5539 5540 template<uint32_t kN, typename V> 5541 BL_INLINE_NODEBUG Vec<16, typename V::ElementType> extract_i128(const V& a) noexcept { 5542 return from_simd<Vec<16, typename V::ElementType>>( 5543 _mm256_extracti128_si256(a.v, kN)); 5544 } 5545 5546 BL_INLINE_NODEBUG Vec4xF64 cvt_4xi32_f64(const Vec4xI32& a) noexcept { return Vec4xF64{_mm256_cvtepi32_pd(a.v)}; } 5547 5548 #endif // BL_TARGET_OPT_AVX 5549 5550 #if defined(BL_TARGET_OPT_AVX2) 5551 5552 template<typename V, typename W> 5553 BL_INLINE_NODEBUG V loadu_256_mask32(const void* src, const W& msk) noexcept { 5554 return V{_mm256_maskload_epi32(static_cast<const int*>(src), simd_cast<__m256i>(msk.v))}; 5555 } 5556 5557 template<size_t W, typename T1, typename T2> 5558 BL_INLINE_NODEBUG void storeu_128_mask32(void* dst, const Vec<W, T1>& src, const Vec<W, T2>& msk) noexcept { 5559 _mm_maskstore_epi32(static_cast<int*>(dst), simd_cast<__m128i>(msk.v), simd_cast<__m128i>(src.v)); 5560 } 5561 5562 template<size_t W, typename T1, typename T2> 5563 BL_INLINE_NODEBUG void storeu_256_mask32(void* dst, const Vec<W, T1>& src, const Vec<W, T2>& msk) noexcept { 5564 _mm256_maskstore_epi32(static_cast<int*>(dst), simd_cast<__m256i>(msk.v), simd_cast<__m256i>(src.v)); 5565 } 5566 5567 #endif // BL_TARGET_OPT_AVX2 5568 5569 // SIMD - Public - Utilities - Div255 & Div65535 5570 // ============================================= 5571 5572 template<typename V> 5573 BL_INLINE_NODEBUG V div255_u16(const V& a) noexcept { 5574 V x = add_u16(a, make_u16<V>(0x80u)); 5575 return mulh_u16(x, make_u16<V>(0x0101u)); 5576 } 5577 5578 template<typename V> 5579 BL_INLINE_NODEBUG V div65535_u32(const V& a) noexcept { 5580 V x = add_u32(a, make_u32<V>(0x8000u)); 5581 return srli_u32<16>(add_i32(x, srli_u32<16>(x))); 5582 } 5583 5584 // SIMD - Public - Utilities - Mask Extraction 5585 // =========================================== 5586 5587 // SIMD - Public - Utilities - Array Lookup 5588 // ======================================== 5589 5590 template<uint32_t kN> 5591 struct ArrayLookupResult { 5592 uint32_t _mask; 5593 5594 BL_INLINE_NODEBUG bool matched() const noexcept { return _mask != 0; } 5595 BL_INLINE_NODEBUG uint32_t index() const noexcept { return bl::IntOps::ctz(_mask); } 5596 5597 using Iterator = bl::ParametrizedBitOps<bl::BitOrder::kLSB, uint32_t>::BitIterator; 5598 BL_INLINE_NODEBUG Iterator iterate() const noexcept { return Iterator(_mask); } 5599 }; 5600 5601 template<> 5602 struct ArrayLookupResult<64> { 5603 uint64_t _mask; 5604 5605 BL_INLINE_NODEBUG bool matched() const noexcept { return _mask != 0; } 5606 BL_INLINE_NODEBUG uint32_t index() const noexcept { return bl::IntOps::ctz(_mask); } 5607 5608 using Iterator = bl::ParametrizedBitOps<bl::BitOrder::kLSB, uint64_t>::BitIterator; 5609 BL_INLINE_NODEBUG Iterator iterate() const noexcept { return Iterator(_mask); } 5610 }; 5611 5612 BL_INLINE_NODEBUG ArrayLookupResult<4> array_lookup_result_from_4x_u32(Vec4xU32 pred) noexcept { 5613 return ArrayLookupResult<4u>{extract_sign_bits_i32(pred)}; 5614 } 5615 5616 #if defined(BL_TARGET_OPT_AVX2) 5617 BL_INLINE_NODEBUG ArrayLookupResult<8> array_lookup_result_from_8x_u32(Vec8xU32 pred) noexcept { 5618 return ArrayLookupResult<8u>{extract_sign_bits_i32(pred)}; 5619 } 5620 #endif // BL_TARGET_OPT_AVX2 5621 5622 BL_INLINE_NODEBUG ArrayLookupResult<8> array_lookup_result_from_8x_u16(Vec8xU16 pred) noexcept { 5623 return ArrayLookupResult<8u>{extract_sign_bits_i8(packs_128_i16_i8(pred, make_zero<Vec8xU16>()))}; 5624 } 5625 5626 BL_INLINE_NODEBUG ArrayLookupResult<16> array_lookup_result_from_16x_u8(Vec16xU8 pred) noexcept { 5627 return ArrayLookupResult<16u>{extract_sign_bits_i8(pred)}; 5628 } 5629 5630 BL_INLINE_NODEBUG ArrayLookupResult<32> array_lookup_result_from_32x_u8(Vec16xU8 pred0, Vec16xU8 pred1) noexcept { 5631 return ArrayLookupResult<32u>{extract_sign_bits_i8(pred0, pred1)}; 5632 } 5633 5634 #if defined(BL_TARGET_OPT_AVX2) 5635 BL_INLINE_NODEBUG ArrayLookupResult<32> array_lookup_result_from_32x_u8(Vec32xU8 pred0) noexcept { 5636 return ArrayLookupResult<32u>{extract_sign_bits_i8(pred0)}; 5637 } 5638 #endif // BL_TARGET_OPT_AVX2 5639 5640 BL_INLINE_NODEBUG ArrayLookupResult<64> array_lookup_result_from_64x_u8(Vec16xU8 pred0, Vec16xU8 pred1, Vec16xU8 pred2, Vec16xU8 pred3) noexcept { 5641 return ArrayLookupResult<64u>{extract_sign_bits_i8(pred0, pred1, pred2, pred3)}; 5642 } 5643 5644 #if defined(BL_TARGET_OPT_AVX2) 5645 BL_INLINE_NODEBUG ArrayLookupResult<64> array_lookup_result_from_64x_u8(Vec32xU8 pred0, Vec32xU8 pred1) noexcept { 5646 return ArrayLookupResult<64u>{extract_sign_bits_i8(pred0, pred1)}; 5647 } 5648 #endif // BL_TARGET_OPT_AVX2 5649 5650 template<uint32_t kN> 5651 BL_INLINE_NODEBUG ArrayLookupResult<kN> array_lookup_u32_eq_aligned16(const uint32_t* array, uint32_t value) noexcept; 5652 5653 template<> 5654 BL_INLINE_NODEBUG ArrayLookupResult<4u> array_lookup_u32_eq_aligned16<4u>(const uint32_t* array, uint32_t value) noexcept { 5655 return array_lookup_result_from_4x_u32(cmp_eq_u32(loada<Vec4xU32>(array), make_u32<Vec4xU32>(value))); 5656 } 5657 5658 template<> 5659 BL_INLINE_NODEBUG ArrayLookupResult<8u> array_lookup_u32_eq_aligned16<8u>(const uint32_t* array, uint32_t value) noexcept { 5660 #if defined(BL_TARGET_OPT_AVX2) 5661 return array_lookup_result_from_8x_u32(cmp_eq_u32(loadu<Vec8xU32>(array), make_u32<Vec8xU32>(value))); 5662 #else 5663 Vec4xU32 v = make_u32<Vec4xU32>(value); 5664 Vec4xU32 m0 = cmp_eq_u32(loada<Vec4xU32>(array + 0), v); 5665 Vec4xU32 m1 = cmp_eq_u32(loada<Vec4xU32>(array + 4), v); 5666 Vec4xU32 m = packs_128_i32_i16(m0, m1); 5667 return array_lookup_result_from_8x_u16(vec_cast<Vec8xU16>(m)); 5668 #endif 5669 } 5670 5671 template<> 5672 BL_INLINE_NODEBUG ArrayLookupResult<16u> array_lookup_u32_eq_aligned16<16u>(const uint32_t* array, uint32_t value) noexcept { 5673 #if defined(BL_TARGET_OPT_AVX512) 5674 Vec16xU32 v = make_u32<Vec16xU32>(value); 5675 return ArrayLookupResult<16u>{_cvtmask16_u32(_mm512_cmpeq_epi32_mask(loadu<Vec16xU32>(array).v, v.v))}; 5676 #elif defined(BL_TARGET_OPT_AVX2) 5677 Vec8xU32 v = make_u32<Vec8xU32>(value); 5678 Vec8xU32 m0 = cmp_eq_u32(loadu<Vec8xU32>(array + 0), v); 5679 Vec8xU32 m1 = cmp_eq_u32(loadu<Vec8xU32>(array + 8), v); 5680 uint32_t i0 = extract_sign_bits_i32(m0); 5681 uint32_t i1 = extract_sign_bits_i32(m1); 5682 return ArrayLookupResult<16u>{i0 + (i1 << 8)}; 5683 #else 5684 Vec4xU32 v = make_u32<Vec4xU32>(value); 5685 Vec4xU32 m0 = cmp_eq_u32(loada<Vec4xU32>(array + 0), v); 5686 Vec4xU32 m1 = cmp_eq_u32(loada<Vec4xU32>(array + 4), v); 5687 Vec4xU32 m2 = cmp_eq_u32(loada<Vec4xU32>(array + 8), v); 5688 Vec4xU32 m3 = cmp_eq_u32(loada<Vec4xU32>(array + 12), v); 5689 return array_lookup_result_from_16x_u8(vec_cast<Vec16xU8>(packs_128_i32_i8(m0, m1, m2, m3))); 5690 #endif 5691 } 5692 5693 } // {anonymous} 5694 } // {SIMD} 5695 5696 #undef I 5697 5698 #if defined(BL_SIMD_MISSING_INTRINSICS) 5699 #undef BL_SIMD_MISSING_INTRINSICS 5700 #endif // BL_SIMD_MISSING_INTRINSICS 5701 5702 //! \} 5703 //! \endcond 5704 5705 #endif // BLEND2D_SIMD_SIMDX86_P_H_INCLUDED