unicompiler_a64.cpp (209823B)
1 // This file is part of AsmJit project <https://asmjit.com> 2 // 3 // See <asmjit/core.h> or LICENSE.md for license and copyright information 4 // SPDX-License-Identifier: Zlib 5 6 #include "../core/api-build_p.h" 7 #include "ujitbase.h" 8 9 #if defined(ASMJIT_UJIT_AARCH64) 10 11 #include "unicompiler.h" 12 #include "unicompiler_utils_p.h" 13 #include "unicondition.h" 14 15 ASMJIT_BEGIN_SUB_NAMESPACE(ujit) 16 17 using GPExt = UniCompiler::GPExt; 18 using ASIMDExt = UniCompiler::ASIMDExt; 19 namespace Inst { using namespace a64::Inst; } 20 21 // ujit::UniCompiler - Construction & Destruction 22 // ============================================== 23 24 UniCompiler::UniCompiler(BackendCompiler* cc, const CpuFeatures& features, CpuHints cpu_hints, VecConstTableRef ct_ref) noexcept 25 : cc(cc), 26 _ct_ref(ct_ref), 27 _features(features), 28 _cpu_hints(cpu_hints), 29 _vec_reg_count(32), 30 _common_table_offset(0) { 31 32 _scalar_op_behavior = ScalarOpBehavior::kZeroing; 33 _fmin_fmax_op_behavior = FMinFMaxOpBehavior::kFiniteValue; 34 _fmadd_op_behavior = FMAddOpBehavior::kFMAStoreToAccumulator; 35 _float_to_int_outside_range_behavior = FloatToIntOutsideRangeBehavior::kSaturatedValue; 36 37 _init_extensions(features); 38 } 39 40 UniCompiler::~UniCompiler() noexcept {} 41 42 // ujit::UniCompiler - CPU Architecture, Features and Optimization Options 43 // ======================================================================= 44 45 void UniCompiler::_init_extensions(const CpuFeatures& features) noexcept { 46 uint64_t gp_ext_mask = 0; 47 uint64_t asimd_ext_mask = 0; 48 49 if (features.arm().has_cssc() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kCSSC); 50 if (features.arm().has_flagm() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kFLAGM); 51 if (features.arm().has_flagm2() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kFLAGM2); 52 if (features.arm().has_ls64() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kLS64); 53 if (features.arm().has_ls64_v() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kLS64_V); 54 if (features.arm().has_lse() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kLSE); 55 if (features.arm().has_lse128() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kLSE128); 56 if (features.arm().has_lse2() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kLSE2); 57 58 if (features.arm().has_asimd() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kASIMD); 59 if (features.arm().has_bf16() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kBF16); 60 if (features.arm().has_dotprod() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kDOTPROD); 61 if (features.arm().has_fcma() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kFCMA); 62 if (features.arm().has_fhm() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kFHM); 63 if (features.arm().has_fp16() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kFP16); 64 if (features.arm().has_fp16conv()) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kFP16CONV); 65 if (features.arm().has_fp8() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kFP8); 66 if (features.arm().has_frintts() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kFRINTTS); 67 if (features.arm().has_i8mm() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kI8MM); 68 if (features.arm().has_jscvt() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kJSCVT); 69 if (features.arm().has_pmull() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kPMULL); 70 if (features.arm().has_rdm() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kRDM); 71 if (features.arm().has_sha1() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kSHA1); 72 if (features.arm().has_sha256() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kSHA256); 73 if (features.arm().has_sha3() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kSHA3); 74 if (features.arm().has_sha512() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kSHA512); 75 if (features.arm().has_sm3() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kSM3); 76 if (features.arm().has_sm4() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kSM4); 77 78 _gp_ext_mask = gp_ext_mask; 79 _asimd_ext_mask = asimd_ext_mask; 80 } 81 82 VecWidth UniCompiler::max_vec_width_from_cpu_features() noexcept { 83 return VecWidth::k128; 84 } 85 86 void UniCompiler::init_vec_width(VecWidth vw) noexcept { 87 ASMJIT_ASSERT(vw == VecWidth::k128); 88 Support::maybe_unused(vw); 89 90 _vec_width = VecWidth::k128; 91 _vec_reg_type = RegType::kVec128; 92 _vec_type_id = TypeId::kInt32x4; 93 _vec_multiplier = 1u; 94 } 95 96 bool UniCompiler::has_masked_access_of(uint32_t data_size) const noexcept { 97 switch (data_size) { 98 case 1: return has_cpu_hint(CpuHints::kVecMaskedOps8); 99 case 2: return has_cpu_hint(CpuHints::kVecMaskedOps16); 100 case 4: return has_cpu_hint(CpuHints::kVecMaskedOps32); 101 case 8: return has_cpu_hint(CpuHints::kVecMaskedOps64); 102 103 default: 104 return false; 105 } 106 } 107 108 // ujit::UniCompiler - Embed 109 // ========================= 110 111 void UniCompiler::embed_jump_table(Span<const Label> jump_table, const Label& jump_table_base, uint32_t entry_size) { 112 static const uint8_t zeros[8] {}; 113 114 for (const Label& label : jump_table) { 115 if (label.is_valid()) { 116 cc->embed_label_delta(label, jump_table_base, entry_size); 117 } 118 else { 119 cc->embed(zeros, entry_size); 120 } 121 } 122 } 123 124 // ujit::UniCompiler - Function 125 // ============================ 126 127 void UniCompiler::hook_func() noexcept { 128 FuncNode* func = cc->func(); 129 _func_init_hook = func; 130 } 131 132 void UniCompiler::unhook_func() noexcept { 133 _func_init_hook = nullptr; 134 } 135 136 // ujit::UniCompiler - Constants 137 // ============================= 138 139 void UniCompiler::_init_vec_const_table_ptr() { 140 const void* global = ct_ptr<void>(); 141 142 if (!_common_table_ptr.is_valid()) { 143 ScopedInjector injector(cc, &_func_init_hook); 144 _common_table_ptr = new_gpz("common_table_ptr"); 145 cc->mov(_common_table_ptr, (int64_t)global + _common_table_offset); 146 } 147 } 148 149 Operand UniCompiler::simd_const(const void* c, Bcst bcst_width, VecWidth const_width) { 150 return simd_vec_const(c, bcst_width, const_width); 151 } 152 153 Operand UniCompiler::simd_const(const void* c, Bcst bcst_width, const Vec& similar_to) { 154 Support::maybe_unused(similar_to); 155 return simd_vec_const(c, bcst_width, VecWidth::k128); 156 } 157 158 Operand UniCompiler::simd_const(const void* c, Bcst bcst_width, const VecArray& similar_to) { 159 ASMJIT_ASSERT(!similar_to.is_empty()); 160 Support::maybe_unused(bcst_width, similar_to); 161 162 return simd_vec_const(c, bcst_width, VecWidth::k128); 163 } 164 165 Vec UniCompiler::simd_vec_const(const void* c, Bcst bcst_width, VecWidth const_width) { 166 Support::maybe_unused(bcst_width); 167 Support::maybe_unused(const_width); 168 169 size_t n = _vec_consts.size(); 170 for (size_t i = 0; i < n; i++) { 171 if (_vec_consts[i].ptr == c) { 172 return Vec(OperandSignature{RegTraits<RegType::kVec128>::kSignature}, _vec_consts[i].virt_reg_id); 173 } 174 } 175 176 return Vec(OperandSignature{RegTraits<RegType::kVec128>::kSignature}, _new_vec_const(c, true).id()); 177 } 178 179 Vec UniCompiler::simd_vec_const(const void* c, Bcst bcst_width, const Vec& similar_to) { 180 Support::maybe_unused(similar_to); 181 return simd_vec_const(c, bcst_width, VecWidth::k128); 182 } 183 184 Vec UniCompiler::simd_vec_const(const void* c, Bcst bcst_width, const VecArray& similar_to) { 185 Support::maybe_unused(similar_to); 186 return simd_vec_const(c, bcst_width, VecWidth::k128); 187 } 188 189 Mem UniCompiler::simd_mem_const(const void* c, Bcst bcst_width, VecWidth const_width) { 190 Support::maybe_unused(bcst_width, const_width); 191 return _get_mem_const(c); 192 } 193 194 Mem UniCompiler::simd_mem_const(const void* c, Bcst bcst_width, const Vec& similar_to) { 195 Support::maybe_unused(bcst_width, similar_to); 196 return _get_mem_const(c); 197 } 198 199 Mem UniCompiler::simd_mem_const(const void* c, Bcst bcst_width, const VecArray& similar_to) { 200 Support::maybe_unused(bcst_width, similar_to); 201 return _get_mem_const(c); 202 } 203 204 Mem UniCompiler::_get_mem_const(const void* c) { 205 // Make sure we are addressing a constant from the `ct` constant pool. 206 const void* ct_addr = ct_ptr<void>(); 207 ASMJIT_ASSERT((uintptr_t)c >= (uintptr_t)ct_addr && 208 (uintptr_t)c < (uintptr_t)ct_addr + _ct_ref.size); 209 210 // One GP register is sacrificed to hold the pointer to the `ct`. 211 _init_vec_const_table_ptr(); 212 213 int32_t disp = int32_t((intptr_t)c - (intptr_t)ct_addr); 214 return mem_ptr(_common_table_ptr, disp - _common_table_offset); 215 } 216 217 Vec UniCompiler::_new_vec_const(const void* c, bool is_unique_const) { 218 Support::maybe_unused(is_unique_const); 219 220 Vec vec; 221 const char* special_const_name = nullptr; 222 223 if (special_const_name) { 224 vec = new_vec_with_width(vec_width(), special_const_name); 225 } 226 else { 227 uint64_t u0 = static_cast<const uint64_t*>(c)[0]; 228 uint64_t u1 = static_cast<const uint64_t*>(c)[1]; 229 230 if (u0 != u1) 231 vec = new_vec_with_width(vec_width(), "c_0x%016llX%016llX", (unsigned long long)u1, (unsigned long long)u0); 232 else if ((u0 >> 32) != (u0 & 0xFFFFFFFFu)) 233 vec = new_vec_with_width(vec_width(), "c_0x%016llX", (unsigned long long)u0); 234 else if (((u0 >> 16) & 0xFFFFu) != (u0 & 0xFFFFu)) 235 vec = new_vec_with_width(vec_width(), "c_0x%08X", (unsigned)(u0 & 0xFFFFFFFFu)); 236 else 237 vec = new_vec_with_width(vec_width(), "c_0x%04X", (unsigned)(u0 & 0xFFFFu)); 238 } 239 240 VecConstData const_data; 241 const_data.ptr = c; 242 const_data.virt_reg_id = vec.id(); 243 _vec_consts.append(arena(), const_data); 244 245 if (c == &ct().p_0000000000000000) { 246 ScopedInjector inject(cc, &_func_init_hook); 247 v_zero_i(vec.v128()); 248 } 249 else { 250 // NOTE: _get_mem_const() must be outside of injected code as it uses injection too. 251 Mem m = _get_mem_const(c); 252 253 ScopedInjector inject(cc, &_func_init_hook); 254 v_loadavec(vec, m); 255 } 256 257 return vec; 258 } 259 260 Vec UniCompiler::simd_const_16b(const void* data16) { 261 size_t n = _vec_consts_ex.size(); 262 263 for (size_t i = 0; i < n; i++) { 264 if (memcmp(_vec_consts_ex[i].data, data16, 16) == 0) { 265 return Vec(OperandSignature{RegTraits<RegType::kVec128>::kSignature}, _vec_consts_ex[i].virt_reg_id); 266 } 267 } 268 269 Vec vec = new_vec128("const"); 270 VecConstDataEx entry; 271 272 memcpy(entry.data, data16, 16); 273 entry.virt_reg_id = vec.id(); 274 _vec_consts_ex.append(arena(), entry); 275 276 Mem mem = cc->new_const(ConstPoolScope::kLocal, data16, 16); 277 { 278 ScopedInjector inject(cc, &_func_init_hook); 279 v_loadavec(vec, mem); 280 } 281 282 return vec; 283 } 284 285 // ujit::UniCompiler - Stack 286 // ========================= 287 288 Mem UniCompiler::tmp_stack(StackId id, uint32_t size) { 289 ASMJIT_ASSERT(Support::is_power_of_2(size)); 290 ASMJIT_ASSERT(size <= 32); 291 292 // Only used by asserts. 293 Support::maybe_unused(size); 294 295 Mem& stack = _tmp_stack[size_t(id)]; 296 if (!stack.base_id()) { 297 stack = cc->new_stack(32, 16, "tmp_stack"); 298 } 299 return stack; 300 } 301 302 // ujit::UniCompiler - General Purpose Instructions - Utilities 303 // ============================================================ 304 305 struct MemInst { 306 uint16_t inst_id; 307 uint16_t mem_size; 308 }; 309 310 static ASMJIT_NOINLINE void gp_emit_mem_op(UniCompiler& uc, Gp r, Mem m, MemInst ii) { 311 BackendCompiler* cc = uc.cc; 312 InstId inst_id = ii.inst_id; 313 314 if (m.has_index() && m.has_shift()) { 315 // AArch64 limitation: shift can be the same size as the size of the read operation - HWord << 1, Word << 2, etc... 316 // Other shift operations are not supported at the architectural level, so we have to perform it explicitly. 317 uint32_t mem_size = ii.mem_size ? uint32_t(ii.mem_size) : r.size(); 318 uint32_t shift = m.shift(); 319 320 if (mem_size != (1u << shift)) { 321 Gp tmp = uc.new_gpz("@mem_addr"); 322 cc->add(tmp, m.base_reg().as<Gp>(), m.index_reg().as<Gp>(), a64::Shift(m.shift_op(), shift)); 323 m = a64::ptr(tmp); 324 } 325 } 326 327 cc->emit(inst_id, r, m); 328 } 329 330 static constexpr Gp gp_zero_regs[2] = { a64::wzr, a64::xzr }; 331 332 static ASMJIT_INLINE const Gp& gp_zero_as(const Gp& ref) noexcept { 333 return gp_zero_regs[size_t(ref.is_gp64())]; 334 } 335 336 static ASMJIT_NOINLINE Gp gp_force_reg(UniCompiler& uc, const Operand_& op, const Gp& ref) { 337 ASMJIT_ASSERT(op.is_gp() || op.is_mem() || op.is_imm()); 338 339 Gp reg; 340 341 if (op.is_gp()) { 342 reg = op.as<Gp>(); 343 reg.set_signature(ref.signature()); 344 return reg; 345 } 346 347 if (op.is_imm() && op.as<Imm>().value() == 0) { 348 return gp_zero_as(ref); 349 } 350 351 BackendCompiler* cc = uc.cc; 352 reg = uc.new_similar_reg(ref, "@tmp"); 353 354 if (op.is_mem()) { 355 gp_emit_mem_op(uc, reg, op.as<Mem>(), MemInst{uint16_t(Inst::kIdLdr), uint16_t(reg.size())}); 356 } 357 else { 358 cc->mov(reg, op.as<Imm>()); 359 } 360 return reg; 361 } 362 363 // ujit::UniCompiler - General Purpose Instructions - Conditions 364 // ============================================================= 365 366 struct ConditionOpInfo { 367 uint16_t inst_id; 368 uint16_t reserved; 369 }; 370 371 static constexpr ConditionOpInfo condition_op_info[size_t(UniOpCond::kMaxValue) + 1] = { 372 { Inst::kIdAnds, 0 }, // UniOpCond::kAssignAnd 373 { Inst::kIdOrr , 0 }, // UniOpCond::kAssignOr 374 { Inst::kIdEor , 0 }, // UniOpCond::kAssignXor 375 { Inst::kIdAdds, 0 }, // UniOpCond::kAssignAdd 376 { Inst::kIdSubs, 0 }, // UniOpCond::kAssignSub 377 { Inst::kIdNone, 0 }, // UniOpCond::kAssignShr 378 { Inst::kIdTst , 0 }, // UniOpCond::kTest 379 { Inst::kIdNone, 0 }, // UniOpCond::kBitTest 380 { Inst::kIdCmp , 0 } // UniOpCond::kCompare 381 }; 382 383 class ConditionApplier : public UniCondition { 384 public: 385 ASMJIT_INLINE ConditionApplier(const UniCondition& condition) noexcept : UniCondition(condition) { 386 // The first operand must always be a register. 387 ASMJIT_ASSERT(a.is_gp()); 388 } 389 390 ASMJIT_NOINLINE void optimize(UniCompiler& uc) noexcept { 391 Support::maybe_unused(uc); 392 393 switch (op) { 394 case UniOpCond::kCompare: 395 if (b.is_imm() && b.as<Imm>().value() == 0 && (cond == CondCode::kEqual || cond == CondCode::kNotEqual)) { 396 op = UniOpCond::kTest; 397 b = a; 398 reverse(); 399 } 400 break; 401 402 case UniOpCond::kBitTest: { 403 if (b.is_imm()) { 404 uint64_t bit_index = b.as<Imm>().value_as<uint64_t>(); 405 op = UniOpCond::kTest; 406 b = Imm(1u << bit_index); 407 } 408 break; 409 } 410 411 default: 412 break; 413 } 414 } 415 416 ASMJIT_INLINE void reverse() noexcept { 417 cond = a64::reverse_cond(cond); 418 } 419 420 ASMJIT_NOINLINE void emit(UniCompiler& uc) { 421 BackendCompiler* cc = uc.cc; 422 ConditionOpInfo info = condition_op_info[size_t(op)]; 423 424 Gp aGp = a.as<Gp>(); 425 426 switch (op) { 427 case UniOpCond::kAssignAnd: { 428 if (b.is_imm() && a64::Utils::is_logical_imm(b.as<Imm>().value_as<uint64_t>(), aGp.size() * 8)) { 429 cc->emit(info.inst_id, aGp, aGp, b.as<Imm>()); 430 } 431 else { 432 cc->emit(info.inst_id, aGp, aGp, gp_force_reg(uc, b, aGp)); 433 } 434 return; 435 } 436 437 case UniOpCond::kAssignAdd: 438 case UniOpCond::kAssignSub: { 439 if (b.is_imm() && a64::Utils::is_add_sub_imm(b.as<Imm>().value_as<uint64_t>())) { 440 cc->emit(info.inst_id, aGp, aGp, b.as<Imm>()); 441 } 442 else { 443 cc->emit(info.inst_id, aGp, aGp, gp_force_reg(uc, b, aGp)); 444 } 445 return; 446 } 447 448 case UniOpCond::kAssignXor: 449 if (b.is_imm()) { 450 const Imm& bImm = b.as<Imm>(); 451 if (bImm.value() == -1 || (aGp.size() == 4 && bImm.value_as<uint32_t>() == 0xFFFFFFFFu)) { 452 cc->mvn_(aGp, aGp); 453 cc->tst(aGp, aGp); 454 return; 455 } 456 } 457 [[fallthrough]]; 458 459 case UniOpCond::kAssignOr: { 460 if (b.is_imm() && a64::Utils::is_logical_imm(b.as<Imm>().value_as<uint64_t>(), aGp.size() * 8)) { 461 cc->emit(info.inst_id, aGp, aGp, b.as<Imm>()); 462 cc->tst(aGp, aGp); 463 } 464 else { 465 cc->emit(info.inst_id, aGp, aGp, gp_force_reg(uc, b, aGp)); 466 cc->tst(aGp, aGp); 467 } 468 return; 469 } 470 471 case UniOpCond::kAssignShr: { 472 if (b.is_imm()) { 473 cc->adds(aGp, gp_zero_as(aGp), aGp, a64::lsr(b.as<Imm>().value_as<uint32_t>())); 474 } 475 else { 476 cc->lsr(aGp, aGp, gp_force_reg(uc, b, aGp)); 477 cc->tst(aGp, aGp); 478 } 479 return; 480 } 481 482 case UniOpCond::kTest: { 483 if (b.is_imm() && a64::Utils::is_logical_imm(b.as<Imm>().value_as<uint64_t>(), aGp.size() * 8)) { 484 cc->emit(info.inst_id, aGp, b.as<Imm>()); 485 } 486 else { 487 cc->emit(info.inst_id, aGp, gp_force_reg(uc, b, aGp)); 488 } 489 return; 490 } 491 492 case UniOpCond::kCompare: { 493 if (b.is_imm() && a64::Utils::is_add_sub_imm(b.as<Imm>().value_as<uint64_t>())) { 494 cc->emit(info.inst_id, aGp, b.as<Imm>()); 495 } 496 else { 497 cc->emit(info.inst_id, aGp, gp_force_reg(uc, b, aGp)); 498 } 499 return; 500 } 501 502 case UniOpCond::kBitTest: { 503 Gp tmp = uc.new_similar_reg(aGp); 504 cc->lsr(tmp, aGp, gp_force_reg(uc, b, aGp)); 505 cc->tst(tmp, Imm(1)); 506 return; 507 } 508 509 default: 510 ASMJIT_NOT_REACHED(); 511 } 512 } 513 }; 514 515 // ujit::UniCompiler - General Purpose Instructions - Emit 516 // ======================================================= 517 518 void UniCompiler::emit_mov(const Gp& dst, const Operand_& src) { 519 if (src.is_mem()) { 520 gp_emit_mem_op(*this, dst, src.as<Mem>(), MemInst{uint16_t(Inst::kIdLdr), uint16_t(dst.size())}); 521 } 522 else { 523 cc->emit(Inst::kIdMov, dst, src); 524 } 525 } 526 527 void UniCompiler::emit_m(UniOpM op, const Mem& m_) { 528 static constexpr MemInst st_inst[] = { 529 { Inst::kIdNone, 0 }, // kPrefetch 530 { Inst::kIdStr , 0 }, // kStoreZeroReg 531 { Inst::kIdStrb, 1 }, // kStoreZeroU8 532 { Inst::kIdStrh, 2 }, // kStoreZeroU16 533 { Inst::kIdStr , 4 }, // kStoreZeroU32 534 { Inst::kIdStr , 8 } // kStoreZeroU64 535 }; 536 537 Gp zero = gp_zero_regs[size_t(op == UniOpM::kStoreZeroReg || op == UniOpM::kStoreZeroU64)]; 538 MemInst ii = st_inst[size_t(op)]; 539 540 if (!ii.inst_id) { 541 return; 542 } 543 544 gp_emit_mem_op(*this, zero, m_, ii); 545 } 546 547 void UniCompiler::emit_rm(UniOpRM op, const Gp& dst, const Mem& src) { 548 static constexpr MemInst ld_inst[] = { 549 { Inst::kIdLdr , 0 }, // kLoadReg 550 { Inst::kIdLdrsb, 1 }, // kLoadI8 551 { Inst::kIdLdrb , 1 }, // kLoadU8 552 { Inst::kIdLdrsh, 2 }, // kLoadI16 553 { Inst::kIdLdrh , 2 }, // kLoadU16 554 { Inst::kIdLdr , 4 }, // kLoadI32 555 { Inst::kIdLdr , 4 }, // kLoadU32 556 { Inst::kIdLdr , 8 }, // kLoadI64 557 { Inst::kIdLdr , 8 }, // kLoadU64 558 { Inst::kIdLdrb , 1 }, // kLoadMergeU8 559 { Inst::kIdLdrb , 1 }, // kLoadShiftU8 560 { Inst::kIdLdrh , 2 }, // kLoadMergeU16 561 { Inst::kIdLdrh , 2 } // kLoadShiftU16 562 }; 563 564 static constexpr uint32_t ld_32_mask = 565 (1u << uint32_t(uint32_t(UniOpRM::kLoadU8 ))) | 566 (1u << uint32_t(uint32_t(UniOpRM::kLoadU16))) | 567 (1u << uint32_t(uint32_t(UniOpRM::kLoadU32))) ; 568 569 Gp r(dst); 570 Mem m(src); 571 MemInst ii = ld_inst[size_t(op)]; 572 573 switch (op) { 574 case UniOpRM::kLoadReg: 575 case UniOpRM::kLoadI8: 576 case UniOpRM::kLoadU8: 577 case UniOpRM::kLoadI16: 578 case UniOpRM::kLoadU16: 579 case UniOpRM::kLoadI32: 580 case UniOpRM::kLoadU32: 581 case UniOpRM::kLoadI64: 582 case UniOpRM::kLoadU64: { 583 if (op == UniOpRM::kLoadI32 && dst.is_gp64()) { 584 ii.inst_id = uint16_t(Inst::kIdLdrsw); 585 } 586 587 if ((ld_32_mask >> uint32_t(op)) & 1u) { 588 r = r.w(); 589 } 590 591 gp_emit_mem_op(*this, r, m, ii); 592 return; 593 } 594 595 case UniOpRM::kLoadShiftU8: 596 case UniOpRM::kLoadShiftU16: { 597 Gp tmp = new_similar_reg(r); 598 gp_emit_mem_op(*this, tmp.r32(), m, ii); 599 cc->orr(r, tmp, r, a64::lsl(ii.mem_size * 8)); 600 return; 601 } 602 603 case UniOpRM::kLoadMergeU8: 604 case UniOpRM::kLoadMergeU16: { 605 Gp tmp = new_similar_reg(r); 606 gp_emit_mem_op(*this, tmp.r32(), m, ii); 607 cc->orr(r, r, tmp); 608 return; 609 } 610 611 default: { 612 ASMJIT_NOT_REACHED(); 613 } 614 } 615 } 616 617 struct UniOpMRInfo { 618 uint32_t op_inst : 14; 619 uint32_t store_inst : 14; 620 uint32_t size : 4; 621 }; 622 623 void UniCompiler::emit_mr(UniOpMR op, const Mem& dst, const Gp& src) { 624 static constexpr UniOpMRInfo op_info_table[] = { 625 { Inst::kIdNone, Inst::kIdStr , 0 }, // kStoreReg 626 { Inst::kIdNone, Inst::kIdStrb, 1 }, // kStoreU8 627 { Inst::kIdNone, Inst::kIdStrh, 2 }, // kStoreU16 628 { Inst::kIdNone, Inst::kIdStr , 4 }, // kStoreU32 629 { Inst::kIdNone, Inst::kIdStr , 8 }, // kStoreU64 630 { Inst::kIdAdd , Inst::kIdStr , 0 }, // kAddReg 631 { Inst::kIdAdd , Inst::kIdStrb, 1 }, // kAddU8 632 { Inst::kIdAdd , Inst::kIdStrh, 2 }, // kAddU16 633 { Inst::kIdAdd , Inst::kIdStr , 4 }, // kAddU32 634 { Inst::kIdAdd , Inst::kIdStr , 8 } // kAddU64 635 }; 636 637 Mem m(dst); 638 Gp r(src); 639 const UniOpMRInfo& op_info = op_info_table[size_t(op)]; 640 641 if (op_info.size >= 1u && op_info.size <= 4) { 642 r = r.w(); 643 } 644 645 if (op_info.op_inst == Inst::kIdNone) { 646 cc->emit(op_info.store_inst, r, m); 647 } 648 else { 649 Gp tmp = new_similar_reg(r, "@tmp"); 650 switch (op_info.size) { 651 case 0: load(tmp, m); break; 652 case 1: load_u8(tmp, m); break; 653 case 2: load_u16(tmp, m); break; 654 case 4: load_u32(tmp, m); break; 655 case 8: load_u64(tmp, m); break; 656 } 657 cc->emit(op_info.op_inst, tmp, tmp, r); 658 cc->emit(op_info.store_inst, tmp, m); 659 } 660 } 661 662 void UniCompiler::emit_cmov(const Gp& dst, const Operand_& sel, const UniCondition& condition) { 663 ConditionApplier ca(condition); 664 ca.optimize(*this); 665 ca.emit(*this); 666 cc->csel(dst, gp_force_reg(*this, sel, dst), dst, condition.cond); 667 } 668 669 void UniCompiler::emit_select(const Gp& dst, const Operand_& sel1_, const Operand_& sel2_, const UniCondition& condition) { 670 ConditionApplier ca(condition); 671 ca.optimize(*this); 672 ca.emit(*this); 673 674 Gp sel1 = gp_force_reg(*this, sel1_, dst); 675 Gp sel2 = gp_force_reg(*this, sel2_, dst); 676 cc->csel(dst, sel1, sel2, condition.cond); 677 } 678 679 void UniCompiler::emit_2i(UniOpRR op, const Gp& dst, const Operand_& src_) { 680 // ArithOp Reg, Any 681 // ---------------- 682 683 if (src_.is_reg_or_mem()) { 684 Gp src = gp_force_reg(*this, src_, dst); 685 686 switch (op) { 687 case UniOpRR::kAbs: { 688 if (has_cssc()) { 689 cc->abs(dst, src); 690 } 691 else { 692 cc->cmp(src, 0); 693 cc->cneg(dst, src, CondCode::kMI); 694 } 695 return; 696 } 697 698 case UniOpRR::kNeg: { 699 cc->neg(dst, src); 700 return; 701 } 702 703 case UniOpRR::kNot: { 704 cc->mvn_(dst, src); 705 return; 706 } 707 708 case UniOpRR::kBSwap: { 709 cc->rev(dst, src); 710 return; 711 } 712 713 case UniOpRR::kCLZ: { 714 cc->clz(dst, src); 715 return; 716 } 717 718 case UniOpRR::kCTZ: { 719 if (has_cssc()) { 720 cc->ctz(dst, src); 721 } 722 else { 723 cc->rbit(dst, src); 724 cc->clz(dst, dst); 725 } 726 return; 727 } 728 729 case UniOpRR::kReflect: { 730 cc->eor(dst, src, src, a64::asr(dst.size() * 8u - 1)); 731 return; 732 } 733 734 default: 735 ASMJIT_NOT_REACHED(); 736 } 737 } 738 739 // Everything should be handled, so this should never be reached! 740 ASMJIT_NOT_REACHED(); 741 } 742 743 static constexpr uint64_t kOp3ICommutativeMask = 744 (uint64_t(1) << unsigned(UniOpRRR::kAnd )) | 745 (uint64_t(1) << unsigned(UniOpRRR::kOr )) | 746 (uint64_t(1) << unsigned(UniOpRRR::kXor )) | 747 (uint64_t(1) << unsigned(UniOpRRR::kAdd )) | 748 (uint64_t(1) << unsigned(UniOpRRR::kMul )) | 749 (uint64_t(1) << unsigned(UniOpRRR::kSMin)) | 750 (uint64_t(1) << unsigned(UniOpRRR::kSMax)) | 751 (uint64_t(1) << unsigned(UniOpRRR::kUMin)) | 752 (uint64_t(1) << unsigned(UniOpRRR::kUMax)) ; 753 754 static ASMJIT_INLINE_NODEBUG bool is_op_3i_commutative(UniOpRRR op) noexcept { 755 return (kOp3ICommutativeMask & (uint64_t(1) << unsigned(op))) != 0; 756 } 757 758 void UniCompiler::emit_3i(UniOpRRR op, const Gp& dst, const Operand_& src1_, const Operand_& src2_) { 759 Operand src1(src1_); 760 Operand src2(src2_); 761 762 if (!src1.is_reg()) { 763 if (src2.is_reg() && is_op_3i_commutative(op)) { 764 ASMJIT_ASSERT(src2.is_gp()); 765 std::swap(src1, src2); 766 } 767 else { 768 src1 = gp_force_reg(*this, src1, dst); 769 } 770 } 771 772 static constexpr uint16_t addsub_inst[2] = { Inst::kIdAdd, Inst::kIdSub }; 773 static constexpr uint16_t sminmax_inst[2] = { Inst::kIdSmin, Inst::kIdSmax }; 774 static constexpr uint16_t uminmax_inst[2] = { Inst::kIdUmin, Inst::kIdUmax }; 775 static constexpr uint16_t logical_inst[4] = { Inst::kIdAnd, Inst::kIdOrr, Inst::kIdEor, Inst::kIdBic }; 776 static constexpr uint16_t shift_inst[3] = { Inst::kIdLsl, Inst::kIdLsr, Inst::kIdAsr }; 777 778 // ArithOp Reg, Reg, Imm 779 // --------------------- 780 781 if (src2.is_imm()) { 782 Gp a = src1.as<Gp>().clone_as(dst); 783 Imm b = src2.as<Imm>(); 784 785 switch (op) { 786 case UniOpRRR::kXor: 787 if (b.value() == -1 || (b.value_as<uint32_t>() == 0xFFFFFFFFu && dst.size() == 4)) { 788 cc->mvn_(dst, a); 789 return; 790 } 791 [[fallthrough]]; 792 793 case UniOpRRR::kAnd: 794 case UniOpRRR::kOr: 795 case UniOpRRR::kBic: { 796 if (a64::Utils::is_logical_imm(b.value_as<uint64_t>(), a.size() * 8u)) { 797 cc->emit(logical_inst[size_t(op) - size_t(UniOpRRR::kAnd)], dst, a, b); 798 return; 799 } 800 801 // If the immediate value is not encodable, we have to use a register. 802 break; 803 } 804 805 case UniOpRRR::kAdd: 806 case UniOpRRR::kSub: { 807 uint64_t value = b.value_as<uint64_t>(); 808 unsigned reverse = int64_t(value) < 0; 809 810 if (reverse) 811 value = uint64_t(0) - value; 812 813 if (op == UniOpRRR::kSub) 814 reverse = reverse ^ 1u; 815 816 // TODO: [JIT] Just testing the idea of patching the previous instruction to have a post-index addressing. 817 if (!reverse && uint64_t(value) < 256 && dst.id() == a.id()) { 818 if (cc->cursor()->type() == NodeType::kInst) { 819 InstNode* prev_inst = cc->cursor()->as<InstNode>(); 820 if (prev_inst->inst_id() == Inst::kIdLdr || prev_inst->inst_id() == Inst::kIdStr || 821 prev_inst->inst_id() == Inst::kIdLdr_v || prev_inst->inst_id() == Inst::kIdStr_v) { 822 Mem& mem_op = prev_inst->op(prev_inst->op_count() - 1).as<Mem>(); 823 if (mem_op.base_reg() == a && !mem_op.has_index() && !mem_op.has_offset()) { 824 mem_op.set_offset_mode(arm::OffsetMode::kPostIndex); 825 mem_op.add_offset(int64_t(value)); 826 return; 827 } 828 } 829 } 830 } 831 832 if (Support::is_uint_n<12>(value)) { 833 cc->emit(addsub_inst[reverse], dst, a, Imm(value)); 834 return; 835 } 836 837 // If the immediate value is not encodable, we have to use a register. 838 break; 839 } 840 841 case UniOpRRR::kMul: { 842 uint64_t value = b.value_as<uint64_t>(); 843 if (value > 0u) { 844 if (Support::is_power_of_2(value)) { 845 uint32_t shift = Support::ctz(value); 846 cc->lsl(dst, a, Imm(shift)); 847 return; 848 } 849 850 // We can still support multiplication with `power_of_2 + 1` 851 if (Support::is_power_of_2(--value)) { 852 uint32_t shift = Support::ctz(value); 853 cc->add(dst, a, a, a64::lsl(shift)); 854 return; 855 } 856 } 857 break; 858 } 859 860 case UniOpRRR::kSMin: 861 case UniOpRRR::kSMax: { 862 int64_t value = b.value_as<int64_t>(); 863 864 if (value == 0 || value == -1) { 865 uint32_t shift = dst.size() * 8u - 1u; 866 867 // Signed min/max against -1 and 0 can be implemented by using a variation of bitwise instructions 868 // with the input value combined with its signs (that's why arithmetic shift right is used). 869 static constexpr uint16_t inst_table[4] = { 870 Inst::kIdAnd, // smin(a, 0) == and(a, expand_msb(a)) 871 Inst::kIdOrr, // smin(a, -1) == orr(a, expand_msb(a)) 872 Inst::kIdBic, // smax(a, 0) == bic(a, expand_msb(a)) 873 Inst::kIdOrn // smax(a, -1) == orn(a, expand_msb(a)) 874 }; 875 876 InstId inst_id = inst_table[(size_t(op) - size_t(UniOpRRR::kSMin)) * 2u + size_t(value == -1)]; 877 cc->emit(inst_id, dst, a, a, Imm(a64::asr(shift))); 878 return; 879 } 880 881 if (has_cssc() && Support::is_int_n<8>(value)) { 882 cc->emit(sminmax_inst[size_t(op) - size_t(UniOpRRR::kSMin)], dst, a, b); 883 return; 884 } 885 break; 886 } 887 888 case UniOpRRR::kUMin: 889 case UniOpRRR::kUMax: { 890 uint64_t value = b.value_as<uint64_t>(); 891 if (has_cssc() && Support::is_uint_n<8>(value)) { 892 cc->emit(uminmax_inst[size_t(op) - size_t(UniOpRRR::kUMin)], dst, a, b); 893 return; 894 } 895 break; 896 } 897 898 case UniOpRRR::kSll: 899 case UniOpRRR::kSrl: 900 case UniOpRRR::kSra: { 901 cc->emit(shift_inst[size_t(op) - size_t(UniOpRRR::kSll)], dst, a, b); 902 return; 903 } 904 905 case UniOpRRR::kRol: 906 case UniOpRRR::kRor: { 907 uint32_t width = dst.size() * 8u; 908 uint32_t value = b.value_as<uint32_t>() & (width - 1); 909 910 if (op == UniOpRRR::kRol) 911 value = width - value; 912 913 cc->ror(dst, a, Imm(value)); 914 return; 915 } 916 917 case UniOpRRR::kSBound: { 918 // TODO: CSSC 919 // if (has_cssc() && Support::is_uint8(value)) { 920 // } 921 break; 922 } 923 924 default: 925 // Unhandled instruction means to use a register instead of immediate. 926 break; 927 } 928 } 929 930 // ArithOp Reg, Reg, Reg 931 // --------------------- 932 933 { 934 src2 = gp_force_reg(*this, src2, dst); 935 936 Gp a = src1.as<Gp>(); 937 Gp b = src2.as<Gp>(); 938 939 switch (op) { 940 case UniOpRRR::kAnd: 941 case UniOpRRR::kOr: 942 case UniOpRRR::kXor: 943 case UniOpRRR::kBic: { 944 cc->emit(logical_inst[size_t(op) - size_t(UniOpRRR::kAnd)], dst, a, b); 945 return; 946 } 947 948 case UniOpRRR::kAdd: 949 case UniOpRRR::kSub: { 950 cc->emit(addsub_inst[size_t(op) - size_t(UniOpRRR::kAdd)], dst, a, b); 951 return; 952 } 953 954 case UniOpRRR::kMul: { 955 cc->mul(dst, a, b); 956 return; 957 } 958 959 case UniOpRRR::kUDiv: { 960 cc->udiv(dst, a, b); 961 return; 962 } 963 964 case UniOpRRR::kUMod: { 965 Gp tmp = new_similar_reg(dst, "@tmp"); 966 cc->udiv(tmp, a, b); 967 cc->mul(tmp, tmp, b); 968 cc->sub(dst, a, tmp); 969 return; 970 } 971 972 case UniOpRRR::kSMin: 973 case UniOpRRR::kSMax: { 974 if (has_cssc()) { 975 cc->emit(sminmax_inst[size_t(op) - size_t(UniOpRRR::kSMin)], dst, a, b); 976 } 977 else { 978 cc->cmp(a, b); 979 cc->csel(dst, a, b, op == UniOpRRR::kSMin ? CondCode::kLT : CondCode::kGT); 980 } 981 return; 982 } 983 984 case UniOpRRR::kUMin: 985 case UniOpRRR::kUMax: { 986 if (has_cssc()) { 987 cc->emit(uminmax_inst[size_t(op) - size_t(UniOpRRR::kUMin)], dst, a, b); 988 } 989 else { 990 cc->cmp(a, b); 991 cc->csel(dst, a, b, op == UniOpRRR::kUMin ? CondCode::kLO : CondCode::kHI); 992 } 993 return; 994 } 995 996 case UniOpRRR::kSll: 997 case UniOpRRR::kSrl: 998 case UniOpRRR::kSra: { 999 cc->emit(shift_inst[size_t(op) - size_t(UniOpRRR::kSll)], dst, a, b); 1000 return; 1001 } 1002 1003 case UniOpRRR::kRol: { 1004 Gp tmp = new_similar_reg(dst, "@rol_rev"); 1005 cc->neg(tmp, b); 1006 cc->ror(dst, a, tmp); 1007 return; 1008 } 1009 1010 case UniOpRRR::kRor: { 1011 cc->ror(dst, a, b); 1012 return; 1013 } 1014 1015 case UniOpRRR::kSBound: { 1016 cc->cmp(a, b); 1017 cc->bic(dst, a, a, a64::asr(dst.size() * 8u - 1)); 1018 cc->csel(dst, dst, b, CondCode::kSignedLT); 1019 return; 1020 } 1021 1022 default: 1023 // Everything should be handled, so this should never be reached! 1024 ASMJIT_NOT_REACHED(); 1025 } 1026 } 1027 } 1028 1029 void UniCompiler::emit_j(const Operand_& target) { 1030 cc->emit(Inst::kIdB, target); 1031 } 1032 1033 void UniCompiler::emit_j_if(const Label& target, const UniCondition& condition) { 1034 ConditionApplier ca(condition); 1035 ca.optimize(*this); 1036 ca.emit(*this); 1037 cc->b(ca.cond, target); 1038 } 1039 1040 void UniCompiler::adds_u8(const Gp& dst, const Gp& src1, const Gp& src2) { 1041 ASMJIT_ASSERT(dst.size() == src1.size()); 1042 ASMJIT_ASSERT(dst.size() == src2.size()); 1043 1044 cc->add(dst, src1, src2); 1045 if (has_cssc()) { 1046 cc->umin(dst, dst, 255); 1047 return; 1048 } 1049 1050 Gp tmp = new_similar_reg(dst, "@tmp"); 1051 cc->cmp(dst, 255); 1052 cc->mov(tmp, 255); 1053 cc->csel(dst, dst, tmp, CondCode::kLO); 1054 } 1055 1056 void UniCompiler::inv_u8(const Gp& dst, const Gp& src) { 1057 cc->eor(dst, src, 0xFF); 1058 } 1059 1060 void UniCompiler::div_255_u32(const Gp& dst, const Gp& src) { 1061 ASMJIT_ASSERT(dst.size() == src.size()); 1062 1063 // dst = src + 128; 1064 // dst = (dst + (dst >> 8)) >> 8 1065 cc->add(dst, src, 128); 1066 cc->add(dst, dst, dst, a64::lsr(8)); 1067 cc->lsr(dst, dst, 8); 1068 } 1069 1070 void UniCompiler::mul_257_hu16(const Gp& dst, const Gp& src) { 1071 ASMJIT_ASSERT(dst.size() == src.size()); 1072 cc->add(dst, src, src, a64::lsl(8)); 1073 cc->lsr(dst, dst, 16); 1074 } 1075 1076 void UniCompiler::add_scaled(const Gp& dst, const Gp& a_, int b) { 1077 Gp a = a_.clone_as(dst); 1078 1079 if (Support::is_power_of_2(b)) { 1080 uint32_t shift = Support::ctz(b); 1081 cc->add(dst, dst, a, a64::lsl(shift)); 1082 } 1083 else if (b == 3 && dst.id() == a.id()) { 1084 cc->add(dst, dst, dst, a64::lsl(1)); 1085 } 1086 else { 1087 Gp tmp = new_similar_reg(dst, "@tmp"); 1088 cc->mov(tmp, b); 1089 cc->madd(dst, a, tmp, dst); 1090 } 1091 } 1092 1093 void UniCompiler::add_ext(const Gp& dst, const Gp& src_, const Gp& idx_, uint32_t scale, int32_t disp) { 1094 ASMJIT_ASSERT(scale != 0); 1095 1096 Gp src = src_.clone_as(dst); 1097 Gp idx = idx_.clone_as(dst); 1098 1099 if (Support::is_power_of_2(scale)) { 1100 cc->add(dst, src, idx, a64::lsl(Support::ctz(scale))); 1101 } 1102 else { 1103 Gp tmp = new_similar_reg(dst, "@tmp"); 1104 1105 if (scale == 3) { 1106 cc->add(tmp, src, idx, a64::lsl(1)); 1107 cc->add(dst, tmp, idx); 1108 } 1109 else { 1110 cc->mov(tmp, scale); 1111 cc->mul(tmp, tmp, idx); 1112 cc->add(dst, src, tmp); 1113 } 1114 } 1115 1116 if (disp) { 1117 if (disp > 0) 1118 cc->add(dst, dst, disp); 1119 else 1120 cc->sub(dst, dst, -disp); 1121 } 1122 } 1123 1124 void UniCompiler::lea(const Gp& dst, const Mem& src) { 1125 Gp base = src.base_reg().as<Gp>(); 1126 1127 if (src.has_index()) { 1128 add_ext(dst, base, src.index_reg().as<Gp>(), 1u << src.shift(), src.offset_lo32()); 1129 } 1130 else if (src.offset_lo32()) { 1131 cc->add(dst, base, src.offset_lo32()); 1132 } 1133 else { 1134 cc->mov(dst, base); 1135 } 1136 } 1137 1138 // ujit::UniCompiler - Vector Instructions - Constants 1139 // =================================================== 1140 1141 //! Floating point mode is used in places that are generic and implement various functionality that needs more 1142 //! than a single instruction. Typically implementing either higher level concepts or missing functionality. 1143 enum FloatMode : uint32_t { 1144 //! Used by non-floating point instructions. 1145 kNone = 0, 1146 1147 kF32S = 1, 1148 kF64S = 2, 1149 kF32V = 3, 1150 kF64V = 4 1151 }; 1152 1153 enum class ElementSize : uint32_t { 1154 kNA = 0, 1155 k8 = 0, 1156 k16 = 1, 1157 k32 = 2, 1158 k64 = 3 1159 }; 1160 1161 enum class SameVecOp : uint32_t { 1162 kNone = 0, 1163 kZero = 1, 1164 kOnes = 2, 1165 kSrc = 3 1166 }; 1167 1168 enum class VecPart : uint32_t { 1169 kNA = 0, 1170 kLo = 1, 1171 kHi = 2 1172 }; 1173 1174 enum class NarrowingOp : uint32_t { 1175 kNone, 1176 kI16ToI8, 1177 kI16ToU8, 1178 kU16ToU8, 1179 kI32ToI16, 1180 kI32ToU16, 1181 kU32ToU16, 1182 kI64ToI32, 1183 kI64ToU32, 1184 kU64ToU32 1185 }; 1186 1187 enum class NarrowingMode : uint32_t { 1188 kTruncate, 1189 kSaturateSToU, 1190 kSaturateSToS, 1191 kSaturateUToU 1192 }; 1193 1194 // ujit::UniCompiler - Vector Instructions - UniOp Information 1195 // ============================================================ 1196 1197 struct UniOpVInfo { 1198 //! \name Members 1199 //! \{ 1200 1201 uint32_t inst_id : 13; 1202 ASIMDExt asimd_ext : 6; 1203 uint32_t commutative : 1; 1204 uint32_t comparison : 1; 1205 uint32_t reverse : 1; 1206 SameVecOp same_vec_op : 2; 1207 FloatMode float_mode : 3; 1208 ElementSize dst_element : 3; 1209 VecPart dst_part : 2; 1210 ElementSize src_element : 3; 1211 VecPart src_part : 2; 1212 uint32_t imm : 8; 1213 uint32_t reserved2 : 19; 1214 1215 //! \} 1216 }; 1217 1218 #define DEFINE_OP(inst_id, ext, commutative, comparison, reverse, same_vec_op, float_mode, dst_element, dst_part, src_element, src_part, imm) \ 1219 UniOpVInfo { \ 1220 inst_id, \ 1221 ASIMDExt::ext, \ 1222 commutative, \ 1223 comparison, \ 1224 reverse, \ 1225 SameVecOp::same_vec_op, \ 1226 FloatMode::float_mode, \ 1227 ElementSize::dst_element, \ 1228 VecPart::dst_part, \ 1229 ElementSize::src_element, \ 1230 VecPart::src_part, \ 1231 imm, \ 1232 0 \ 1233 } 1234 1235 static constexpr UniOpVInfo opcode_info_2v[size_t(UniOpVV::kMaxValue) + 1] = { 1236 DEFINE_OP(Inst::kIdMov_v , kIntrin, 0, 0, 0, kNone, kNone, kNA, kNA, kNA, kNA, 0x00u), // kMov. 1237 DEFINE_OP(Inst::kIdMov_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kLo, k64, kLo, 0x00u), // kMovU64. 1238 DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kBroadcastU8Z. 1239 DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kBroadcastU16Z. 1240 DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kBroadcastU8. 1241 DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kBroadcastU16. 1242 DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kBroadcastU32. 1243 DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kBroadcastU64. 1244 DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kBroadcastF32. 1245 DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kBroadcastF64. 1246 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kBroadcastV128_U32. 1247 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kBroadcastV128_U64. 1248 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kBroadcastV128_F32. 1249 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kBroadcastV128_F64. 1250 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kBroadcastV256_U32. 1251 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kBroadcastV256_U64. 1252 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kBroadcastV256_F32. 1253 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kBroadcastV256_F64. 1254 DEFINE_OP(Inst::kIdAbs_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kAbsI8. 1255 DEFINE_OP(Inst::kIdAbs_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kAbsI16. 1256 DEFINE_OP(Inst::kIdAbs_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kAbsI32. 1257 DEFINE_OP(Inst::kIdAbs_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kAbsI64. 1258 DEFINE_OP(Inst::kIdMvn_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kNotU32. 1259 DEFINE_OP(Inst::kIdMvn_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kNotU64. 1260 DEFINE_OP(Inst::kIdSshll_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k8 , kLo, 0x00u), // kCvtI8LoToI16 1261 DEFINE_OP(Inst::kIdSshll2_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k8 , kHi, 0x00u), // kCvtI8HiToI16 1262 DEFINE_OP(Inst::kIdUshll_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k8 , kLo, 0x00u), // kCvtU8LoToU16 1263 DEFINE_OP(Inst::kIdUshll2_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k8 , kHi, 0x00u), // kCvtU8HiToU16 1264 DEFINE_OP(Inst::kIdSshll_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k8 , kLo, 0x00u), // kCvtI8ToI32 1265 DEFINE_OP(Inst::kIdUshll_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k8 , kLo, 0x00u), // kCvtU8ToU32 1266 DEFINE_OP(Inst::kIdSshll_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k16, kLo, 0x00u), // kCvtI16LoToI32 1267 DEFINE_OP(Inst::kIdSshll2_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k16, kHi, 0x00u), // kCvtI16HiToI32 1268 DEFINE_OP(Inst::kIdUshll_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k16, kLo, 0x00u), // kCvtU16LoToU32 1269 DEFINE_OP(Inst::kIdUshll2_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k16, kHi, 0x00u), // kCvtU16HiToU32 1270 DEFINE_OP(Inst::kIdSshll_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k32, kLo, 0x00u), // kCvtI32LoToI64 1271 DEFINE_OP(Inst::kIdSshll2_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k32, kHi, 0x00u), // kCvtI32HiToI64 1272 DEFINE_OP(Inst::kIdUshll_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k32, kLo, 0x00u), // kCvtU32LoToU64 1273 DEFINE_OP(Inst::kIdUshll2_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k32, kHi, 0x00u), // kCvtU32HiToU64 1274 DEFINE_OP(Inst::kIdFabs_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kAbsF32S. 1275 DEFINE_OP(Inst::kIdFabs_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kAbsF64S. 1276 DEFINE_OP(Inst::kIdFabs_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kAbsF32. 1277 DEFINE_OP(Inst::kIdFabs_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kAbsF64. 1278 DEFINE_OP(Inst::kIdFneg_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kNegF32S. 1279 DEFINE_OP(Inst::kIdFneg_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kNegF64S. 1280 DEFINE_OP(Inst::kIdFneg_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kNegF32. 1281 DEFINE_OP(Inst::kIdFneg_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kNegF64. 1282 DEFINE_OP(Inst::kIdMvn_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kNotF32. 1283 DEFINE_OP(Inst::kIdMvn_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kNotF64. 1284 DEFINE_OP(Inst::kIdFrintz_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kTruncF32S. 1285 DEFINE_OP(Inst::kIdFrintz_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kTruncF64S. 1286 DEFINE_OP(Inst::kIdFrintz_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kTruncF32. 1287 DEFINE_OP(Inst::kIdFrintz_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kTruncF64. 1288 DEFINE_OP(Inst::kIdFrintm_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kFloorF32S. 1289 DEFINE_OP(Inst::kIdFrintm_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kFloorF64S. 1290 DEFINE_OP(Inst::kIdFrintm_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kFloorF32. 1291 DEFINE_OP(Inst::kIdFrintm_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kFloorF64. 1292 DEFINE_OP(Inst::kIdFrintp_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCeilF32S. 1293 DEFINE_OP(Inst::kIdFrintp_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCeilF64S. 1294 DEFINE_OP(Inst::kIdFrintp_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCeilF32. 1295 DEFINE_OP(Inst::kIdFrintp_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCeilF64. 1296 DEFINE_OP(Inst::kIdFrintn_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kRoundEvenF32S. 1297 DEFINE_OP(Inst::kIdFrintn_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kRoundEvenF64S. 1298 DEFINE_OP(Inst::kIdFrintn_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kRoundEvenF32. 1299 DEFINE_OP(Inst::kIdFrintn_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kRoundEvenF64. 1300 DEFINE_OP(Inst::kIdFrinta_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kRoundHalfAwayF32S. 1301 DEFINE_OP(Inst::kIdFrinta_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kRoundHalfAwayF64S. 1302 DEFINE_OP(Inst::kIdFrinta_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kRoundHalfAwayF32. 1303 DEFINE_OP(Inst::kIdFrinta_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kRoundHalfAwayF64. 1304 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kRoundHalfUpF32S. 1305 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kRoundHalfUpF64S. 1306 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kRoundHalfUpF32. 1307 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kRoundHalfUpF64. 1308 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kRcpF32. 1309 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kRcpF64. 1310 DEFINE_OP(Inst::kIdFsqrt_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kSqrtF32S. 1311 DEFINE_OP(Inst::kIdFsqrt_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kSqrtF64S. 1312 DEFINE_OP(Inst::kIdFsqrt_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kSqrtF32. 1313 DEFINE_OP(Inst::kIdFsqrt_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kSqrtF64. 1314 DEFINE_OP(Inst::kIdFcvt_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k32, kNA, 0x00u), // kCvtF32ToF64S. 1315 DEFINE_OP(Inst::kIdFcvt_v , kASIMD , 0, 0, 0, kNone, kF64S, k32, kNA, k64, kNA, 0x00u), // kCvtF64ToF32S. 1316 DEFINE_OP(Inst::kIdScvtf_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCvtI32ToF32. 1317 DEFINE_OP(Inst::kIdFcvtl_v , kASIMD , 0, 0, 0, kNone, kF32V, k64, kNA, k32, kLo, 0x00u), // kCvtF32LoToF64. 1318 DEFINE_OP(Inst::kIdFcvtl2_v , kASIMD , 0, 0, 0, kNone, kF32V, k64, kNA, k32, kHi, 0x00u), // kCvtF32HiToF64. 1319 DEFINE_OP(Inst::kIdFcvtn_v , kASIMD , 0, 0, 0, kNone, kF64V, k32, kLo, k64, kNA, 0x00u), // kCvtF64ToF32Lo. 1320 DEFINE_OP(Inst::kIdFcvtn2_v , kASIMD , 0, 0, 0, kNone, kF64V, k32, kHi, k64, kNA, 0x00u), // kCvtF64ToF32Hi. 1321 DEFINE_OP(Inst::kIdSshll_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k32, kLo, 0x00u), // kCvtI32LoToF64. 1322 DEFINE_OP(Inst::kIdSshll2_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k32, kHi, 0x00u), // kCvtI32HiToF64. 1323 DEFINE_OP(Inst::kIdFcvtzs_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCvtTruncF32ToI32. 1324 DEFINE_OP(Inst::kIdFcvtzs_v , kIntrin, 0, 0, 0, kNone, kF64V, k32, kLo, k64, kLo, 0x00u), // kCvtTruncF64ToI32Lo. 1325 DEFINE_OP(Inst::kIdFcvtzs_v , kIntrin, 0, 0, 0, kNone, kF64V, k32, kHi, k64, kHi, 0x00u), // kCvtTruncF64ToI32Hi. 1326 DEFINE_OP(Inst::kIdFcvtns_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCvtRoundF32ToI32. 1327 DEFINE_OP(Inst::kIdFcvtns_v , kIntrin, 0, 0, 0, kNone, kF64V, k32, kLo, k64, kLo, 0x00u), // kCvtRoundF64ToI32Lo. 1328 DEFINE_OP(Inst::kIdFcvtns_v , kIntrin, 0, 0, 0, kNone, kF64V, k32, kHi, k64, kHi, 0x00u) // kCvtRoundF64ToI32Hi. 1329 }; 1330 1331 static constexpr UniOpVInfo opcode_info_2vs[size_t(UniOpVR::kMaxValue) + 1] = { 1332 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kMov. 1333 DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kMovU32. 1334 DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kMovU64. 1335 DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kInsertU8. 1336 DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kInsertU16. 1337 DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInsertU32. 1338 DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInsertU64. 1339 DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kExtractU8. 1340 DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kExtractU16. 1341 DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kExtractU32. 1342 DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kExtractU64. 1343 DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, kNA, kNA, 0x00u), // kCvtIntToF32. 1344 DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, kNA, kNA, 0x00u), // kCvtIntToF64. 1345 DEFINE_OP(Inst::kIdFcvtzs_v , kASIMD , 0, 0, 0, kNone, kF32S, kNA, kNA, k32, kNA, 0x00u), // kCvtTruncF32ToInt. 1346 DEFINE_OP(Inst::kIdFcvtns_v , kASIMD , 0, 0, 0, kNone, kF32S, kNA, kNA, k32, kNA, 0x00u), // kCvtRoundF32ToInt. 1347 DEFINE_OP(Inst::kIdFcvtzs_v , kASIMD , 0, 0, 0, kNone, kF64S, kNA, kNA, k64, kNA, 0x00u), // kCvtTruncF64ToInt. 1348 DEFINE_OP(Inst::kIdFcvtns_v , kASIMD , 0, 0, 0, kNone, kF64S, kNA, kNA, k64, kNA, 0x00u) // kCvtRoundF64ToInt. 1349 }; 1350 1351 static constexpr UniOpVInfo opcode_info_2vi[size_t(UniOpVVI::kMaxValue) + 1] = { 1352 DEFINE_OP(Inst::kIdShl_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSllU16. 1353 DEFINE_OP(Inst::kIdShl_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kSllU32. 1354 DEFINE_OP(Inst::kIdShl_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSllU64. 1355 DEFINE_OP(Inst::kIdUshr_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSrlU16. 1356 DEFINE_OP(Inst::kIdUshr_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kSrlU32. 1357 DEFINE_OP(Inst::kIdUshr_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSrlU64. 1358 DEFINE_OP(Inst::kIdSshr_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSraI16. 1359 DEFINE_OP(Inst::kIdSshr_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kSraI32. 1360 DEFINE_OP(Inst::kIdSshr_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSraI64. 1361 DEFINE_OP(Inst::kIdExt_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kSllbU128. 1362 DEFINE_OP(Inst::kIdExt_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kSrlbU128. 1363 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSwizzleU16x4. 1364 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSwizzleLoU16x4. 1365 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSwizzleHiU16x4. 1366 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kSwizzleU32x4. 1367 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSwizzleU64x2. 1368 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kSwizzleF32x4. 1369 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kSwizzleF64x2. 1370 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSwizzleU64x4. 1371 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kSwizzleF64x4. 1372 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kExtractV128_I32. 1373 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kExtractV128_I64. 1374 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kExtractV128_F32. 1375 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kExtractV128_F64. 1376 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kExtractV256_I32. 1377 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kExtractV256_I64. 1378 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kExtractV256_F32. 1379 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kExtractV256_F64. 1380 DEFINE_OP(Inst::kIdUrshr_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSrlRndU16. 1381 DEFINE_OP(Inst::kIdUrshr_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kSrlRndU32. 1382 DEFINE_OP(Inst::kIdUrshr_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSrlRndU64. 1383 DEFINE_OP(Inst::kIdUsra_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSrlAccU16. 1384 DEFINE_OP(Inst::kIdUsra_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kSrlAccU32. 1385 DEFINE_OP(Inst::kIdUsra_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSrlAccU64. 1386 DEFINE_OP(Inst::kIdUrsra_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSrlRndAccU16. 1387 DEFINE_OP(Inst::kIdUrsra_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kSrlRndAccU32. 1388 DEFINE_OP(Inst::kIdUrsra_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSrlRndAccU64. 1389 DEFINE_OP(Inst::kIdShrn_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k16, kLo, 0x00u), // kSrlnLoU16. 1390 DEFINE_OP(Inst::kIdShrn2_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k16, kHi, 0x00u), // kSrlnHiU16. 1391 DEFINE_OP(Inst::kIdShrn_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k32, kLo, 0x00u), // kSrlnLoU32. 1392 DEFINE_OP(Inst::kIdShrn2_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k32, kHi, 0x00u), // kSrlnHiU32. 1393 DEFINE_OP(Inst::kIdShrn_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k64, kLo, 0x00u), // kSrlnLoU64. 1394 DEFINE_OP(Inst::kIdShrn2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k64, kHi, 0x00u), // kSrlnHiU64. 1395 DEFINE_OP(Inst::kIdRshrn_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kLo, k16, kNA, 0x00u), // kSrlnRndLoU16. 1396 DEFINE_OP(Inst::kIdRshrn2_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kHi, k16, kNA, 0x00u), // kSrlnRndHiU16. 1397 DEFINE_OP(Inst::kIdRshrn_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kLo, k32, kNA, 0x00u), // kSrlnRndLoU32. 1398 DEFINE_OP(Inst::kIdRshrn2_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kHi, k32, kNA, 0x00u), // kSrlnRndHiU32. 1399 DEFINE_OP(Inst::kIdRshrn_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kLo, k64, kNA, 0x00u), // kSrlnRndLoU64. 1400 DEFINE_OP(Inst::kIdRshrn2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kHi, k64, kNA, 0x00u), // kSrlnRndHiU64. 1401 }; 1402 1403 static constexpr UniOpVInfo opcode_info_3v[size_t(UniOpVVV::kMaxValue) + 1] = { 1404 DEFINE_OP(Inst::kIdAnd_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kAndU32. 1405 DEFINE_OP(Inst::kIdAnd_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kAndU64. 1406 DEFINE_OP(Inst::kIdOrr_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kOrU32. 1407 DEFINE_OP(Inst::kIdOrr_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kOrU64. 1408 DEFINE_OP(Inst::kIdEor_v , kASIMD , 1, 0, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kXorU32. 1409 DEFINE_OP(Inst::kIdEor_v , kASIMD , 1, 0, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kXorU64. 1410 DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 1, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kAndnU32. 1411 DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 1, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kAndnU64. 1412 DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kBicU32. 1413 DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kBicU64. 1414 DEFINE_OP(Inst::kIdNone , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kAvgrU8. 1415 DEFINE_OP(Inst::kIdNone , kASIMD , 1, 0, 0, kSrc , kNone, k16, kNA, k16, kNA, 0x00u), // kAvgrU16. 1416 DEFINE_OP(Inst::kIdAdd_v , kASIMD , 1, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kAddU8. 1417 DEFINE_OP(Inst::kIdAdd_v , kASIMD , 1, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kAddU16. 1418 DEFINE_OP(Inst::kIdAdd_v , kASIMD , 1, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kAddU32. 1419 DEFINE_OP(Inst::kIdAdd_v , kASIMD , 1, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kAddU64. 1420 DEFINE_OP(Inst::kIdSub_v , kASIMD , 0, 0, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kSubU8. 1421 DEFINE_OP(Inst::kIdSub_v , kASIMD , 0, 0, 0, kZero, kNone, k16, kNA, k16, kNA, 0x00u), // kSubU16. 1422 DEFINE_OP(Inst::kIdSub_v , kASIMD , 0, 0, 0, kZero, kNone, k32, kNA, k32, kNA, 0x00u), // kSubU32. 1423 DEFINE_OP(Inst::kIdSub_v , kASIMD , 0, 0, 0, kZero, kNone, k64, kNA, k64, kNA, 0x00u), // kSubU64. 1424 DEFINE_OP(Inst::kIdSqadd_v , kASIMD , 1, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kAddsI8. 1425 DEFINE_OP(Inst::kIdUqadd_v , kASIMD , 1, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kAddsU8. 1426 DEFINE_OP(Inst::kIdSqadd_v , kASIMD , 1, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kAddsI16. 1427 DEFINE_OP(Inst::kIdUqadd_v , kASIMD , 1, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kAddsU16. 1428 DEFINE_OP(Inst::kIdSqsub_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kSubsI8. 1429 DEFINE_OP(Inst::kIdUqsub_v , kASIMD , 0, 0, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kSubsU8. 1430 DEFINE_OP(Inst::kIdSqsub_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSubsI16. 1431 DEFINE_OP(Inst::kIdUqsub_v , kASIMD , 0, 0, 0, kZero, kNone, k16, kNA, k16, kNA, 0x00u), // kSubsU16. 1432 DEFINE_OP(Inst::kIdMul_v , kASIMD , 1, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kMulU16. 1433 DEFINE_OP(Inst::kIdMul_v , kASIMD , 1, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kMulU32. 1434 DEFINE_OP(Inst::kIdNone , kIntrin, 1, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kMulU64. 1435 DEFINE_OP(Inst::kIdNone , kIntrin, 1, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kMulhI16. 1436 DEFINE_OP(Inst::kIdNone , kIntrin, 1, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kMulhU16. 1437 DEFINE_OP(Inst::kIdNone , kIntrin, 1, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kMulU64_LoU32. 1438 DEFINE_OP(Inst::kIdNone , kIntrin, 1, 0, 0, kNone, kNone, k32, kNA, k16, kNA, 0x00u), // kMHAddI16_I32. 1439 DEFINE_OP(Inst::kIdSmin_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kMinI8. 1440 DEFINE_OP(Inst::kIdUmin_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kMinU8. 1441 DEFINE_OP(Inst::kIdSmin_v , kASIMD , 1, 0, 0, kSrc , kNone, k16, kNA, k16, kNA, 0x00u), // kMinI16. 1442 DEFINE_OP(Inst::kIdUmin_v , kASIMD , 1, 0, 0, kSrc , kNone, k16, kNA, k16, kNA, 0x00u), // kMinU16. 1443 DEFINE_OP(Inst::kIdSmin_v , kASIMD , 1, 0, 0, kSrc , kNone, k32, kNA, k32, kNA, 0x00u), // kMinI32. 1444 DEFINE_OP(Inst::kIdUmin_v , kASIMD , 1, 0, 0, kSrc , kNone, k32, kNA, k32, kNA, 0x00u), // kMinU32. 1445 DEFINE_OP(Inst::kIdCmgt_v , kIntrin, 1, 0, 0, kSrc , kNone, k64, kNA, k64, kNA, 0x00u), // kMinI64. 1446 DEFINE_OP(Inst::kIdCmhi_v , kIntrin, 1, 0, 0, kSrc , kNone, k64, kNA, k64, kNA, 0x00u), // kMinU64. 1447 DEFINE_OP(Inst::kIdSmax_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kMaxI8. 1448 DEFINE_OP(Inst::kIdUmax_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kMaxU8. 1449 DEFINE_OP(Inst::kIdSmax_v , kASIMD , 1, 0, 0, kSrc , kNone, k16, kNA, k16, kNA, 0x00u), // kMaxI16. 1450 DEFINE_OP(Inst::kIdUmax_v , kASIMD , 1, 0, 0, kSrc , kNone, k16, kNA, k16, kNA, 0x00u), // kMaxU16. 1451 DEFINE_OP(Inst::kIdSmax_v , kASIMD , 1, 0, 0, kSrc , kNone, k32, kNA, k32, kNA, 0x00u), // kMaxI32. 1452 DEFINE_OP(Inst::kIdUmax_v , kASIMD , 1, 0, 0, kSrc , kNone, k32, kNA, k32, kNA, 0x00u), // kMaxU32. 1453 DEFINE_OP(Inst::kIdCmgt_v , kIntrin, 1, 0, 0, kSrc , kNone, k64, kNA, k64, kNA, 0x01u), // kMaxI64. 1454 DEFINE_OP(Inst::kIdCmhi_v , kIntrin, 1, 0, 0, kSrc , kNone, k64, kNA, k64, kNA, 0x01u), // kMaxU64. 1455 DEFINE_OP(Inst::kIdCmeq_v , kASIMD , 1, 1, 0, kOnes, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpEqU8. 1456 DEFINE_OP(Inst::kIdCmeq_v , kASIMD , 1, 1, 0, kOnes, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpEqU16. 1457 DEFINE_OP(Inst::kIdCmeq_v , kASIMD , 1, 1, 0, kOnes, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpEqU32. 1458 DEFINE_OP(Inst::kIdCmeq_v , kASIMD , 1, 1, 0, kOnes, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpEqU64. 1459 DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpGtI8. 1460 DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpGtU8. 1461 DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 0, kZero, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpGtI16. 1462 DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 0, kZero, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpGtU16. 1463 DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 0, kZero, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpGtI32. 1464 DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 0, kZero, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpGtU32. 1465 DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 0, kZero, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpGtI64. 1466 DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 0, kZero, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpGtU64. 1467 DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 0, kOnes, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpGeI8. 1468 DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 0, kOnes, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpGeU8. 1469 DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 0, kOnes, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpGeI16. 1470 DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 0, kOnes, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpGeU16. 1471 DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 0, kOnes, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpGeI32. 1472 DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 0, kOnes, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpGeU32. 1473 DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 0, kOnes, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpGeI64. 1474 DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 0, kOnes, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpGeU64. 1475 DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 1, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpLtI8. 1476 DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 1, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpLtU8. 1477 DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 1, kZero, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpLtI16. 1478 DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 1, kZero, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpLtU16. 1479 DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 1, kZero, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpLtI32. 1480 DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 1, kZero, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpLtU32. 1481 DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 1, kZero, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpLtI64. 1482 DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 1, kZero, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpLtU64. 1483 DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 1, kOnes, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpLeI8. 1484 DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 1, kOnes, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpLeU8. 1485 DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 1, kOnes, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpLeI16. 1486 DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 1, kOnes, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpLeU16. 1487 DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 1, kOnes, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpLeI32. 1488 DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 1, kOnes, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpLeU32. 1489 DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 1, kOnes, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpLeI64. 1490 DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 1, kOnes, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpLeU64. 1491 DEFINE_OP(Inst::kIdAnd_v , kASIMD , 1, 0, 0, kSrc , kF32V, k8 , kNA, k8 , kNA, 0x00u), // kAndF32. 1492 DEFINE_OP(Inst::kIdAnd_v , kASIMD , 1, 0, 0, kSrc , kF64V, k8 , kNA, k8 , kNA, 0x00u), // kAndF64. 1493 DEFINE_OP(Inst::kIdOrr_v , kASIMD , 1, 0, 0, kSrc , kF32V, k8 , kNA, k8 , kNA, 0x00u), // kOrF32. 1494 DEFINE_OP(Inst::kIdOrr_v , kASIMD , 1, 0, 0, kSrc , kF64V, k8 , kNA, k8 , kNA, 0x00u), // kOrF64. 1495 DEFINE_OP(Inst::kIdEor_v , kASIMD , 1, 0, 0, kZero, kF32V, k8 , kNA, k8 , kNA, 0x00u), // kXorF32. 1496 DEFINE_OP(Inst::kIdEor_v , kASIMD , 1, 0, 0, kZero, kF64V, k8 , kNA, k8 , kNA, 0x00u), // kXorF64. 1497 DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 1, kZero, kF32V, k8 , kNA, k8 , kNA, 0x00u), // kAndnF32. 1498 DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 1, kZero, kF64V, k8 , kNA, k8 , kNA, 0x00u), // kAndnF64. 1499 DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 0, kZero, kF32V, k8 , kNA, k8 , kNA, 0x00u), // kBicF32. 1500 DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 0, kZero, kF64V, k8 , kNA, k8 , kNA, 0x00u), // kBicF64. 1501 DEFINE_OP(Inst::kIdFadd_v , kASIMD , 1, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kAddF32S. 1502 DEFINE_OP(Inst::kIdFadd_v , kASIMD , 1, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kAddF64S. 1503 DEFINE_OP(Inst::kIdFadd_v , kASIMD , 1, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kAddF32. 1504 DEFINE_OP(Inst::kIdFadd_v , kASIMD , 1, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kAddF64. 1505 DEFINE_OP(Inst::kIdFsub_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kSubF32S. 1506 DEFINE_OP(Inst::kIdFsub_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kSubF64S. 1507 DEFINE_OP(Inst::kIdFsub_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kSubF32. 1508 DEFINE_OP(Inst::kIdFsub_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kSubF64. 1509 DEFINE_OP(Inst::kIdFmul_v , kASIMD , 1, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kMulF32S. 1510 DEFINE_OP(Inst::kIdFmul_v , kASIMD , 1, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kMulF64S. 1511 DEFINE_OP(Inst::kIdFmul_v , kASIMD , 1, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kMulF32. 1512 DEFINE_OP(Inst::kIdFmul_v , kASIMD , 1, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kMulF64. 1513 DEFINE_OP(Inst::kIdFdiv_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kDivF32S. 1514 DEFINE_OP(Inst::kIdFdiv_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kDivF64S. 1515 DEFINE_OP(Inst::kIdFdiv_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kDivF32. 1516 DEFINE_OP(Inst::kIdFdiv_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kDivF64. 1517 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kModF32S. 1518 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kModF64S. 1519 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kModF32. 1520 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kModF64. 1521 DEFINE_OP(Inst::kIdFminnm_v , kASIMD , 1, 0, 0, kSrc , kF32S, k32, kNA, k32, kNA, 0x00u), // kMinF32S. 1522 DEFINE_OP(Inst::kIdFminnm_v , kASIMD , 1, 0, 0, kSrc , kF64S, k64, kNA, k64, kNA, 0x00u), // kMinF64S. 1523 DEFINE_OP(Inst::kIdFminnm_v , kASIMD , 1, 0, 0, kSrc , kF32V, k32, kNA, k32, kNA, 0x00u), // kMinF32. 1524 DEFINE_OP(Inst::kIdFminnm_v , kASIMD , 1, 0, 0, kSrc , kF64V, k64, kNA, k64, kNA, 0x00u), // kMinF64. 1525 DEFINE_OP(Inst::kIdFmaxnm_v , kASIMD , 1, 0, 0, kSrc , kF32S, k32, kNA, k32, kNA, 0x00u), // kMaxF32S. 1526 DEFINE_OP(Inst::kIdFmaxnm_v , kASIMD , 1, 0, 0, kSrc , kF64S, k64, kNA, k64, kNA, 0x00u), // kMaxF64S. 1527 DEFINE_OP(Inst::kIdFmaxnm_v , kASIMD , 1, 0, 0, kSrc , kF32V, k32, kNA, k32, kNA, 0x00u), // kMaxF32. 1528 DEFINE_OP(Inst::kIdFmaxnm_v , kASIMD , 1, 0, 0, kSrc , kF64V, k64, kNA, k64, kNA, 0x00u), // kMaxF64. 1529 DEFINE_OP(Inst::kIdFcmeq_v , kASIMD , 1, 1, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCmpEqF32S (eq ordered quiet). 1530 DEFINE_OP(Inst::kIdFcmeq_v , kASIMD , 1, 1, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCmpEqF64S (eq ordered quiet). 1531 DEFINE_OP(Inst::kIdFcmeq_v , kASIMD , 1, 1, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCmpEqF32 (eq ordered quiet). 1532 DEFINE_OP(Inst::kIdFcmeq_v , kASIMD , 1, 1, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCmpEqF64 (eq ordered quiet). 1533 DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCmpNeF32S (ne ordered quiet). 1534 DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCmpNeF64S (ne ordered quiet). 1535 DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCmpNeF32 (ne ordered quiet). 1536 DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCmpNeF64 (ne ordered quiet). 1537 DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCmpGtF32S (gt ordered quiet). 1538 DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCmpGtF64S (gt ordered quiet). 1539 DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCmpGtF32 (gt ordered quiet). 1540 DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCmpGtF64 (gt ordered quiet). 1541 DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCmpGeF32S (ge ordered quiet). 1542 DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCmpGeF64S (ge ordered quiet). 1543 DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCmpGeF32 (ge ordered quiet). 1544 DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCmpGeF64 (ge ordered quiet). 1545 DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 1, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCmpLtF32S (lt ordered quiet). 1546 DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 1, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCmpLtF64S (lt ordered quiet). 1547 DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 1, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCmpLtF32 (lt ordered quiet). 1548 DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 1, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCmpLtF64 (lt ordered quiet). 1549 DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 1, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCmpLeF32S (le ordered quiet). 1550 DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 1, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCmpLeF64S (le ordered quiet). 1551 DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 1, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCmpLeF32 (le ordered quiet). 1552 DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 1, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCmpLeF64 (le ordered quiet). 1553 DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCmpOrdF32S (ordered quiet). 1554 DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCmpOrdF64S (ordered quiet). 1555 DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCmpOrdF32 (ordered quiet). 1556 DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCmpOrdF64 (ordered quiet). 1557 DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x01u), // kCmpUnordF32S (unordered quiet). 1558 DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x01u), // kCmpUnordF64S (unordered quiet). 1559 DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x01u), // kCmpUnordF32 (unordered quiet). 1560 DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x01u), // kCmpUnordF64 (unordered quiet). 1561 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kHAddF64. 1562 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kCombineLoHiU64. 1563 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kCombineLoHiF64. 1564 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kSrc , kNone, k64, kNA, k64, kNA, 0x00u), // kCombineHiLoU64. 1565 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kSrc , kNone, k64, kNA, k64, kNA, 0x00u), // kCombineHiLoF64. 1566 DEFINE_OP(Inst::kIdZip1_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kInterleaveLoU8. 1567 DEFINE_OP(Inst::kIdZip2_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kInterleaveHiU8. 1568 DEFINE_OP(Inst::kIdZip1_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kInterleaveLoU16. 1569 DEFINE_OP(Inst::kIdZip2_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kInterleaveHiU16. 1570 DEFINE_OP(Inst::kIdZip1_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInterleaveLoU32. 1571 DEFINE_OP(Inst::kIdZip2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInterleaveHiU32. 1572 DEFINE_OP(Inst::kIdZip1_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInterleaveLoU64. 1573 DEFINE_OP(Inst::kIdZip2_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInterleaveHiU64. 1574 DEFINE_OP(Inst::kIdZip1_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInterleaveLoF32. 1575 DEFINE_OP(Inst::kIdZip2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInterleaveHiF32. 1576 DEFINE_OP(Inst::kIdZip1_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInterleaveLoF64. 1577 DEFINE_OP(Inst::kIdZip2_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInterleaveHiF64. 1578 DEFINE_OP(Inst::kIdSqxtn_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k16, kNA, 0x00u), // kPacksI16_I8. 1579 DEFINE_OP(Inst::kIdSqxtun_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k16, kNA, 0x00u), // kPacksI16_U8. 1580 DEFINE_OP(Inst::kIdSqxtn_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k32, kNA, 0x00u), // kPacksI32_I16. 1581 DEFINE_OP(Inst::kIdSqxtun_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k32, kNA, 0x00u), // kPacksI32_U16. 1582 DEFINE_OP(Inst::kIdTbl_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kSwizzlev_U8. 1583 DEFINE_OP(Inst::kIdSmull_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kLo, 0x00u), // kMulwLoI8. 1584 DEFINE_OP(Inst::kIdUmull_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kLo, 0x00u), // kMulwLoU8. 1585 DEFINE_OP(Inst::kIdSmull2_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kHi, 0x00u), // kMulwHiI8. 1586 DEFINE_OP(Inst::kIdUmull2_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kHi, 0x00u), // kMulwHiU8. 1587 DEFINE_OP(Inst::kIdSmull_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kLo, 0x00u), // kMulwLoI16. 1588 DEFINE_OP(Inst::kIdUmull_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kLo, 0x00u), // kMulwLoU16. 1589 DEFINE_OP(Inst::kIdSmull2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kHi, 0x00u), // kMulwHiI16. 1590 DEFINE_OP(Inst::kIdUmull2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kHi, 0x00u), // kMulwHiU16. 1591 DEFINE_OP(Inst::kIdSmull_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kLo, 0x00u), // kMulwLoI32. 1592 DEFINE_OP(Inst::kIdUmull_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kLo, 0x00u), // kMulwLoU32. 1593 DEFINE_OP(Inst::kIdSmull2_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kHi, 0x00u), // kMulwHiI32. 1594 DEFINE_OP(Inst::kIdUmull2_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kHi, 0x00u), // kMulwHiU32. 1595 DEFINE_OP(Inst::kIdSmlal_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kLo, 0x00u), // kMAddwLoI8. 1596 DEFINE_OP(Inst::kIdUmlal_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kLo, 0x00u), // kMAddwLoU8. 1597 DEFINE_OP(Inst::kIdSmlal2_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kHi, 0x00u), // kMAddwHiI8. 1598 DEFINE_OP(Inst::kIdUmlal2_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kHi, 0x00u), // kMAddwHiU8. 1599 DEFINE_OP(Inst::kIdSmlal_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kLo, 0x00u), // kMAddwLoI16. 1600 DEFINE_OP(Inst::kIdUmlal_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kLo, 0x00u), // kMAddwLoU16. 1601 DEFINE_OP(Inst::kIdSmlal2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kHi, 0x00u), // kMAddwHiI16. 1602 DEFINE_OP(Inst::kIdUmlal2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kHi, 0x00u), // kMAddwHiU16. 1603 DEFINE_OP(Inst::kIdSmlal_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kLo, 0x00u), // kMAddwLoI32. 1604 DEFINE_OP(Inst::kIdUmlal_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kLo, 0x00u), // kMAddwLoU32. 1605 DEFINE_OP(Inst::kIdSmlal2_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kHi, 0x00u), // kMAddwHiI32. 1606 DEFINE_OP(Inst::kIdUmlal2_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kHi, 0x00u) // kMAddwHiU32. 1607 }; 1608 1609 static constexpr UniOpVInfo opcode_info_3vi[size_t(UniOpVVVI::kMaxValue) + 1] = { 1610 DEFINE_OP(Inst::kIdExt_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kAlignr_U128. 1611 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInterleaveShuffleU32x4. 1612 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInterleaveShuffleU64x2. 1613 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInterleaveShuffleF32x4. 1614 DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInterleaveShuffleF64x2. 1615 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInsertV128_U32. 1616 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInsertV128_F32. 1617 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInsertV128_U64. 1618 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInsertV128_F64. 1619 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInsertV256_U32. 1620 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInsertV256_F32. 1621 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInsertV256_U64. 1622 DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u) // kInsertV256_F64. 1623 }; 1624 1625 static constexpr UniOpVInfo opcode_info_4v[size_t(UniOpVVV::kMaxValue) + 1] = { 1626 DEFINE_OP(Inst::kIdBsl_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kBlendV_U8. 1627 DEFINE_OP(Inst::kIdMla_v , kIntrin, 1, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kMAddU16. 1628 DEFINE_OP(Inst::kIdMla_v , kIntrin, 1, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kMAddU32. 1629 DEFINE_OP(Inst::kIdFmadd_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kMAddF32S. 1630 DEFINE_OP(Inst::kIdFmadd_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kMAddF64S. 1631 DEFINE_OP(Inst::kIdFmla_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kMAddF32. 1632 DEFINE_OP(Inst::kIdFmla_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kMAddF64. 1633 DEFINE_OP(Inst::kIdFnmsub_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kMSubF32S. 1634 DEFINE_OP(Inst::kIdFnmsub_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kMSubF64S. 1635 DEFINE_OP(Inst::kIdFmla_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x01u), // kMSubF32. 1636 DEFINE_OP(Inst::kIdFmla_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x01u), // kMSubF64. 1637 DEFINE_OP(Inst::kIdFmsub_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kNMAddF32S. 1638 DEFINE_OP(Inst::kIdFmsub_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kNMAddF64S. 1639 DEFINE_OP(Inst::kIdFmls_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kNMAddF32. 1640 DEFINE_OP(Inst::kIdFmls_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kNMAddF64. 1641 DEFINE_OP(Inst::kIdFnmadd_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kNMSubF32S. 1642 DEFINE_OP(Inst::kIdFnmadd_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kNMSubF64S. 1643 DEFINE_OP(Inst::kIdFmls_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x01u), // kNMSubF32. 1644 DEFINE_OP(Inst::kIdFmls_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x01u) // kNMSubF64. 1645 }; 1646 1647 #undef DEFINE_OP 1648 1649 struct UniOpVMInfo { 1650 //! \name Members 1651 //! \{ 1652 1653 uint32_t cvt_op : 16; 1654 uint32_t mem_size : 8; 1655 ElementSize element : 3; 1656 1657 //! \} 1658 }; 1659 1660 #define DEFINE_OP(cvt_op, size, element) UniOpVMInfo { uint32_t(cvt_op), size, ElementSize::element } 1661 1662 static constexpr UniOpVMInfo opcode_info_2vm[size_t(UniOpVM::kMaxValue) + 1] = { 1663 DEFINE_OP(0 , 1, k8 ), // kLoad8. 1664 DEFINE_OP(0 , 2, k16), // kLoad16_U16. 1665 DEFINE_OP(0 , 4, k32), // kLoad32_U32. 1666 DEFINE_OP(0 , 4, k32), // kLoad32_F32. 1667 DEFINE_OP(0 , 8, k32), // kLoad64_U32. 1668 DEFINE_OP(0 , 8, k64), // kLoad64_U64. 1669 DEFINE_OP(0 , 8, k32), // kLoad64_F32. 1670 DEFINE_OP(0 , 8, k64), // kLoad64_F64. 1671 DEFINE_OP(0 , 16, k32), // kLoad128_U32. 1672 DEFINE_OP(0 , 16, k64), // kLoad128_U64. 1673 DEFINE_OP(0 , 16, k32), // kLoad128_F32. 1674 DEFINE_OP(0 , 16, k64), // kLoad128_F64. 1675 DEFINE_OP(0 , 32, k32), // kLoad256_U32. 1676 DEFINE_OP(0 , 32, k64), // kLoad256_U64. 1677 DEFINE_OP(0 , 32, k32), // kLoad256_F32. 1678 DEFINE_OP(0 , 32, k64), // kLoad256_F64. 1679 DEFINE_OP(0 , 64, k32), // kLoad512_U32. 1680 DEFINE_OP(0 , 64, k64), // kLoad512_U64. 1681 DEFINE_OP(0 , 64, k32), // kLoad512_F32. 1682 DEFINE_OP(0 , 64, k64), // kLoad512_F64. 1683 DEFINE_OP(0 , 0, k32), // kLoadN_U32. 1684 DEFINE_OP(0 , 0, k64), // kLoadN_U64. 1685 DEFINE_OP(0 , 0, k32), // kLoadN_F32. 1686 DEFINE_OP(0 , 0, k64), // kLoadN_F64. 1687 DEFINE_OP(1 , 2, kNA), // kLoadCvt16_U8ToU64. 1688 DEFINE_OP(1 , 4, kNA), // kLoadCvt32_U8ToU64. 1689 DEFINE_OP(1 , 8, kNA), // kLoadCvt64_U8ToU64. 1690 DEFINE_OP(UniOpVV::kCvtI8LoToI16 , 4, kNA), // kLoadCvt32_I8ToI16. 1691 DEFINE_OP(UniOpVV::kCvtU8LoToU16 , 4, kNA), // kLoadCvt32_U8ToU16. 1692 DEFINE_OP(UniOpVV::kCvtI8ToI32 , 4, kNA), // kLoadCvt32_I8ToI32. 1693 DEFINE_OP(UniOpVV::kCvtU8ToU32 , 4, kNA), // kLoadCvt32_U8ToU32. 1694 DEFINE_OP(UniOpVV::kCvtI16LoToI32, 4, kNA), // kLoadCvt32_I16ToI32. 1695 DEFINE_OP(UniOpVV::kCvtU16LoToU32, 4, kNA), // kLoadCvt32_U16ToU32. 1696 DEFINE_OP(UniOpVV::kCvtI32LoToI64, 4, kNA), // kLoadCvt32_I32ToI64. 1697 DEFINE_OP(UniOpVV::kCvtU32LoToU64, 4, kNA), // kLoadCvt32_U32ToU64. 1698 DEFINE_OP(UniOpVV::kCvtI8LoToI16 , 8, kNA), // kLoadCvt64_I8ToI16. 1699 DEFINE_OP(UniOpVV::kCvtU8LoToU16 , 8, kNA), // kLoadCvt64_U8ToU16. 1700 DEFINE_OP(UniOpVV::kCvtI8ToI32 , 8, kNA), // kLoadCvt64_I8ToI32. 1701 DEFINE_OP(UniOpVV::kCvtU8ToU32 , 8, kNA), // kLoadCvt64_U8ToU32. 1702 DEFINE_OP(UniOpVV::kCvtI16LoToI32, 8, kNA), // kLoadCvt64_I16ToI32. 1703 DEFINE_OP(UniOpVV::kCvtU16LoToU32, 8, kNA), // kLoadCvt64_U16ToU32. 1704 DEFINE_OP(UniOpVV::kCvtI32LoToI64, 8, kNA), // kLoadCvt64_I32ToI64. 1705 DEFINE_OP(UniOpVV::kCvtU32LoToU64, 8, kNA), // kLoadCvt64_U32ToU64. 1706 DEFINE_OP(UniOpVV::kCvtI8LoToI16 , 16, kNA), // kLoadCvt128_I8ToI16. 1707 DEFINE_OP(UniOpVV::kCvtU8LoToU16 , 16, kNA), // kLoadCvt128_U8ToU16. 1708 DEFINE_OP(UniOpVV::kCvtI8ToI32 , 16, kNA), // kLoadCvt128_I8ToI32. 1709 DEFINE_OP(UniOpVV::kCvtU8ToU32 , 16, kNA), // kLoadCvt128_U8ToU32. 1710 DEFINE_OP(UniOpVV::kCvtI16LoToI32, 16, kNA), // kLoadCvt128_I16ToI32. 1711 DEFINE_OP(UniOpVV::kCvtU16LoToU32, 16, kNA), // kLoadCvt128_U16ToU32. 1712 DEFINE_OP(UniOpVV::kCvtI32LoToI64, 16, kNA), // kLoadCvt128_I32ToI64. 1713 DEFINE_OP(UniOpVV::kCvtU32LoToU64, 16, kNA), // kLoadCvt128_U32ToU64. 1714 DEFINE_OP(UniOpVV::kCvtI8LoToI16 , 32, kNA), // kLoadCvt256_I8ToI16. 1715 DEFINE_OP(UniOpVV::kCvtU8LoToU16 , 32, kNA), // kLoadCvt256_U8ToU16. 1716 DEFINE_OP(UniOpVV::kCvtI16LoToI32, 32, kNA), // kLoadCvt256_I16ToI32. 1717 DEFINE_OP(UniOpVV::kCvtU16LoToU32, 32, kNA), // kLoadCvt256_U16ToU32. 1718 DEFINE_OP(UniOpVV::kCvtI32LoToI64, 32, kNA), // kLoadCvt256_I32ToI64. 1719 DEFINE_OP(UniOpVV::kCvtU32LoToU64, 32, kNA), // kLoadCvt256_U32ToU64. 1720 DEFINE_OP(1 , 0, kNA), // kLoadCvtN_U8ToU64. 1721 DEFINE_OP(UniOpVV::kCvtI8LoToI16 , 0, kNA), // kLoadCvtN_I8ToI16. 1722 DEFINE_OP(UniOpVV::kCvtU8LoToU16 , 0, kNA), // kLoadCvtN_U8ToU16. 1723 DEFINE_OP(UniOpVV::kCvtI8ToI32 , 0, kNA), // kLoadCvtN_I8ToI32. 1724 DEFINE_OP(UniOpVV::kCvtU8ToU32 , 0, kNA), // kLoadCvtN_U8ToU32. 1725 DEFINE_OP(UniOpVV::kCvtI16LoToI32, 0, kNA), // kLoadCvtN_I16ToI32. 1726 DEFINE_OP(UniOpVV::kCvtU16LoToU32, 0, kNA), // kLoadCvtN_U16ToU32. 1727 DEFINE_OP(UniOpVV::kCvtI32LoToI64, 0, kNA), // kLoadCvtN_I32ToI64. 1728 DEFINE_OP(UniOpVV::kCvtU32LoToU64, 0, kNA), // kLoadCvtN_U32ToU64. 1729 DEFINE_OP(0 , 1, k8 ), // kLoadInsertU8. 1730 DEFINE_OP(0 , 2, k16), // kLoadInsertU16. 1731 DEFINE_OP(0 , 4, k32), // kLoadInsertU32. 1732 DEFINE_OP(0 , 8, k64), // kLoadInsertU64. 1733 DEFINE_OP(0 , 4, k32), // kLoadInsertF32. 1734 DEFINE_OP(0 , 8, k64), // kLoadInsertF32x2. 1735 DEFINE_OP(0 , 8, k64) // kLoadInsertF64. 1736 }; 1737 1738 static constexpr UniOpVMInfo opcode_info_2mv[size_t(UniOpMV::kMaxValue) + 1] = { 1739 DEFINE_OP(NarrowingOp::kNone , 1, k8 ), // kStore8. 1740 DEFINE_OP(NarrowingOp::kNone , 2, k16), // kStore16_U16. 1741 DEFINE_OP(NarrowingOp::kNone , 4, k32), // kStore32_U32. 1742 DEFINE_OP(NarrowingOp::kNone , 4, k32), // kStore32_F32. 1743 DEFINE_OP(NarrowingOp::kNone , 8, k32), // kStore64_U32. 1744 DEFINE_OP(NarrowingOp::kNone , 8, k64), // kStore64_U64. 1745 DEFINE_OP(NarrowingOp::kNone , 8, k32), // kStore64_F32. 1746 DEFINE_OP(NarrowingOp::kNone , 8, k64), // kStore64_F64. 1747 DEFINE_OP(NarrowingOp::kNone , 16, k32), // kStore128_U32. 1748 DEFINE_OP(NarrowingOp::kNone , 16, k64), // kStore128_U64. 1749 DEFINE_OP(NarrowingOp::kNone , 16, k32), // kStore128_F32. 1750 DEFINE_OP(NarrowingOp::kNone , 16, k64), // kStore128_F64. 1751 DEFINE_OP(NarrowingOp::kNone , 32, k32), // kStore256_U32. 1752 DEFINE_OP(NarrowingOp::kNone , 32, k64), // kStore256_U64. 1753 DEFINE_OP(NarrowingOp::kNone , 32, k32), // kStore256_F32. 1754 DEFINE_OP(NarrowingOp::kNone , 32, k64), // kStore256_F64. 1755 DEFINE_OP(NarrowingOp::kNone , 64, k32), // kStore512_U32. 1756 DEFINE_OP(NarrowingOp::kNone , 64, k64), // kStore512_U64. 1757 DEFINE_OP(NarrowingOp::kNone , 64, k32), // kStore512_F32. 1758 DEFINE_OP(NarrowingOp::kNone , 64, k64), // kStore512_F64. 1759 DEFINE_OP(NarrowingOp::kNone , 0, k32), // kStoreN_U32. 1760 DEFINE_OP(NarrowingOp::kNone , 0, k64), // kStoreN_U64. 1761 DEFINE_OP(NarrowingOp::kNone , 0, k32), // kStoreN_F32. 1762 DEFINE_OP(NarrowingOp::kNone , 0, k64) // kStoreN_F64. 1763 /* 1764 DEFINE_OP(NarrowingOp::kU16ToU8 , 8, kNA), // kStoreCvtz64_U16ToU8. 1765 DEFINE_OP(NarrowingOp::kU32ToU16 , 8, kNA), // kStoreCvtz64_U32ToU16. 1766 DEFINE_OP(NarrowingOp::kU64ToU32 , 8, kNA), // kStoreCvtz64_U64ToU32. 1767 DEFINE_OP(NarrowingOp::kI16ToI8 , 8, kNA), // kStoreCvts64_I16ToI8. 1768 DEFINE_OP(NarrowingOp::kI16ToU8 , 8, kNA), // kStoreCvts64_I16ToU8. 1769 DEFINE_OP(NarrowingOp::kU16ToU8 , 8, kNA), // kStoreCvts64_U16ToU8. 1770 DEFINE_OP(NarrowingOp::kI32ToI16 , 8, kNA), // kStoreCvts64_I32ToI16. 1771 DEFINE_OP(NarrowingOp::kU32ToU16 , 8, kNA), // kStoreCvts64_U32ToU16. 1772 DEFINE_OP(NarrowingOp::kI64ToI32 , 8, kNA), // kStoreCvts64_I64ToI32. 1773 DEFINE_OP(NarrowingOp::kU64ToU32 , 8, kNA), // kStoreCvts64_U64ToU32. 1774 DEFINE_OP(NarrowingOp::kU16ToU8 , 16, kNA), // kStoreCvtz128_U16ToU8. 1775 DEFINE_OP(NarrowingOp::kU32ToU16 , 16, kNA), // kStoreCvtz128_U32ToU16. 1776 DEFINE_OP(NarrowingOp::kU64ToU32 , 16, kNA), // kStoreCvtz128_U64ToU32. 1777 DEFINE_OP(NarrowingOp::kI16ToI8 , 16, kNA), // kStoreCvts128_I16ToI8. 1778 DEFINE_OP(NarrowingOp::kI16ToU8 , 16, kNA), // kStoreCvts128_I16ToU8. 1779 DEFINE_OP(NarrowingOp::kU16ToU8 , 16, kNA), // kStoreCvts128_U16ToU8. 1780 DEFINE_OP(NarrowingOp::kI32ToI16 , 16, kNA), // kStoreCvts128_I32ToI16. 1781 DEFINE_OP(NarrowingOp::kU32ToU16 , 16, kNA), // kStoreCvts128_U32ToU16. 1782 DEFINE_OP(NarrowingOp::kI64ToI32 , 16, kNA), // kStoreCvts128_I64ToI32. 1783 DEFINE_OP(NarrowingOp::kU64ToU32 , 16, kNA), // kStoreCvts128_U64ToU32. 1784 DEFINE_OP(NarrowingOp::kU16ToU8 , 32, kNA), // kStoreCvtz256_U16ToU8. 1785 DEFINE_OP(NarrowingOp::kU32ToU16 , 32, kNA), // kStoreCvtz256_U32ToU16. 1786 DEFINE_OP(NarrowingOp::kU64ToU32 , 32, kNA), // kStoreCvtz256_U64ToU32. 1787 DEFINE_OP(NarrowingOp::kI16ToI8 , 32, kNA), // kStoreCvts256_I16ToI8. 1788 DEFINE_OP(NarrowingOp::kI16ToU8 , 32, kNA), // kStoreCvts256_I16ToU8. 1789 DEFINE_OP(NarrowingOp::kU16ToU8 , 32, kNA), // kStoreCvts256_U16ToU8. 1790 DEFINE_OP(NarrowingOp::kI32ToI16 , 32, kNA), // kStoreCvts256_I32ToI16. 1791 DEFINE_OP(NarrowingOp::kU32ToU16 , 32, kNA), // kStoreCvts256_U32ToU16. 1792 DEFINE_OP(NarrowingOp::kI64ToI32 , 32, kNA), // kStoreCvts256_I64ToI32. 1793 DEFINE_OP(NarrowingOp::kU64ToU32 , 32, kNA), // kStoreCvts256_U64ToU32. 1794 DEFINE_OP(NarrowingOp::kU16ToU8 , 0, kNA), // kStoreCvtzN_U16ToU8. 1795 DEFINE_OP(NarrowingOp::kU32ToU16 , 0, kNA), // kStoreCvtzN_U32ToU16. 1796 DEFINE_OP(NarrowingOp::kU64ToU32 , 0, kNA), // kStoreCvtzN_U64ToU32. 1797 DEFINE_OP(NarrowingOp::kI16ToI8 , 0, kNA), // kStoreCvtsN_I16ToI8. 1798 DEFINE_OP(NarrowingOp::kI16ToU8 , 0, kNA), // kStoreCvtsN_I16ToU8. 1799 DEFINE_OP(NarrowingOp::kU16ToU8 , 0, kNA), // kStoreCvtsN_U16ToU8. 1800 DEFINE_OP(NarrowingOp::kI32ToI16 , 0, kNA), // kStoreCvtsN_I32ToI16. 1801 DEFINE_OP(NarrowingOp::kU32ToU16 , 0, kNA), // kStoreCvtsN_U32ToU16. 1802 DEFINE_OP(NarrowingOp::kI64ToI32 , 0, kNA), // kStoreCvtsN_I64ToI32. 1803 DEFINE_OP(NarrowingOp::kU64ToU32 , 0, kNA) // kStoreCvtsN_U64ToU32. 1804 */ 1805 }; 1806 1807 #undef DEFINE_OP 1808 1809 // ujit::UniCompiler - Vector Instructions - Utility Functions 1810 // =========================================================== 1811 1812 static constexpr uint32_t float_mode_mem_size_table[5] = { 0, 4, 8, 0, 0 }; 1813 1814 static ASMJIT_INLINE bool is_same_vec(const Vec& a, const Operand_& b) noexcept { 1815 return b.is_vec() && a.id() == b.id(); 1816 } 1817 1818 static ASMJIT_INLINE void vec_set_vec_type(Vec& vec, ElementSize sz) noexcept { 1819 static constexpr uint32_t signatures[5] = { 1820 RegTraits<RegType::kVec8>::kSignature, 1821 RegTraits<RegType::kVec16>::kSignature, 1822 RegTraits<RegType::kVec32>::kSignature, 1823 RegTraits<RegType::kVec64>::kSignature, 1824 RegTraits<RegType::kVec128>::kSignature 1825 }; 1826 vec.set_signature(OperandSignature{signatures[size_t(sz)]}); 1827 } 1828 1829 static ASMJIT_INLINE void vec_set_type(Vec& vec, ElementSize sz) noexcept { 1830 vec.set_element_type(a64::VecElementType(uint32_t(sz) + 1)); 1831 } 1832 1833 static ASMJIT_INLINE void vec_set_type_and_index(Vec& vec, ElementSize sz, uint32_t idx) noexcept { 1834 vec.set_element_type(a64::VecElementType(uint32_t(sz) + 1)); 1835 vec.set_element_index(idx); 1836 } 1837 1838 static ASMJIT_NOINLINE void vec_load_mem(UniCompiler& uc, const Vec& dst, Mem src, uint32_t mem_size) { 1839 BackendCompiler* cc = uc.cc; 1840 1841 if (src.has_index() && src.has_shift()) { 1842 // AArch64 limitation: index shift can be the same size as the size of the read operation, so H << 1, S << 2, 1843 // etc... Other shift values are not supported at the architectural level, so we have to precalculate the address. 1844 uint32_t shift = src.shift(); 1845 if (mem_size != (1u << shift) || src.has_offset()) { 1846 Gp base = src.base_reg().as<Gp>(); 1847 Gp index = src.index_reg().as<Gp>(); 1848 1849 if (src.is_pre_index()) { 1850 cc->add(base, base, index, a64::Shift(src.shift_op(), shift)); 1851 src = a64::ptr(base, src.offset_lo32()); 1852 } 1853 else { 1854 Gp tmp = uc.new_gpz("@mem_addr"); 1855 cc->add(tmp, base, index, a64::Shift(src.shift_op(), shift)); 1856 src = a64::ptr(tmp, src.offset_lo32()); 1857 } 1858 } 1859 } 1860 1861 switch (mem_size) { 1862 case 1: cc->ldr(dst.b(), src); break; 1863 case 2: cc->ldr(dst.h(), src); break; 1864 case 4: cc->ldr(dst.s(), src); break; 1865 case 8: cc->ldr(dst.d(), src); break; 1866 case 16: cc->ldr(dst.q(), src); break; 1867 default: 1868 ASMJIT_NOT_REACHED(); 1869 } 1870 } 1871 1872 static ASMJIT_NOINLINE Vec vec_from_mem(UniCompiler& uc, const Mem& op, const Vec& ref, uint32_t mem_size = 0) { 1873 Vec vec = uc.new_vec128("@tmp"); 1874 if (mem_size == 0) 1875 mem_size = ref.size(); 1876 vec_load_mem(uc, vec, op, mem_size); 1877 return vec.clone_as(ref); 1878 } 1879 1880 static ASMJIT_INLINE Vec as_vec(UniCompiler& uc, const Operand_& op, const Vec& ref, uint32_t mem_size = 0) { 1881 if (op.is_vec()) 1882 return op.as<Vec>().clone_as(ref); 1883 else 1884 return vec_from_mem(uc, op.as<Mem>(), ref, mem_size); 1885 } 1886 1887 static ASMJIT_INLINE Vec as_vec(UniCompiler& uc, const Operand_& op, const Vec& ref, FloatMode fm) { 1888 if (op.is_vec()) 1889 return op.as<Vec>().clone_as(ref); 1890 else 1891 return vec_from_mem(uc, op.as<Mem>(), ref, float_mode_mem_size_table[size_t(fm)]); 1892 } 1893 1894 static ASMJIT_NOINLINE Vec vec_mov(UniCompiler& uc, const Vec& dst_, const Operand_& src_) { 1895 BackendCompiler* cc = uc.cc; 1896 1897 Vec dst(dst_); 1898 vec_set_type(dst, ElementSize::k8); 1899 1900 if (src_.is_vec()) { 1901 if (dst.id() != src_.id()) { 1902 Vec src = src_.as<Vec>(); 1903 vec_set_type(src, ElementSize::k8); 1904 cc->mov(dst, src.as<Vec>()); 1905 } 1906 return dst; 1907 } 1908 1909 if (src_.is_mem()) { 1910 vec_load_mem(uc, dst, src_.as<Mem>(), dst.size()); 1911 return dst; 1912 } 1913 1914 ASMJIT_NOT_REACHED(); 1915 } 1916 1917 static ASMJIT_NOINLINE void vec_neg(UniCompiler& uc, const Vec& dst, const Vec& src, FloatMode fm) { 1918 BackendCompiler* cc = uc.cc; 1919 1920 if (fm == FloatMode::kF32S) 1921 cc->mvn_(dst.s(), src.s()); 1922 else if (fm == FloatMode::kF64S) 1923 cc->mvn_(dst.d(), src.d()); 1924 else 1925 cc->mvn_(dst.q(), src.q()); 1926 } 1927 1928 // ujit::UniCompiler - Vector Instructions - Swizzle 32 Impl 1929 // ========================================================= 1930 1931 // [DCBA] <- Mov (DCBA, dcba) 1932 // [AAAA] <- Dup0 (DCBA, dcba) 1933 // [BBBB] <- Dup1 (DCBA, dcba) 1934 // [CCCC] <- Dup2 (DCBA, dcba) 1935 // [DDDD] <- Dup3 (DCBA, dcba) 1936 // [CDAB] <- Rev64 (DCBA, dcba) 1937 // [aDCB] <- Ext4 (DCBA, dcba) [dcb|aDCB|A ] 1938 // [baDC] <- Ext8 (DCBA, dcba) [ dc|baDC|BA ] 1939 // [cbaD] <- Ext12 (DCBA, dcba) [ d|cbaD|CBA] 1940 // [bBaA] <- Zip1_4S(DCBA, dcba) 1941 // [baBA] <- Zip1_2D(DCBA, dcba) 1942 // [dDcC] <- Zip2_4S(DCBA, dcba) 1943 // [dcDC] <- Zip2_2D(DCBA, dcba) 1944 // [caCA] <- Uzp1_4S(DCBA, dcba) 1945 // [baBA] <- Uzp1_2D(DCBA, dcba) 1946 // [dbDB] <- Uzp2_4S(DCBA, dcba) 1947 // [dcDC] <- Uzp2_2D(DCBA, dcba) 1948 // [cCaA] <- Trn1_4S(DCBA, dcba) 1949 // [dDbB] <- Trn2_4S(DCBA, dcba) 1950 struct Swizzle32Data { 1951 enum class OpTarget : uint8_t { 1952 kDst = 0, 1953 k1 = 1, 1954 k2 = 2, 1955 kA = 3, 1956 1957 k_ = kDst 1958 }; 1959 1960 struct Op { 1961 //! Swizzle operation does nothing - 'mov' (this can be only the first operation). 1962 static constexpr uint8_t kMov = 1; 1963 //! Swizzle operation performs an insert - moves a value from one lane to another. 1964 static constexpr uint8_t kIns = 2; 1965 //! Swizzle operation duplicates a lane across all others - 'dup'. 1966 static constexpr uint8_t kDup = 3; 1967 //! Swizzle operation rotates a vector - 'ext'. 1968 static constexpr uint8_t kExt = 4; 1969 //! Swizzle operation swaps lo/hi elements of 64-bit lanes - 'rev64'. 1970 static constexpr uint8_t kRev64 = 5; 1971 //! Swizzle operation can be implemented as a single zip[1|2], uzp[1|2], or trn[1|2] instruction with 32-bit or 64-bit elements. 1972 static constexpr uint8_t kZipUnzip = 7; 1973 1974 static constexpr uint8_t kIns0To1 = uint8_t(kIns ) | (0 << 4) | (1 << 6); 1975 static constexpr uint8_t kIns0To2 = uint8_t(kIns ) | (0 << 4) | (2 << 6); 1976 static constexpr uint8_t kIns0To3 = uint8_t(kIns ) | (0 << 4) | (3 << 6); 1977 static constexpr uint8_t kIns1To0 = uint8_t(kIns ) | (1 << 4) | (0 << 6); 1978 static constexpr uint8_t kIns1To2 = uint8_t(kIns ) | (1 << 4) | (2 << 6); 1979 static constexpr uint8_t kIns1To3 = uint8_t(kIns ) | (1 << 4) | (3 << 6); 1980 static constexpr uint8_t kIns2To0 = uint8_t(kIns ) | (2 << 4) | (0 << 6); 1981 static constexpr uint8_t kIns2To1 = uint8_t(kIns ) | (2 << 4) | (1 << 6); 1982 static constexpr uint8_t kIns2To3 = uint8_t(kIns ) | (2 << 4) | (3 << 6); 1983 static constexpr uint8_t kIns3To0 = uint8_t(kIns ) | (3 << 4) | (0 << 6); 1984 static constexpr uint8_t kIns3To1 = uint8_t(kIns ) | (3 << 4) | (1 << 6); 1985 static constexpr uint8_t kIns3To2 = uint8_t(kIns ) | (3 << 4) | (2 << 6); 1986 static constexpr uint8_t kDup0 = uint8_t(kDup ) | (0 << 4); 1987 static constexpr uint8_t kDup1 = uint8_t(kDup ) | (1 << 4); 1988 static constexpr uint8_t kDup2 = uint8_t(kDup ) | (2 << 4); 1989 static constexpr uint8_t kDup3 = uint8_t(kDup ) | (3 << 4); 1990 static constexpr uint8_t kExt4 = uint8_t(kExt ) | (1 << 4); 1991 static constexpr uint8_t kExt8 = uint8_t(kExt ) | (2 << 4); 1992 static constexpr uint8_t kExt12 = uint8_t(kExt ) | (3 << 4); 1993 static constexpr uint8_t kZip1_4S = uint8_t(kZipUnzip) | (0 << 4) | (0 << 7); 1994 static constexpr uint8_t kZip1_2D = uint8_t(kZipUnzip) | (0 << 4) | (1 << 7); 1995 static constexpr uint8_t kZip2_4S = uint8_t(kZipUnzip) | (1 << 4) | (0 << 7); 1996 static constexpr uint8_t kZip2_2D = uint8_t(kZipUnzip) | (1 << 4) | (1 << 7); 1997 static constexpr uint8_t kUzp1_4S = uint8_t(kZipUnzip) | (2 << 4) | (0 << 7); 1998 static constexpr uint8_t kUzp1_2D = uint8_t(kZipUnzip) | (2 << 4) | (1 << 7); 1999 static constexpr uint8_t kUzp2_4S = uint8_t(kZipUnzip) | (3 << 4) | (0 << 7); 2000 static constexpr uint8_t kUzp2_2D = uint8_t(kZipUnzip) | (3 << 4) | (1 << 7); 2001 static constexpr uint8_t kTrn1_4S = uint8_t(kZipUnzip) | (4 << 4); 2002 static constexpr uint8_t kTrn2_4S = uint8_t(kZipUnzip) | (5 << 4); 2003 2004 // Alias to nothing to make the table easier to read. 2005 static constexpr uint8_t k_ = 0; 2006 2007 uint8_t data; 2008 2009 ASMJIT_INLINE_NODEBUG bool is_valid() const noexcept { return data != 0; } 2010 ASMJIT_INLINE_NODEBUG uint32_t type() const noexcept { return data & 0xF; } 2011 2012 ASMJIT_INLINE_NODEBUG uint32_t dup_idx() const noexcept { return (data >> 4) & 0x3; } 2013 ASMJIT_INLINE_NODEBUG uint32_t ext_imm() const noexcept { return (data >> 2) & (0x3 << 2); } 2014 2015 ASMJIT_INLINE_NODEBUG uint32_t zip_op() const noexcept { return (data >> 4) & 0x7; } 2016 ASMJIT_INLINE_NODEBUG bool zip_s4() const noexcept { return (data & (1 << 7)) == 0; } 2017 2018 ASMJIT_INLINE_NODEBUG uint32_t ins_src() const noexcept { return (data >> 4) & 0x3; } 2019 ASMJIT_INLINE_NODEBUG uint32_t ins_dst() const noexcept { return (data >> 6) & 0x3; } 2020 }; 2021 2022 //! \name Members 2023 //! \{ 2024 2025 Op ops[3]; 2026 uint8_t flags; 2027 2028 //! \} 2029 2030 //! \name Accessors 2031 //! \{ 2032 2033 ASMJIT_INLINE_NODEBUG bool is_defined() const noexcept { return ops[0].data != 0; } 2034 ASMJIT_INLINE_NODEBUG Op op(uint32_t index) const noexcept { return ops[index]; } 2035 ASMJIT_INLINE_NODEBUG OpTarget op_target(uint32_t index) const noexcept { return OpTarget((flags >> (index * 2)) & 0x3); } 2036 2037 //! \} 2038 }; 2039 2040 // This table provides all combinations for all possible 32-bit swizzles (there is 256 combinations in total). 2041 // It prioritizes lane moves, and then operations that can have either one or two inputs. Each operation has 2042 // a target, which specifies whether it replaces the destination or one or both sources that are then passed 2043 // to a next operation. The last operation must always be `OpTarget::kDst` so the result ends up in the right 2044 // register. 2045 // 2046 // In general the decomposition of operations needed for all swizzles is as follows: 2047 // 2048 // - 1 Op Swizzles: 17 2049 // - 2 Op Swizzles: 156 2050 // - 3 Op Swizzles: 83 2051 // 2052 // Which means that luckily most used swizzles would fall into 1 or 2 operations. 2053 // 2054 // NOTE: Moves (InsXToY) operations only happen on the destination as they are destructive, which is perfectly 2055 // okay as moving them into earlier steps didn't really improve anything. 2056 #define OP(swiz, op0, target0, op1, target1, op2, target2) { \ 2057 { \ 2058 {Swizzle32Data::Op::k##op0}, \ 2059 {Swizzle32Data::Op::k##op1}, \ 2060 {Swizzle32Data::Op::k##op2} \ 2061 }, \ 2062 ( \ 2063 (uint16_t(Swizzle32Data::OpTarget::k##target0) << 0) | \ 2064 (uint16_t(Swizzle32Data::OpTarget::k##target1) << 2) | \ 2065 (uint16_t(Swizzle32Data::OpTarget::k##target2) << 4) \ 2066 ) \ 2067 } 2068 2069 static constexpr Swizzle32Data swizzle_32_data[256] = { 2070 OP(0000, Dup0 , _, _ , _, _ , _), OP(0001, Rev64 , _, Ins1To2, _, Ins1To3, _), OP(0002, Ext8 , _, Ins2To1, _, Ins2To3, _), OP(0003, Dup0 , 2, Ext12 , _, _ , _), 2071 OP(0010, Zip1_2D, _, Ins0To3, _, _ , _), OP(0011, Rev64 , A, Zip1_4S, _, _ , _), OP(0012, Ext8 , _, Ins3To1, _, Ins2To3, _), OP(0013, Rev64 , 2, Ext12 , _, Ins2To3, _), 2072 OP(0020, Uzp1_4S, _, Ins0To3, _, _ , _), OP(0021, Ext4 , _, Ins3To2, _, _ , _), OP(0022, Ext4 , A, Trn2_4S, _, _ , _), OP(0023, Ext4 , _, Ins2To0, _, Ins3To2, _), 2073 OP(0030, Ext8 , _, Ins2To0, _, Ins2To3, _), OP(0031, Dup0 , 2, Uzp2_4S, _, _ , _), OP(0032, Ext8 , _, Ins2To3, _, _ , _), OP(0033, Ext4 , A, Zip2_4S, _, _ , _), 2074 OP(0100, Zip1_4S, _, Ins0To3, _, _ , _), OP(0101, Rev64 , A, Zip1_2D, _, _ , _), OP(0102, Ext12 , _, Ins3To0, _, Ins1To3, _), OP(0103, Ext12 , _, Ins1To3, _, _ , _), 2075 OP(0110, Rev64 , 2, Zip1_4S, _, _ , _), OP(0111, Dup1 , 1, Ext4 , _, _ , _), OP(0112, Rev64 , 2, Ext8 , _, Ins2To1, _), OP(0113, Ext12 , _, Ins1To3, _, Ins2To1, _), 2076 OP(0120, Ext4 , _, Ins0To2, _, Ins3To0, _), OP(0121, Ext4 , _, Ins0To2, _, _ , _), OP(0122, Ext4 , _, Ins0To2, _, Ins1To0, _), OP(0123, Rev64 , A, Ext8 , _, _ , _), 2077 OP(0130, Ext12 , 2, Zip1_4S, _, _ , _), OP(0131, Ext4 , _, Ins2To1, _, Ins0To2, _), OP(0132, Rev64 , 2, Ext8 , _, _ , _), OP(0133, Ext12 , _, Ins1To3, _, Ins0To1, _), 2078 OP(0200, Trn1_4S, _, Ins0To3, _, _ , _), OP(0201, Rev64 , _, Ins3To2, _, Ins1To3, _), OP(0202, Ext4 , A, Uzp2_4S, _, _ , _), OP(0203, Uzp1_4S, 2, Ext12 , _, _ , _), 2079 OP(0210, Mov , _, Ins0To3, _, _ , _), OP(0211, Mov , _, Ins0To3, _, Ins1To0, _), OP(0212, Mov , _, Ins0To3, _, Ins2To0, _), OP(0213, Rev64 , A, Ext8 , 1, Zip1_4S, _), 2080 OP(0220, Ext8 , 2, Uzp1_4S, _, _ , _), OP(0221, Ext4 , _, Ins1To2, _, _ , _), OP(0222, Dup2 , 1, Ext4 , _, _ , _), OP(0223, Ext4 , _, Ins2To0, _, Ins1To2, _), 2081 OP(0230, Rev64 , 1, Ext4 , _, _ , _), OP(0231, Ext4 , 2, Uzp2_4S, _, _ , _), OP(0232, Ext8 , _, Ins2To3, _, Ins0To2, _), OP(0233, Rev64 , 1, Ext4 , _, Ins1To0, _), 2082 OP(0300, Rev64 , _, Ins1To0, _, Ins1To3, _), OP(0301, Rev64 , _, Ins1To3, _, _ , _), OP(0302, Dup0 , 2, Zip2_4S, _, _ , _), OP(0303, Ext4 , A, Zip2_2D, _, _ , _), 2083 OP(0310, Ext12 , 2, Zip1_2D, _, _ , _), OP(0311, Ext4 , _, Ins0To1, _, _ , _), OP(0312, Dup0 , 2, Zip1_4S, 2, Zip2_4S, _), OP(0313, Uzp2_4S, 1, Ext4 , _, _ , _), 2084 OP(0320, Ext4 , _, Ins3To0, _, _ , _), OP(0321, Ext4 , _, _ , _, _ , _), OP(0322, Ext4 , _, Ins1To0, _, _ , _), OP(0323, Ext4 , _, Ins2To0, _, _ , _), 2085 OP(0330, Ext4 , _, Ins2To1, _, Ins3To0, _), OP(0331, Ext4 , _, Ins2To1, _, _ , _), OP(0332, Ext4 , 2, Zip2_4S, _, _ , _), OP(0333, Dup3 , 1, Ext4 , _, _ , _), 2086 OP(1000, Zip1_4S, _, Ins0To2, _, _ , _), OP(1001, Rev64 , 1, Zip1_4S, _, _ , _), OP(1002, Ext8 , _, Ins2To1, _, _ , _), OP(1003, Ext4 , 1, Ext8 , _, _ , _), 2087 OP(1010, Zip1_2D, _, _ , _, _ , _), OP(1011, Zip1_2D, _, Ins1To0, _, _ , _), OP(1012, Ext8 , _, Ins3To1, _, _ , _), OP(1013, Ext8 , _, Ins1To0, _, Ins3To1, _), 2088 OP(1020, Zip1_4S, 2, Uzp1_4S, _, _ , _), OP(1021, Ext4 , 1, Zip1_2D, _, _ , _), OP(1022, Ext8 , _, Ins0To1, _, _ , _), OP(1023, Rev64 , 1, Ext8 , _, _ , _), 2089 OP(1030, Ext8 , _, Ins2To0, _, _ , _), OP(1031, Ext8 , _, Ins3To0, _, _ , _), OP(1032, Ext8 , _, _ , _, _ , _), OP(1033, Ext8 , _, Ins1To0, _, _ , _), 2090 OP(1100, Zip1_4S, _, _ , _, _ , _), OP(1101, Zip1_4S, _, Ins2To0, _, _ , _), OP(1102, Ext8 , _, Ins2To1, _, Ins3To2, _), OP(1103, Ext12 , _, Ins2To3, _, _ , _), 2091 OP(1110, Zip1_4S, _, Ins2To1, _, _ , _), OP(1111, Dup1 , _, _ , _, _ , _), OP(1112, Ext8 , _, Ins3To1, _, Ins3To2, _), OP(1113, Dup1 , 2, Ext12 , _, _ , _), 2092 OP(1120, Dup1 , 2, Uzp1_4S, _, _ , _), OP(1121, Ext4 , _, Ins0To2, _, Ins0To3, _), OP(1122, Ext8 , _, Ins0To1, _, Ins3To2, _), OP(1123, Ext12 , _, Ins3To1, _, Ins2To3, _), 2093 OP(1130, Ext8 , _, Ins2To0, _, Ins3To2, _), OP(1131, Uzp2_4S, _, Ins0To3, _, _ , _), OP(1132, Ext8 , _, Ins3To2, _, _ , _), OP(1133, Ext8 , A, Trn2_4S, _, _ , _), 2094 OP(1200, Zip1_4S, 2, Trn1_4S, _, _ , _), OP(1201, Ext4 , 1, Zip1_4S, _, _ , _), OP(1202, Dup2 , 1, Zip1_4S, _, _ , _), OP(1203, Dup1 , 2, Uzp1_4S, 2, Ext12 , _), 2095 OP(1210, Mov , _, Ins1To3, _, _ , _), OP(1211, Mov , _, Ins1To0, _, Ins1To3, _), OP(1212, Mov , _, Ins1To3, _, Ins2To0, _), OP(1213, Mov , _, Ins3To0, _, Ins1To3, _), 2096 OP(1220, Mov , _, Ins1To3, _, Ins2To1, _), OP(1221, Ext4 , _, Ins0To3, _, Ins1To2, _), OP(1222, Ext8 , _, Ins0To1, _, Ins0To2, _), OP(1223, Rev64 , 1, Ext8 , _, Ins1To2, _), 2097 OP(1230, Rev64 , A, Ext4 , _, _ , _), OP(1231, Ext8 , _, Ins0To2, _, Ins3To0, _), OP(1232, Ext8 , _, Ins0To2, _, _ , _), OP(1233, Ext8 , _, Ins0To2, _, Ins1To0, _), 2098 OP(1300, Rev64 , _, Ins0To3, _, Ins1To0, _), OP(1301, Rev64 , _, Ins0To3, _, _ , _), OP(1302, Ext8 , 1, Zip1_4S, _, _ , _), OP(1303, Dup3 , 1, Zip1_4S, _, _ , _), 2099 OP(1310, Mov , _, Ins3To2, _, Ins1To3, _), OP(1311, Trn2_4S, _, Ins0To3, _, _ , _), OP(1312, Dup1 , 2, Zip2_4S, _, _ , _), OP(1313, Ext8 , A, Uzp2_4S, _, _ , _), 2100 OP(1320, Ext12 , 2, Uzp1_4S, _, _ , _), OP(1321, Ext4 , _, Ins0To3, _, _ , _), OP(1322, Ext4 , _, Ins0To3, _, Ins1To0, _), OP(1323, Ext4 , _, Ins0To3, _, Ins2To0, _), 2101 OP(1330, Ext8 , _, Ins2To0, _, Ins1To2, _), OP(1331, Ext8 , 2, Uzp2_4S, _, _ , _), OP(1332, Ext8 , _, Ins1To2, _, _ , _), OP(1333, Ext8 , _, Ins1To0, _, Ins1To2, _), 2102 OP(2000, Uzp1_4S, _, Ins0To1, _, _ , _), OP(2001, Rev64 , _, Ins1To2, _, _ , _), OP(2002, Ext8 , 1, Uzp1_4S, _, _ , _), OP(2003, Ext12 , _, Ins1To2, _, _ , _), 2103 OP(2010, Zip1_4S, 1, Uzp1_4S, _, _ , _), OP(2011, Dup1 , 1, Uzp1_4S, _, _ , _), OP(2012, Ext8 , _, Ins3To1, _, Ins0To3, _), OP(2013, Ext12 , 1, Uzp1_4S, _, _ , _), 2104 OP(2020, Uzp1_4S, _, _ , _, _ , _), OP(2021, Rev64 , _, Ins1To2, _, Ins3To1, _), OP(2022, Uzp1_4S, _, Ins1To0, _, _ , _), OP(2023, Ext12 , _, Ins1To2, _, Ins3To1, _), 2105 OP(2030, Ext8 , _, Ins0To3, _, Ins2To0, _), OP(2031, Rev64 , 1, Uzp1_4S, _, _ , _), OP(2032, Ext8 , _, Ins0To3, _, _ , _), OP(2033, Dup3 , 1, Uzp1_4S, _, _ , _), 2106 OP(2100, Ext12 , _, Ins1To0, _, _ , _), OP(2101, Rev64 , _, Ins0To2, _, _ , _), OP(2102, Ext12 , _, Ins3To0, _, _ , _), OP(2103, Ext12 , _, _ , _, _ , _), 2107 OP(2110, Ext4 , 2, Zip1_4S, _, _ , _), OP(2111, Rev64 , _, Ins0To1, _, Ins0To2, _), OP(2112, Ext12 , _, Ins2To1, _, Ins3To0, _), OP(2113, Ext12 , _, Ins2To1, _, _ , _), 2108 OP(2120, Dup2 , 2, Zip1_4S, _, _ , _), OP(2121, Ext4 , A, Zip1_2D, _, _ , _), OP(2122, Ext12 , _, Ins3To0, _, Ins3To1, _), OP(2123, Ext12 , _, Ins3To1, _, _ , _), 2109 OP(2130, Dup2 , 2, Ext12 , 2, Zip1_4S, _), OP(2131, Rev64 , _, Ins2To1, _, Ins0To2, _), OP(2132, Ext4 , 2, Ext8 , _, _ , _), OP(2133, Ext12 , _, Ins0To1, _, _ , _), 2110 OP(2200, Trn1_4S, _, _ , _, _ , _), OP(2201, Rev64 , _, Ins3To2, _, _ , _), OP(2202, Trn1_4S, _, Ins2To0, _, _ , _), OP(2203, Ext12 , _, Ins3To2, _, _ , _), 2111 OP(2210, Mov , _, Ins2To3, _, _ , _), OP(2211, Ext4 , A, Zip1_4S, _, _ , _), OP(2212, Mov , _, Ins2To0, _, Ins2To3, _), OP(2213, Mov , _, Ins3To0, _, Ins2To3, _), 2112 OP(2220, Uzp1_4S, _, Ins1To2, _, _ , _), OP(2221, Rev64 , _, Ins3To1, _, Ins3To2, _), OP(2222, Dup2 , _, _ , _, _ , _), OP(2223, Dup2 , 2, Ext12 , _, _ , _), 2113 OP(2230, Mov , _, Ins3To1, _, Ins2To3, _), OP(2231, Dup2 , 2, Uzp2_4S, _, _ , _), OP(2232, Zip2_2D, _, Ins0To3, _, _ , _), OP(2233, Rev64 , A, Zip2_4S, _, _ , _), 2114 OP(2300, Rev64 , _, Ins1To0, _, _ , _), OP(2301, Rev64 , _, _ , _, _ , _), OP(2302, Rev64 , _, Ins3To0, _, _ , _), OP(2303, Rev64 , _, Ins2To0, _, _ , _), 2115 OP(2310, Dup2 , 2, Ext12 , 2, Zip1_2D, _), OP(2311, Rev64 , _, Ins0To1, _, _ , _), OP(2312, Ext12 , 2, Zip2_4S, _, _ , _), OP(2313, Rev64 , _, Ins0To1, _, Ins2To0, _), 2116 OP(2320, Rev64 , _, Ins1To0, _, Ins3To1, _), OP(2321, Rev64 , _, Ins3To1, _, _ , _), OP(2322, Zip2_4S, _, Ins0To3, _, _ , _), OP(2323, Rev64 , A, Zip2_2D, _, _ , _), 2117 OP(2330, Rev64 , _, Ins1To0, _, Ins2To1, _), OP(2331, Rev64 , _, Ins2To1, _, _ , _), OP(2332, Rev64 , 2, Zip2_4S, _, _ , _), OP(2333, Rev64 , _, Ins2To0, _, Ins2To1, _), 2118 OP(3000, Mov , _, Ins0To1, _, Ins0To2, _), OP(3001, Rev64 , _, Ins2To3, _, Ins1To2, _), OP(3002, Ext8 , _, Ins1To3, _, Ins2To1, _), OP(3003, Ext12 , _, Ins0To3, _, Ins1To2, _), 2119 OP(3010, Mov , _, Ins0To2, _, _ , _), OP(3011, Mov , _, Ins0To2, _, Ins1To0, _), OP(3012, Rev64 , A, Ext12 , _, _ , _), OP(3013, Rev64 , 2, Ext12 , _, _ , _), 2120 OP(3020, Dup0 , 1, Zip2_4S, _, _ , _), OP(3021, Dup1 , 1, Ext4 , 1, Zip2_4S, _), OP(3022, Ext8 , _, Ins1To3, _, Ins0To1, _), OP(3023, Ext4 , 1, Zip2_4S, _, _ , _), 2121 OP(3030, Mov , _, Ins0To2, _, Ins3To1, _), OP(3031, Ext8 , _, Ins3To0, _, Ins1To3, _), OP(3032, Ext8 , _, Ins1To3, _, _ , _), OP(3033, Ext8 , _, Ins1To0, _, Ins1To3, _), 2122 OP(3100, Dup0 , 1, Uzp2_4S, _, _ , _), OP(3101, Rev64 , _, Ins2To3, _, Ins0To2, _), OP(3102, Ext4 , 1, Uzp2_4S, _, _ , _), OP(3103, Ext12 , _, Ins0To3, _, _ , _), 2123 OP(3110, Mov , _, Ins1To2, _, _ , _), OP(3111, Uzp2_4S, _, Ins0To1, _, _ , _), OP(3112, Mov , _, Ins2To0, _, Ins1To2, _), OP(3113, Ext8 , 1, Uzp2_4S, _, _ , _), 2124 OP(3120, Rev64 , 2, Uzp1_4S, _, _ , _), OP(3121, Dup1 , 1, Zip2_4S, _, _ , _), OP(3122, Dup2 , 1, Uzp2_4S, _, _ , _), OP(3123, Ext12 , _, Ins3To1, _, Ins0To3, _), 2125 OP(3130, Dup3 , 2, Zip1_4S, _, _ , _), OP(3131, Uzp2_4S, _, _ , _, _ , _), OP(3132, Zip2_4S, 1, Uzp2_4S, _, _ , _), OP(3133, Uzp2_4S, _, Ins1To0, _, _ , _), 2126 OP(3200, Mov , _, Ins0To1, _, _ , _), OP(3201, Dup1 , 1, Ext4 , 1, Zip2_2D, _), OP(3202, Mov , _, Ins0To1, _, Ins2To0, _), OP(3203, Ext4 , 1, Zip2_2D, _, _ , _), 2127 OP(3210, Mov , _, _ , _, _ , _), OP(3211, Mov , _, Ins1To0, _, _ , _), OP(3212, Mov , _, Ins2To0, _, _ , _), OP(3213, Mov , _, Ins3To0, _, _ , _), 2128 OP(3220, Mov , _, Ins2To1, _, _ , _), OP(3221, Ext12 , 1, Zip2_4S, _, _ , _), OP(3222, Zip2_4S, _, Ins0To2, _, _ , _), OP(3223, Rev64 , 1, Zip2_4S, _, _ , _), 2129 OP(3230, Mov , _, Ins3To1, _, _ , _), OP(3231, Zip2_4S, 2, Uzp2_4S, _, _ , _), OP(3232, Zip2_2D, _, _ , _, _ , _), OP(3233, Zip2_2D, _, Ins1To0, _, _ , _), 2130 OP(3300, Mov , _, Ins0To1, _, Ins3To2, _), OP(3301, Rev64 , _, Ins2To3, _, _ , _), OP(3302, Rev64 , _, Ins3To0, _, Ins2To3, _), OP(3303, Rev64 , _, Ins2To0, _, Ins2To3, _), 2131 OP(3310, Mov , _, Ins3To2, _, _ , _), OP(3311, Trn2_4S, _, _ , _, _ , _), OP(3312, Zip2_4S, 1, Trn2_4S, _, _ , _), OP(3313, Trn2_4S, _, Ins2To0, _, _ , _), 2132 OP(3320, Dup3 , 2, Uzp1_4S, _, _ , _), OP(3321, Ext4 , _, Ins2To3, _, _ , _), OP(3322, Zip2_4S, _, _ , _, _ , _), OP(3323, Zip2_4S, _, Ins2To0, _, _ , _), 2133 OP(3330, Mov , _, Ins3To1, _, Ins3To2, _), OP(3331, Uzp2_4S, _, Ins1To2, _, _ , _), OP(3332, Zip2_4S, _, Ins2To1, _, _ , _), OP(3333, Dup3 , _, _ , _, _ , _) 2134 }; 2135 2136 #undef OP 2137 2138 static void emit_swizzle32_impl(UniCompiler& uc, const Vec& dst, const Vec& src, uint32_t imm) { 2139 ASMJIT_ASSERT((imm & 0xFCFCFCFC) == 0); 2140 2141 BackendCompiler* cc = uc.cc; 2142 2143 uint32_t table_index = ((imm & 0x03000000) >> (24 - 6)) | 2144 ((imm & 0x00030000) >> (16 - 4)) | 2145 ((imm & 0x00000300) >> (8 - 2)) | (imm & 0x00000003); 2146 Swizzle32Data swiz = swizzle_32_data[table_index]; 2147 2148 if (swiz.is_defined()) { 2149 Vec op_src[2] = { src, src }; 2150 Vec op_dst; 2151 2152 for (uint32_t i = 0; i < 3; i++) { 2153 Swizzle32Data::Op op = swiz.op(i); 2154 Swizzle32Data::OpTarget target = swiz.op_target(i); 2155 2156 if (!op.is_valid()) 2157 break; 2158 2159 if (target == Swizzle32Data::OpTarget::kDst) { 2160 op_dst = dst; 2161 } 2162 else { 2163 op_dst = uc.new_similar_reg(dst, "@tmp"); 2164 } 2165 2166 switch (op.type()) { 2167 case Swizzle32Data::Op::kMov: { 2168 vec_mov(uc, op_dst, op_src[0]); 2169 break; 2170 } 2171 2172 case Swizzle32Data::Op::kIns: { 2173 uint32_t src_lane = op.ins_src(); 2174 uint32_t dst_lane = op.ins_dst(); 2175 // Insert is always the last operation that only uses the destination register. 2176 cc->mov(op_dst.s(dst_lane), op_dst.s(src_lane)); 2177 break; 2178 } 2179 2180 case Swizzle32Data::Op::kDup: { 2181 // Use `dup` if the swizzle is actually a broadcast of a single element. 2182 uint32_t idx = op.dup_idx(); 2183 cc->dup(op_dst.s4(), op_src[0].s(idx)); 2184 break; 2185 } 2186 2187 case Swizzle32Data::Op::kExt: { 2188 // Use `ext` if the swizzle is rotating the vector. 2189 uint32_t n = op.ext_imm(); 2190 cc->ext(op_dst.b16(), op_src[0].b16(), op_src[1].b16(), n); 2191 break; 2192 } 2193 2194 case Swizzle32Data::Op::kRev64: { 2195 // Use `rev64` to swap lo/hi elements of 64-bit lanes. 2196 cc->rev64(op_dst.s4(), op_src[0].s4()); 2197 break; 2198 } 2199 2200 case Swizzle32Data::Op::kZipUnzip: { 2201 // Use `zip[1|2]`, 'uzp[1|2]', or 'trn[1|2]` if the swizzle can be implemented this way. 2202 static constexpr uint16_t zip_unzip_inst[8] = { 2203 Inst::kIdZip1_v, 2204 Inst::kIdZip2_v, 2205 Inst::kIdUzp1_v, 2206 Inst::kIdUzp2_v, 2207 Inst::kIdTrn1_v, 2208 Inst::kIdTrn2_v 2209 }; 2210 2211 InstId inst_id = zip_unzip_inst[op.zip_op()]; 2212 if (op.zip_s4()) 2213 cc->emit(inst_id, op_dst.s4(), op_src[0].s4(), op_src[1].s4()); 2214 else 2215 cc->emit(inst_id, op_dst.d2(), op_src[0].d2(), op_src[1].d2()); 2216 break; 2217 } 2218 2219 default: 2220 ASMJIT_NOT_REACHED(); 2221 } 2222 2223 if (uint32_t(target) & uint32_t(Swizzle32Data::OpTarget::k1)) op_src[0] = op_dst; 2224 if (uint32_t(target) & uint32_t(Swizzle32Data::OpTarget::k2)) op_src[1] = op_dst; 2225 } 2226 } 2227 else { 2228 // NOTE: This code is never used at the moment. It's kept if for some reason we would want to avoid using 2229 // more than 1 or 2 instructions to perform the swizzle. For example on hardware where TBL is faster than 2230 // other operations combined. 2231 uint8_t pred_data[16] = { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF }; 2232 2233 uint32_t d = (imm >> 22) & (0x3 << 2); 2234 uint32_t c = (imm >> 14) & (0x3 << 2); 2235 uint32_t b = (imm >> 6) & (0x3 << 2); 2236 uint32_t a = (imm << 2) & (0x3 << 2); 2237 2238 pred_data[ 0] = uint8_t(a); 2239 pred_data[ 1] = uint8_t(a + 1u); 2240 pred_data[ 2] = uint8_t(a + 2u); 2241 pred_data[ 3] = uint8_t(a + 3u); 2242 pred_data[ 4] = uint8_t(b); 2243 pred_data[ 5] = uint8_t(b + 1u); 2244 pred_data[ 6] = uint8_t(b + 2u); 2245 pred_data[ 7] = uint8_t(b + 3u); 2246 pred_data[ 8] = uint8_t(c); 2247 pred_data[ 9] = uint8_t(c + 1u); 2248 pred_data[10] = uint8_t(c + 2u); 2249 pred_data[11] = uint8_t(c + 3u); 2250 pred_data[12] = uint8_t(d); 2251 pred_data[13] = uint8_t(d + 1u); 2252 pred_data[14] = uint8_t(d + 2u); 2253 pred_data[15] = uint8_t(d + 3u); 2254 2255 Vec pred = uc.simd_const_16b(pred_data); 2256 cc->tbl(dst.b16(), src.b16(), pred.b16()); 2257 } 2258 } 2259 2260 // ujit::UniCompiler - Vector Instructions - Interleaved Shuffle 32 Impl 2261 // ===================================================================== 2262 2263 struct InterleavedShuffle32Ops { 2264 struct Op { 2265 //! Operation swaps lo/hi elements of 64-bit lanes - 'rev64'. 2266 static constexpr uint32_t kOpRev64 = 0; 2267 //! Operation performs a 32-bit insert - moves a value from a source lane to a destination lane. 2268 static constexpr uint32_t kOpInsS = 1; 2269 //! Operation performs a 64-bit insert - moves a value from a source lane to a destination lane. 2270 static constexpr uint32_t kOpInsD = 2; 2271 //! Operation can be implemented as a single zip[1|2], uzp[1|2], trn[1|2], or ext instruction with 32-bit or 64-bit elements. 2272 static constexpr uint32_t kOpPerm = 3; 2273 //! Operation duplicates a lane across all others - 'dup'. 2274 static constexpr uint32_t kOpDup = 4; 2275 2276 static constexpr uint32_t kInsS_0To0 = (kOpInsS << 8) | (0 << 0) | (0 << 2); 2277 static constexpr uint32_t kInsS_0To1 = (kOpInsS << 8) | (0 << 0) | (1 << 2); 2278 static constexpr uint32_t kInsS_0To2 = (kOpInsS << 8) | (0 << 0) | (2 << 2); 2279 static constexpr uint32_t kInsS_0To3 = (kOpInsS << 8) | (0 << 0) | (3 << 2); 2280 static constexpr uint32_t kInsS_1To0 = (kOpInsS << 8) | (1 << 0) | (0 << 2); 2281 static constexpr uint32_t kInsS_1To1 = (kOpInsS << 8) | (1 << 0) | (1 << 2); 2282 static constexpr uint32_t kInsS_1To2 = (kOpInsS << 8) | (1 << 0) | (2 << 2); 2283 static constexpr uint32_t kInsS_1To3 = (kOpInsS << 8) | (1 << 0) | (3 << 2); 2284 static constexpr uint32_t kInsS_2To0 = (kOpInsS << 8) | (2 << 0) | (0 << 2); 2285 static constexpr uint32_t kInsS_2To1 = (kOpInsS << 8) | (2 << 0) | (1 << 2); 2286 static constexpr uint32_t kInsS_2To2 = (kOpInsS << 8) | (2 << 0) | (2 << 2); 2287 static constexpr uint32_t kInsS_2To3 = (kOpInsS << 8) | (2 << 0) | (3 << 2); 2288 static constexpr uint32_t kInsS_3To0 = (kOpInsS << 8) | (3 << 0) | (0 << 2); 2289 static constexpr uint32_t kInsS_3To1 = (kOpInsS << 8) | (3 << 0) | (1 << 2); 2290 static constexpr uint32_t kInsS_3To2 = (kOpInsS << 8) | (3 << 0) | (2 << 2); 2291 static constexpr uint32_t kInsS_3To3 = (kOpInsS << 8) | (3 << 0) | (3 << 2); 2292 static constexpr uint32_t kInsD_0To0 = (kOpInsD << 8) | (0 << 0) | (0 << 2); 2293 static constexpr uint32_t kInsD_0To1 = (kOpInsD << 8) | (0 << 0) | (1 << 2); 2294 static constexpr uint32_t kInsD_1To0 = (kOpInsD << 8) | (1 << 0) | (0 << 2); 2295 static constexpr uint32_t kInsD_1To1 = (kOpInsD << 8) | (1 << 0) | (1 << 2); 2296 2297 static constexpr uint32_t kZip1_4S = (kOpPerm << 8) | (0 << 0) | (0 << 3); 2298 static constexpr uint32_t kZip1_2D = (kOpPerm << 8) | (0 << 0) | (1 << 3); 2299 static constexpr uint32_t kZip2_4S = (kOpPerm << 8) | (1 << 0) | (0 << 3); 2300 static constexpr uint32_t kZip2_2D = (kOpPerm << 8) | (1 << 0) | (1 << 3); 2301 static constexpr uint32_t kUzp1_4S = (kOpPerm << 8) | (2 << 0) | (0 << 3); 2302 static constexpr uint32_t kUzp1_2D = (kOpPerm << 8) | (2 << 0) | (1 << 3); 2303 static constexpr uint32_t kUzp2_4S = (kOpPerm << 8) | (3 << 0) | (0 << 3); 2304 static constexpr uint32_t kUzp2_2D = (kOpPerm << 8) | (3 << 0) | (1 << 3); 2305 static constexpr uint32_t kTrn1_4S = (kOpPerm << 8) | (4 << 0); 2306 static constexpr uint32_t kTrn2_4S = (kOpPerm << 8) | (5 << 0); 2307 static constexpr uint32_t kExt4 = (kOpPerm << 8) | (7 << 0) | (0 << 3); 2308 static constexpr uint32_t kExt8 = (kOpPerm << 8) | (6 << 0) | (1 << 3); 2309 static constexpr uint32_t kExt12 = (kOpPerm << 8) | (7 << 0) | (1 << 3); 2310 2311 static constexpr uint32_t kRev64 = (kOpRev64 << 8) | (1 << 0); // Dummy bit so we don't end up having all zeros. 2312 2313 static constexpr uint32_t kDup0 = (kOpDup << 8) | (0 << 0); 2314 static constexpr uint32_t kDup1 = (kOpDup << 8) | (1 << 0); 2315 static constexpr uint32_t kDup2 = (kOpDup << 8) | (2 << 0); 2316 static constexpr uint32_t kDup3 = (kOpDup << 8) | (3 << 0); 2317 2318 static constexpr uint32_t kSrcAA = (0 << 4) | (0 << 6); 2319 static constexpr uint32_t kSrcAB = (0 << 4) | (1 << 6); 2320 static constexpr uint32_t kSrcAC = (0 << 4) | (2 << 6); 2321 static constexpr uint32_t kSrcAD = (0 << 4) | (3 << 6); 2322 static constexpr uint32_t kSrcBA = (1 << 4) | (0 << 6); 2323 static constexpr uint32_t kSrcBB = (1 << 4) | (1 << 6); 2324 static constexpr uint32_t kSrcBC = (1 << 4) | (2 << 6); 2325 static constexpr uint32_t kSrcBD = (1 << 4) | (3 << 6); 2326 static constexpr uint32_t kSrcCA = (2 << 4) | (0 << 6); 2327 static constexpr uint32_t kSrcCB = (2 << 4) | (1 << 6); 2328 static constexpr uint32_t kSrcCC = (2 << 4) | (2 << 6); 2329 static constexpr uint32_t kSrcCD = (2 << 4) | (3 << 6); 2330 static constexpr uint32_t kSrcDA = (3 << 4) | (0 << 6); 2331 static constexpr uint32_t kSrcDB = (3 << 4) | (1 << 6); 2332 static constexpr uint32_t kSrcDC = (3 << 4) | (2 << 6); 2333 static constexpr uint32_t kSrcDD = (3 << 4) | (3 << 6); 2334 2335 static constexpr uint32_t kSrcA = kSrcAA; 2336 static constexpr uint32_t kSrcB = kSrcBB; 2337 static constexpr uint32_t kSrcC = kSrcCC; 2338 static constexpr uint32_t kSrcD = kSrcDD; 2339 2340 // Alias to nothing to make the table easier to read. 2341 static constexpr uint32_t k_ = 0; 2342 static constexpr uint32_t kSrc_ = 0; 2343 2344 uint32_t data; 2345 2346 ASMJIT_INLINE_NODEBUG bool is_valid() const noexcept { return data != 0u; } 2347 ASMJIT_INLINE_NODEBUG uint32_t op() const noexcept { return data >> 8; } 2348 ASMJIT_INLINE_NODEBUG uint32_t is_ins_op() const noexcept { return op() == kOpInsS || op() == kOpInsD; } 2349 2350 ASMJIT_INLINE_NODEBUG uint32_t dup_idx() const noexcept { return data & 0x3u; } 2351 2352 ASMJIT_INLINE_NODEBUG uint32_t perm_op() const noexcept { return data & 0x7u; } 2353 ASMJIT_INLINE_NODEBUG bool perm_s4() const noexcept { return (data & (1u << 3)) == 0u; } 2354 ASMJIT_INLINE_NODEBUG uint32_t perm_ext_imm() const noexcept { return ((data & 0x1) << 2) + (data & 0x8); } 2355 2356 ASMJIT_INLINE_NODEBUG uint32_t ins_src() const noexcept { return (data >> 0) & 0x3u; } 2357 ASMJIT_INLINE_NODEBUG uint32_t ins_dst() const noexcept { return (data >> 2) & 0x3u; } 2358 2359 ASMJIT_INLINE_NODEBUG uint32_t src_a() const noexcept { return (data >> 4) & 0x3u; } 2360 ASMJIT_INLINE_NODEBUG uint32_t src_b() const noexcept { return (data >> 6) & 0x3u; } 2361 }; 2362 2363 //! \name Members 2364 //! \{ 2365 2366 uint32_t combined; 2367 2368 //! \} 2369 2370 //! \name Accessors 2371 //! \{ 2372 2373 ASMJIT_INLINE_NODEBUG uint32_t count() const noexcept { 2374 return 1u + uint32_t(((combined >> 11) & 0x7FFu) != 0u) + uint32_t((combined >> 22) != 0u); 2375 } 2376 2377 ASMJIT_INLINE_NODEBUG Op op(uint32_t index) const noexcept { 2378 return Op{(combined >> (index * 11u)) & 0x7FFu}; 2379 } 2380 2381 //! \} 2382 }; 2383 2384 // These tables provide all combinations for all possible 32-bit interleaved shuffles (there is 256 combinations 2385 // in total). It prioritizes lane moves, and then operations that can have either one or two inputs. Initially, 2386 // there are two sources (A, B), which can be used by any shuffle operation, which result is then referred as C. 2387 // Then, all consecutive shuffles can use any of A, B, and C as their operands. The last operation is the result. 2388 // 2389 // Statistics: 2390 // 2391 // - 1 Instruction: 5 2392 // - 2 Instructions: 113 2393 // - 3 Instructions: 138 2394 2395 #define OP(swiz, op0, src0, op1, src1, op2, src2) { \ 2396 ((InterleavedShuffle32Ops::Op::k##op0) << 0) | ((InterleavedShuffle32Ops::Op::kSrc##src0) << 0) | \ 2397 ((InterleavedShuffle32Ops::Op::k##op1) << 11) | ((InterleavedShuffle32Ops::Op::kSrc##src1) << 11) | \ 2398 ((InterleavedShuffle32Ops::Op::k##op2) << 22) | ((InterleavedShuffle32Ops::Op::kSrc##src2) << 22) \ 2399 } 2400 2401 static constexpr InterleavedShuffle32Ops interleaved_shuffle32_ops[256] = { 2402 OP(4400, Trn1_4S , AB, Zip1_4S , CC, _ , _ ), OP(4401, Rev64 , A , Dup0 , B , Zip1_2D , CD), OP(4402, Ext8 , AA, Dup0 , B , Uzp1_4S , CD), OP(4403, Ext12 , AA, Dup0 , B , Zip1_2D , CD), 2403 OP(4410, Dup0 , B , Zip1_2D , AC, _ , _ ), OP(4411, Dup1 , A , Dup0 , B , Uzp2_4S , CD), OP(4412, Ext4 , AB, InsS_3To2, C , Rev64 , C ), OP(4413, Ext12 , AA, Dup0 , B , Uzp1_4S , CD), 2404 OP(4420, Dup0 , B , Uzp1_4S , AC, _ , _ ), OP(4421, Ext4 , AB, InsS_3To2, C , _ , _ ), OP(4422, Ext8 , AB, Trn1_4S , CC, _ , _ ), OP(4423, Rev64 , A , Dup0 , B , Zip2_2D , CD), 2405 OP(4430, Trn1_4S , AB, Ext12 , AB, Zip1_4S , CD), OP(4431, Dup0 , B , Uzp2_4S , AC, _ , _ ), OP(4432, Dup0 , B , Zip2_2D , AC, _ , _ ), OP(4433, Ext12 , AB, Zip1_4S , CC, _ , _ ), 2406 OP(4500, Rev64 , B , Dup0 , A , Zip1_2D , DC), OP(4501, Zip1_2D , AB, Rev64 , C , _ , _ ), OP(4502, Uzp1_4S , BA, Ext8 , CB, Rev64 , D ), OP(4503, Ext12 , AA, Rev64 , B , Zip1_2D , CD), 2407 OP(4510, Rev64 , B , Zip1_2D , AC, _ , _ ), OP(4511, Rev64 , B , Dup1 , A , Zip1_2D , DC), OP(4512, Uzp2_4S , AB, Uzp1_4S , AC, Ext4 , DB), OP(4513, Trn2_4S , AB, Zip1_2D , CB, Ext12 , AD), 2408 OP(4520, Dup1 , B , Uzp1_4S , CB, Uzp1_4S , AD), OP(4521, Ext4 , AB, InsS_1To2, B , _ , _ ), OP(4522, Rev64 , B , Dup2 , A , Zip1_2D , DC), OP(4523, Ext8 , AB, Rev64 , C , _ , _ ), 2409 OP(4530, Ext12 , AA, Zip1_2D , CB, Rev64 , D ), OP(4531, Uzp2_4S , AB, InsS_0To3, B , _ , _ ), OP(4532, Rev64 , B , Ext8 , AC, _ , _ ), OP(4533, Rev64 , B , Dup3 , A , Zip1_2D , DC), 2410 OP(4600, Ext8 , BB, Dup0 , A , Uzp1_4S , DC), OP(4601, Uzp1_4S , BB, Zip1_2D , AC, Rev64 , D ), OP(4602, Uzp1_4S , AB, Rev64 , C , _ , _ ), OP(4603, Uzp1_4S , AB, InsS_3To1, C , Ext12 , AC), 2411 OP(4610, Dup2 , B , Trn1_4S , CB, Zip1_2D , AD), OP(4611, Ext8 , BB, Dup1 , A , Uzp1_4S , DC), OP(4612, Trn1_4S , AB, Zip1_4S , AC, Zip2_4S , CD), OP(4613, Rev64 , B , Uzp2_4S , AC, Rev64 , D ), 2412 OP(4620, Ext8 , BB, Uzp1_4S , AC, _ , _ ), OP(4621, Ext4 , AB, InsS_2To2, B , _ , _ ), OP(4622, Ext8 , BB, Dup2 , A , Uzp1_4S , DC), OP(4623, Trn1_4S , AB, Zip2_2D , AC, Ext4 , DB), 2413 OP(4630, Ext12 , AB, Zip1_4S , AC, InsS_3To2, C ), OP(4631, Ext4 , BB, Uzp2_4S , AC, _ , _ ), OP(4632, Zip2_2D , AB, InsS_0To3, B , _ , _ ), OP(4633, Ext8 , BB, Dup3 , A , Uzp1_4S , DC), 2414 OP(4700, Ext12 , BB, Dup0 , A , Zip1_2D , DC), OP(4701, Ext12 , BB, Rev64 , A , Zip1_2D , DC), OP(4702, Uzp1_4S , AB, InsS_3To3, B , Rev64 , C ), OP(4703, Ext12 , BB, Ext12 , AA, Zip1_2D , DC), 2415 OP(4710, Ext12 , BB, Zip1_2D , AC, _ , _ ), OP(4711, Ext12 , BB, Dup1 , A , Zip1_2D , DC), OP(4712, Trn2_4S , BA, InsS_2To0, A , InsS_0To3, B ), OP(4713, Uzp2_4S , AB, InsS_0To2, C , Ext4 , CB), 2416 OP(4720, Dup3 , B , Uzp1_4S , CB, Uzp1_4S , AD), OP(4721, Ext4 , AB, InsS_3To2, B , _ , _ ), OP(4722, Ext12 , BB, Dup2 , A , Zip1_2D , DC), OP(4723, Ext12 , BB, Rev64 , A , Ext8 , DC), 2417 OP(4730, Rev64 , A , InsS_3To3, B , Ext4 , CB), OP(4731, Dup3 , B , Ext4 , CB, Uzp2_4S , AD), OP(4732, Ext12 , BB, Ext8 , AC, _ , _ ), OP(4733, Ext12 , BB, Dup3 , A , Zip1_2D , DC), 2418 OP(5400, Dup0 , A , Zip1_2D , CB, _ , _ ), OP(5401, Rev64 , A , Zip1_2D , CB, _ , _ ), OP(5402, Ext8 , AB, InsS_0To1, A , _ , _ ), OP(5403, Ext12 , AA, Zip1_2D , CB, _ , _ ), 2419 OP(5410, Zip1_2D , AB, _ , _ , _ , _ ), OP(5411, Dup1 , A , Zip1_2D , CB, _ , _ ), OP(5412, Zip1_2D , AB, InsS_2To0, A , _ , _ ), OP(5413, Zip1_2D , AB, InsS_3To0, A , _ , _ ), 2420 OP(5420, Uzp1_4S , BA, Ext8 , CB, _ , _ ), OP(5421, Ext12 , BA, Ext8 , CB, _ , _ ), OP(5422, Dup2 , A , Zip1_2D , CB, _ , _ ), OP(5423, Rev64 , A , Ext8 , CB, _ , _ ), 2421 OP(5430, Zip1_2D , AB, InsS_3To1, A , _ , _ ), OP(5431, Uzp2_4S , BA, Ext8 , CB, _ , _ ), OP(5432, Ext8 , AB, _ , _ , _ , _ ), OP(5433, Dup3 , A , Zip1_2D , CB, _ , _ ), 2422 OP(5500, Dup1 , B , Dup0 , A , Uzp2_4S , DC), OP(5501, Rev64 , A , Dup1 , B , Zip1_2D , CD), OP(5502, Ext8 , AA, Dup1 , B , Uzp1_4S , CD), OP(5503, Ext12 , AA, Dup1 , B , Zip1_2D , CD), 2423 OP(5510, Dup1 , B , Zip1_2D , AC, _ , _ ), OP(5511, Trn2_4S , AB, Zip1_4S , CC, _ , _ ), OP(5512, Uzp2_4S , AB, Zip2_2D , AC, Trn1_4S , DC), OP(5513, Ext12 , AA, Dup1 , B , Uzp1_4S , CD), 2424 OP(5520, Dup1 , B , Uzp1_4S , AC, _ , _ ), OP(5521, Ext12 , BA, Dup1 , B , Zip2_2D , CD), OP(5522, Dup2 , A , Dup1 , B , Uzp2_4S , CD), OP(5523, Rev64 , A , Dup1 , B , Zip2_2D , CD), 2425 OP(5530, Ext12 , AB, Trn1_4S , AC, InsS_3To2, D ), OP(5531, Dup1 , B , Uzp2_4S , AC, _ , _ ), OP(5532, Dup1 , B , Zip2_2D , AC, _ , _ ), OP(5533, Ext12 , AB, Trn1_4S , CC, _ , _ ), 2426 OP(5600, Trn1_4S , BA, InsS_1To3, B , InsS_1To0, C ), OP(5601, Ext4 , BB, Zip1_2D , AC, Rev64 , D ), OP(5602, Uzp1_4S , AB, InsS_1To2, B , Rev64 , C ), OP(5603, Ext12 , AA, Zip1_2D , CB, InsS_2To2, B ), 2427 OP(5610, Zip1_2D , AB, InsS_2To2, B , _ , _ ), OP(5611, Zip1_4S , AB, Trn1_4S , CB, Zip2_4S , DC), OP(5612, Trn2_4S , AB, Zip2_4S , AB, Zip1_4S , DC), OP(5613, Trn2_4S , AB, Zip2_4S , CB, Zip1_4S , DC), 2428 OP(5620, Dup2 , B , Uzp2_4S , CB, Uzp1_4S , AD), OP(5621, Rev64 , B , Ext4 , AC, InsS_3To2, C ), OP(5622, Uzp1_4S , AB, Zip1_2D , CB, Trn2_4S , CD), OP(5623, Uzp1_4S , AB, Uzp2_4S , CB, Ext12 , AD), 2429 OP(5630, Ext12 , AB, Trn1_4S , AC, InsS_3To2, C ), OP(5631, Dup2 , B , Zip1_2D , CB, Uzp2_4S , AD), OP(5632, Zip2_2D , AB, InsS_1To3, B , _ , _ ), OP(5633, Ext12 , AB, InsS_3To1, C , Ext12 , AC), 2430 OP(5700, Ext12 , BB, Dup0 , A , Uzp1_4S , DC), OP(5701, Uzp2_4S , BB, Zip1_2D , AC, Rev64 , D ), OP(5702, Rev64 , B , Uzp1_4S , AC, Rev64 , D ), OP(5703, Uzp2_4S , BB, InsS_0To0, A , Ext12 , AC), 2431 OP(5710, Zip1_2D , AB, InsS_3To2, B , _ , _ ), OP(5711, Ext12 , BB, Dup1 , A , Uzp1_4S , DC), OP(5712, Uzp2_4S , AB, InsS_2To1, A , Rev64 , C ), OP(5713, Uzp2_4S , AB, Rev64 , C , _ , _ ), 2432 OP(5720, Ext12 , BB, Uzp1_4S , AC, _ , _ ), OP(5721, Rev64 , B , Ext4 , AC, InsS_2To2, C ), OP(5722, Ext12 , BB, Dup2 , A , Uzp1_4S , DC), OP(5723, Uzp2_4S , BB, Zip2_2D , AC, Rev64 , D ), 2433 OP(5730, Ext12 , AB, Trn1_4S , AC, InsS_3To2, B ), OP(5731, Ext8 , BB, Uzp2_4S , AC, _ , _ ), OP(5732, Ext8 , AB, InsS_3To2, B , _ , _ ), OP(5733, Ext12 , BB, Dup3 , A , Uzp1_4S , DC), 2434 OP(6400, Dup0 , A , Uzp1_4S , CB, _ , _ ), OP(6401, Dup1 , A , Uzp1_4S , CA, Uzp1_4S , DB), OP(6402, Ext8 , AA, Uzp1_4S , CB, _ , _ ), OP(6403, Dup3 , A , Uzp1_4S , CA, Uzp1_4S , DB), 2435 OP(6410, Uzp1_4S , BB, Zip1_2D , AC, _ , _ ), OP(6411, Dup1 , A , Uzp1_4S , CB, _ , _ ), OP(6412, Dup2 , A , Uzp2_4S , CA, Uzp1_4S , DB), OP(6413, Ext12 , AA, Uzp1_4S , CB, _ , _ ), 2436 OP(6420, Uzp1_4S , AB, _ , _ , _ , _ ), OP(6421, Uzp1_4S , AB, InsS_1To0, A , _ , _ ), OP(6422, Dup2 , A , Uzp1_4S , CB, _ , _ ), OP(6423, Uzp1_4S , AB, InsS_3To0, A , _ , _ ), 2437 OP(6430, Uzp1_4S , AB, InsS_3To1, A , _ , _ ), OP(6431, Rev64 , B , Uzp2_4S , AC, _ , _ ), OP(6432, Uzp1_4S , BB, Zip2_2D , AC, _ , _ ), OP(6433, Dup3 , A , Uzp1_4S , CB, _ , _ ), 2438 OP(6500, Ext12 , BB, Dup0 , A , Zip2_2D , DC), OP(6501, Ext4 , BB, Rev64 , A , Zip1_2D , DC), OP(6502, Rev64 , A , Ext12 , CB, InsS_1To1, C ), OP(6503, Ext12 , AB, InsS_0To1, A , _ , _ ), 2439 OP(6510, Ext4 , BB, Zip1_2D , AC, _ , _ ), OP(6511, Ext12 , BB, Dup1 , A , Zip2_2D , DC), OP(6512, Rev64 , A , Ext12 , CB, InsS_0To1, C ), OP(6513, Ext12 , AB, InsS_1To1, A , _ , _ ), 2440 OP(6520, Uzp1_4S , AB, InsS_1To2, B , _ , _ ), OP(6521, Trn2_4S , AB, Zip2_4S , AB, Zip1_4S , CD), OP(6522, Ext12 , BB, Dup2 , A , Zip2_2D , DC), OP(6523, Ext12 , AB, InsS_2To1, A , _ , _ ), 2441 OP(6530, Ext4 , BA, Trn2_4S , CA, Ext8 , DC), OP(6531, Uzp2_4S , AB, InsS_2To3, B , _ , _ ), OP(6532, Ext12 , BB, Zip2_2D , AC, _ , _ ), OP(6533, Ext12 , AB, InsS_0To1, C , _ , _ ), 2442 OP(6600, Dup2 , B , Dup0 , A , Uzp2_4S , DC), OP(6601, Rev64 , A , Dup2 , B , Zip1_2D , CD), OP(6602, Ext8 , AA, Dup2 , B , Uzp1_4S , CD), OP(6603, Ext12 , AA, Dup2 , B , Zip1_2D , CD), 2443 OP(6610, Dup2 , B , Zip1_2D , AC, _ , _ ), OP(6611, Dup2 , B , Dup1 , A , Uzp2_4S , DC), OP(6612, Zip2_4S , AB, Trn2_4S , AC, Zip1_4S , CD), OP(6613, Ext12 , AA, Dup2 , B , Uzp1_4S , CD), 2444 OP(6620, Dup2 , B , Uzp1_4S , AC, _ , _ ), OP(6621, Ext12 , BA, Dup2 , B , Zip2_2D , CD), OP(6622, Trn1_4S , AB, Zip2_4S , CC, _ , _ ), OP(6623, Rev64 , A , Dup2 , B , Zip2_2D , CD), 2445 OP(6630, Trn1_4S , AB, InsS_3To1, A , InsS_3To2, C ), OP(6631, Dup2 , B , Uzp2_4S , AC, _ , _ ), OP(6632, Dup2 , B , Zip2_2D , AC, _ , _ ), OP(6633, Dup3 , A , Dup2 , B , Uzp2_4S , CD), 2446 OP(6700, Rev64 , B , Dup0 , A , Zip2_2D , DC), OP(6701, Ext8 , BA, Rev64 , C , Ext8 , DD), OP(6702, Uzp1_4S , BA, Zip2_2D , CB, Rev64 , D ), OP(6703, Ext4 , AA, Rev64 , B , Zip2_2D , CD), 2447 OP(6710, Rev64 , B , InsD_0To0, A , _ , _ ), OP(6711, Rev64 , B , Dup1 , A , Zip2_2D , DC), OP(6712, Ext12 , BA, Zip2_2D , CB, Rev64 , D ), OP(6713, Uzp2_4S , BA, Zip2_2D , CB, Rev64 , D ), 2448 OP(6720, Uzp1_4S , AB, InsS_3To2, B , _ , _ ), OP(6721, Ext12 , BA, Rev64 , B , Zip2_2D , CD), OP(6722, Rev64 , B , Dup2 , A , Zip2_2D , DC), OP(6723, Zip2_2D , AB, Rev64 , C , _ , _ ), 2449 OP(6730, Ext4 , AA, Zip2_2D , CB, Rev64 , D ), OP(6731, Dup3 , B , Uzp1_4S , CB, Uzp2_4S , AD), OP(6732, Rev64 , B , Zip2_2D , AC, _ , _ ), OP(6733, Rev64 , B , Dup3 , A , Zip2_2D , DC), 2450 OP(7400, Uzp1_4S , AB, InsS_3To3, B , InsS_0To1, C ), OP(7401, Ext12 , BB, Zip1_2D , AC, Rev64 , D ), OP(7402, Ext8 , AB, Trn1_4S , CA, InsS_3To3, B ), OP(7403, Rev64 , B , InsS_0To0, A , Ext12 , AC), 2451 OP(7410, Zip1_2D , AB, InsS_3To3, B , _ , _ ), OP(7411, Trn2_4S , AB, InsS_0To2, B , InsS_0To1, C ), OP(7412, Ext4 , AB, Trn2_4S , CB, InsS_0To1, C ), OP(7413, Ext4 , AB, Zip2_4S , CB, InsS_0To1, C ), 2452 OP(7420, Uzp1_4S , AB, InsS_3To3, B , _ , _ ), OP(7421, Ext12 , BA, Trn1_4S , BC, Ext8 , CD), OP(7422, Ext4 , AB, Trn2_4S , CB, InsS_0To1, D ), OP(7423, Ext12 , BB, Ext8 , AC, Rev64 , D ), 2453 OP(7430, Uzp2_4S , AB, InsS_0To0, A , InsS_0To2, B ), OP(7431, Uzp2_4S , AB, InsS_0To2, B , _ , _ ), OP(7432, Zip2_2D , AB, InsS_0To2, B , _ , _ ), OP(7433, Trn2_4S , AB, Ext4 , AB, Zip2_4S , DC), 2454 OP(7500, Dup0 , A , Uzp2_4S , CB, _ , _ ), OP(7501, Uzp2_4S , AB, InsS_0To1, A , _ , _ ), OP(7502, Ext4 , AA, Uzp2_4S , CB, _ , _ ), OP(7503, Dup3 , A , Ext4 , CA, Uzp2_4S , DB), 2455 OP(7510, Uzp2_4S , BB, Zip1_2D , AC, _ , _ ), OP(7511, Dup1 , A , Uzp2_4S , CB, _ , _ ), OP(7512, Dup2 , A , Zip1_2D , CA, Uzp2_4S , DB), OP(7513, Ext8 , AA, Uzp2_4S , CB, _ , _ ), 2456 OP(7520, Rev64 , B , Uzp1_4S , AC, _ , _ ), OP(7521, Uzp2_4S , AB, InsS_2To1, A , _ , _ ), OP(7522, Dup2 , A , Uzp2_4S , CB, _ , _ ), OP(7523, Dup3 , A , Uzp1_4S , CA, Uzp2_4S , DB), 2457 OP(7530, Uzp2_4S , AB, InsS_0To0, A , _ , _ ), OP(7531, Uzp2_4S , AB, _ , _ , _ , _ ), OP(7532, Uzp2_4S , BB, Zip2_2D , AC, _ , _ ), OP(7533, Dup3 , A , Uzp2_4S , CB, _ , _ ), 2458 OP(7600, Dup0 , A , Zip2_2D , CB, _ , _ ), OP(7601, Rev64 , A , InsD_1To1, B , _ , _ ), OP(7602, Zip2_2D , AB, InsS_0To1, A , _ , _ ), OP(7603, Ext4 , AA, Zip2_2D , CB, _ , _ ), 2459 OP(7610, Zip2_2D , BB, Zip1_2D , AC, _ , _ ), OP(7611, Dup1 , A , Zip2_2D , CB, _ , _ ), OP(7612, Zip2_2D , AB, InsS_1To1, A , _ , _ ), OP(7613, Dup3 , A , Zip1_4S , CA, Zip2_2D , DB), 2460 OP(7620, Uzp1_4S , BA, Zip2_2D , CB, _ , _ ), OP(7621, Ext12 , BA, Zip2_2D , CB, _ , _ ), OP(7622, Dup2 , A , Zip2_2D , CB, _ , _ ), OP(7623, Rev64 , A , Zip2_2D , CB, _ , _ ), 2461 OP(7630, Zip2_2D , AB, InsS_0To0, A , _ , _ ), OP(7631, Uzp2_4S , BA, Zip2_2D , CB, _ , _ ), OP(7632, Zip2_2D , AB, _ , _ , _ , _ ), OP(7633, Dup3 , A , Zip2_2D , CB, _ , _ ), 2462 OP(7700, Dup3 , B , Dup0 , A , Uzp2_4S , DC), OP(7701, Rev64 , A , Dup3 , B , Zip1_2D , CD), OP(7702, Ext8 , AA, Dup3 , B , Uzp1_4S , CD), OP(7703, Ext12 , AA, Dup3 , B , Zip1_2D , CD), 2463 OP(7710, Dup3 , B , Zip1_2D , AC, _ , _ ), OP(7711, Dup3 , B , Dup1 , A , Uzp2_4S , DC), OP(7712, Trn2_4S , BA, InsS_2To0, A , InsS_2To3, C ), OP(7713, Ext12 , AA, Dup3 , B , Uzp1_4S , CD), 2464 OP(7720, Dup3 , B , Uzp1_4S , AC, _ , _ ), OP(7721, Ext12 , BA, Dup3 , B , Zip2_2D , CD), OP(7722, Dup3 , B , Dup2 , A , Uzp2_4S , DC), OP(7723, Rev64 , A , Dup3 , B , Zip2_2D , CD), 2465 OP(7730, Uzp2_4S , AB, InsS_0To0, A , InsS_3To2, C ), OP(7731, Dup3 , B , Uzp2_4S , AC, _ , _ ), OP(7732, Dup3 , B , Zip2_2D , AC, _ , _ ), OP(7733, Trn2_4S , AB, Zip2_4S , CC, _ , _ ) 2466 }; 2467 2468 static constexpr InterleavedShuffle32Ops interleaved_shuffle32_ops_dst_same_as_b[256] = { 2469 OP(4400, Trn1_4S , AB, Zip1_4S , CC, _ , _ ), OP(4401, Rev64 , A , Dup0 , B , Zip1_2D , CD), OP(4402, Ext8 , AA, Dup0 , B , Uzp1_4S , CD), OP(4403, Ext12 , AA, Dup0 , B , Zip1_2D , CD), 2470 OP(4410, Dup0 , B , Zip1_2D , AC, _ , _ ), OP(4411, Dup1 , A , Dup0 , B , Uzp2_4S , CD), OP(4412, Ext4 , AB, InsS_3To2, C , Rev64 , C ), OP(4413, Ext12 , AA, Dup0 , B , Uzp1_4S , CD), 2471 OP(4420, Dup0 , B , Uzp1_4S , AC, _ , _ ), OP(4421, Ext4 , AB, InsS_3To2, C , _ , _ ), OP(4422, Ext8 , AB, Trn1_4S , CC, _ , _ ), OP(4423, Rev64 , A , Dup0 , B , Zip2_2D , CD), 2472 OP(4430, Trn1_4S , AB, Ext12 , AB, Zip1_4S , CD), OP(4431, Dup0 , B , Uzp2_4S , AC, _ , _ ), OP(4432, Dup0 , B , Zip2_2D , AC, _ , _ ), OP(4433, Ext12 , AB, Zip1_4S , CC, _ , _ ), 2473 OP(4500, Rev64 , B , Dup0 , A , Zip1_2D , DC), OP(4501, Zip1_2D , AB, Rev64 , C , _ , _ ), OP(4502, Uzp1_4S , BA, Ext8 , CB, Rev64 , D ), OP(4503, Ext12 , AA, Rev64 , B , Zip1_2D , CD), 2474 OP(4510, Rev64 , B , Zip1_2D , AC, _ , _ ), OP(4511, Rev64 , B , Dup1 , A , Zip1_2D , DC), OP(4512, Uzp2_4S , AB, Uzp1_4S , AC, Ext4 , DB), OP(4513, Trn2_4S , AB, Zip1_2D , CB, Ext12 , AD), 2475 OP(4520, Dup1 , B , Uzp1_4S , CB, Uzp1_4S , AD), OP(4521, Ext4 , AB, InsS_1To2, B , _ , _ ), OP(4522, Rev64 , B , Dup2 , A , Zip1_2D , DC), OP(4523, Ext8 , AB, Rev64 , C , _ , _ ), 2476 OP(4530, Ext12 , AA, Zip1_2D , CB, Rev64 , D ), OP(4531, Uzp2_4S , AB, InsS_0To3, B , _ , _ ), OP(4532, Rev64 , B , Ext8 , AC, _ , _ ), OP(4533, Rev64 , B , Dup3 , A , Zip1_2D , DC), 2477 OP(4600, Ext8 , BB, Dup0 , A , Uzp1_4S , DC), OP(4601, Uzp1_4S , BB, Zip1_2D , AC, Rev64 , D ), OP(4602, Uzp1_4S , AB, Rev64 , C , _ , _ ), OP(4603, Uzp1_4S , AB, InsS_3To1, C , Ext12 , AC), 2478 OP(4610, Dup2 , B , Trn1_4S , CB, Zip1_2D , AD), OP(4611, Ext8 , BB, Dup1 , A , Uzp1_4S , DC), OP(4612, Trn1_4S , AB, Zip1_4S , AC, Zip2_4S , CD), OP(4613, Rev64 , B , Uzp2_4S , AC, Rev64 , D ), 2479 OP(4620, Ext8 , BB, Uzp1_4S , AC, _ , _ ), OP(4621, Ext4 , AB, InsS_2To2, B , _ , _ ), OP(4622, Ext8 , BB, Dup2 , A , Uzp1_4S , DC), OP(4623, Trn1_4S , AB, Zip2_2D , AC, Ext4 , DB), 2480 OP(4630, Ext12 , AB, Zip1_4S , AC, InsS_3To2, C ), OP(4631, Ext4 , BB, Uzp2_4S , AC, _ , _ ), OP(4632, Zip2_2D , AB, InsS_0To3, B , _ , _ ), OP(4633, Ext8 , BB, Dup3 , A , Uzp1_4S , DC), 2481 OP(4700, Ext12 , BB, Dup0 , A , Zip1_2D , DC), OP(4701, Ext12 , BB, Rev64 , A , Zip1_2D , DC), OP(4702, Uzp1_4S , AB, InsS_3To3, B , Rev64 , C ), OP(4703, Ext12 , BB, Ext12 , AA, Zip1_2D , DC), 2482 OP(4710, Ext12 , BB, Zip1_2D , AC, _ , _ ), OP(4711, Ext12 , BB, Dup1 , A , Zip1_2D , DC), OP(4712, Ext12 , BB, Zip1_2D , AC, InsS_2To0, A ), OP(4713, Uzp2_4S , AB, InsS_0To2, C , Ext4 , CB), 2483 OP(4720, Dup3 , B , Uzp1_4S , CB, Uzp1_4S , AD), OP(4721, Ext4 , AB, InsS_3To2, B , _ , _ ), OP(4722, Ext12 , BB, Dup2 , A , Zip1_2D , DC), OP(4723, Ext12 , BB, Rev64 , A , Ext8 , DC), 2484 OP(4730, Rev64 , A , InsS_3To3, B , Ext4 , CB), OP(4731, Dup3 , B , Ext4 , CB, Uzp2_4S , AD), OP(4732, Ext12 , BB, Ext8 , AC, _ , _ ), OP(4733, Ext12 , BB, Dup3 , A , Zip1_2D , DC), 2485 OP(5400, Dup0 , A , Zip1_2D , CB, _ , _ ), OP(5401, Rev64 , A , Zip1_2D , CB, _ , _ ), OP(5402, Ext8 , AB, InsS_0To1, A , _ , _ ), OP(5403, Ext12 , AA, Zip1_2D , CB, _ , _ ), 2486 OP(5410, Zip1_2D , AB, _ , _ , _ , _ ), OP(5411, Dup1 , A , Zip1_2D , CB, _ , _ ), OP(5412, Zip1_2D , AB, InsS_2To0, A , _ , _ ), OP(5413, Zip1_2D , AB, InsS_3To0, A , _ , _ ), 2487 OP(5420, Uzp1_4S , BA, Ext8 , CB, _ , _ ), OP(5421, Ext12 , BA, Ext8 , CB, _ , _ ), OP(5422, Dup2 , A , Zip1_2D , CB, _ , _ ), OP(5423, Rev64 , A , Ext8 , CB, _ , _ ), 2488 OP(5430, Zip1_2D , AB, InsS_3To1, A , _ , _ ), OP(5431, Uzp2_4S , BA, Ext8 , CB, _ , _ ), OP(5432, Ext8 , AB, _ , _ , _ , _ ), OP(5433, Dup3 , A , Zip1_2D , CB, _ , _ ), 2489 OP(5500, Dup1 , B , Dup0 , A , Uzp2_4S , DC), OP(5501, Rev64 , A , Dup1 , B , Zip1_2D , CD), OP(5502, Ext8 , AA, Dup1 , B , Uzp1_4S , CD), OP(5503, Ext12 , AA, Dup1 , B , Zip1_2D , CD), 2490 OP(5510, Dup1 , B , Zip1_2D , AC, _ , _ ), OP(5511, Trn2_4S , AB, Zip1_4S , CC, _ , _ ), OP(5512, Uzp2_4S , AB, Zip2_2D , AC, Trn1_4S , DC), OP(5513, Ext12 , AA, Dup1 , B , Uzp1_4S , CD), 2491 OP(5520, Dup1 , B , Uzp1_4S , AC, _ , _ ), OP(5521, Ext12 , BA, Dup1 , B , Zip2_2D , CD), OP(5522, Dup2 , A , Dup1 , B , Uzp2_4S , CD), OP(5523, Rev64 , A , Dup1 , B , Zip2_2D , CD), 2492 OP(5530, Ext12 , AB, Trn1_4S , AC, InsS_3To2, D ), OP(5531, Dup1 , B , Uzp2_4S , AC, _ , _ ), OP(5532, Dup1 , B , Zip2_2D , AC, _ , _ ), OP(5533, Ext12 , AB, Trn1_4S , CC, _ , _ ), 2493 OP(5600, Trn1_4S , BA, InsS_1To3, B , InsS_1To0, C ), OP(5601, Ext4 , BB, Zip1_2D , AC, Rev64 , D ), OP(5602, Uzp1_4S , AB, InsS_1To2, B , Rev64 , C ), OP(5603, Trn1_4S , BA, InsS_1To3, B , InsS_3To0, A ), 2494 OP(5610, Zip1_2D , AB, InsS_2To2, B , _ , _ ), OP(5611, Zip1_4S , AB, Trn1_4S , CB, Zip2_4S , DC), OP(5612, Trn2_4S , AB, Zip2_4S , AB, Zip1_4S , DC), OP(5613, Trn2_4S , AB, Zip2_4S , CB, Zip1_4S , DC), 2495 OP(5620, Dup2 , B , Uzp2_4S , CB, Uzp1_4S , AD), OP(5621, Rev64 , B , Ext4 , AC, InsS_3To2, C ), OP(5622, Uzp1_4S , AB, Zip1_2D , CB, Trn2_4S , CD), OP(5623, Uzp1_4S , AB, Uzp2_4S , CB, Ext12 , AD), 2496 OP(5630, Ext12 , AB, Trn1_4S , AC, InsS_3To2, C ), OP(5631, Dup2 , B , Zip1_2D , CB, Uzp2_4S , AD), OP(5632, Zip2_2D , AB, InsS_1To3, B , _ , _ ), OP(5633, Ext12 , AB, InsS_3To1, C , Ext12 , AC), 2497 OP(5700, Ext12 , BB, Dup0 , A , Uzp1_4S , DC), OP(5701, Uzp2_4S , BB, Zip1_2D , AC, Rev64 , D ), OP(5702, Rev64 , B , Uzp1_4S , AC, Rev64 , D ), OP(5703, Uzp2_4S , BB, InsS_0To0, A , Ext12 , AC), 2498 OP(5710, Zip1_2D , AB, InsS_3To2, B , _ , _ ), OP(5711, Ext12 , BB, Dup1 , A , Uzp1_4S , DC), OP(5712, Uzp2_4S , AB, InsS_2To1, A , Rev64 , C ), OP(5713, Uzp2_4S , AB, Rev64 , C , _ , _ ), 2499 OP(5720, Ext12 , BB, Uzp1_4S , AC, _ , _ ), OP(5721, Rev64 , B , Ext4 , AC, InsS_2To2, C ), OP(5722, Ext12 , BB, Dup2 , A , Uzp1_4S , DC), OP(5723, Uzp2_4S , BB, Zip2_2D , AC, Rev64 , D ), 2500 OP(5730, Ext12 , BB, Uzp1_4S , AC, InsS_3To1, A ), OP(5731, Ext8 , BB, Uzp2_4S , AC, _ , _ ), OP(5732, Ext8 , AB, InsS_3To2, B , _ , _ ), OP(5733, Ext12 , BB, Dup3 , A , Uzp1_4S , DC), 2501 OP(6400, Dup0 , A , Uzp1_4S , CB, _ , _ ), OP(6401, Dup1 , A , Uzp1_4S , CA, Uzp1_4S , DB), OP(6402, Ext8 , AA, Uzp1_4S , CB, _ , _ ), OP(6403, Dup3 , A , Uzp1_4S , CA, Uzp1_4S , DB), 2502 OP(6410, Uzp1_4S , BB, Zip1_2D , AC, _ , _ ), OP(6411, Dup1 , A , Uzp1_4S , CB, _ , _ ), OP(6412, Dup2 , A , Uzp2_4S , CA, Uzp1_4S , DB), OP(6413, Ext12 , AA, Uzp1_4S , CB, _ , _ ), 2503 OP(6420, Uzp1_4S , AB, _ , _ , _ , _ ), OP(6421, Uzp1_4S , AB, InsS_1To0, A , _ , _ ), OP(6422, Dup2 , A , Uzp1_4S , CB, _ , _ ), OP(6423, Uzp1_4S , AB, InsS_3To0, A , _ , _ ), 2504 OP(6430, Uzp1_4S , AB, InsS_3To1, A , _ , _ ), OP(6431, Rev64 , B , Uzp2_4S , AC, _ , _ ), OP(6432, Uzp1_4S , BB, Zip2_2D , AC, _ , _ ), OP(6433, Dup3 , A , Uzp1_4S , CB, _ , _ ), 2505 OP(6500, Ext12 , BB, Dup0 , A , Zip2_2D , DC), OP(6501, Ext4 , BB, Rev64 , A , Zip1_2D , DC), OP(6502, Rev64 , A , Ext12 , CB, InsS_1To1, C ), OP(6503, Ext12 , AB, InsS_0To1, A , _ , _ ), 2506 OP(6510, Ext4 , BB, Zip1_2D , AC, _ , _ ), OP(6511, Ext12 , BB, Dup1 , A , Zip2_2D , DC), OP(6512, Rev64 , A , Ext12 , CB, InsS_0To1, C ), OP(6513, Ext12 , AB, InsS_1To1, A , _ , _ ), 2507 OP(6520, Uzp1_4S , AB, InsS_1To2, B , _ , _ ), OP(6521, Trn2_4S , AB, Zip2_4S , AB, Zip1_4S , CD), OP(6522, Ext12 , BB, Dup2 , A , Zip2_2D , DC), OP(6523, Ext12 , AB, InsS_2To1, A , _ , _ ), 2508 OP(6530, Ext4 , BA, Trn2_4S , CA, Ext8 , DC), OP(6531, Uzp2_4S , AB, InsS_2To3, B , _ , _ ), OP(6532, Ext12 , BB, Zip2_2D , AC, _ , _ ), OP(6533, Ext12 , AB, InsS_0To1, C , _ , _ ), 2509 OP(6600, Dup2 , B , Dup0 , A , Uzp2_4S , DC), OP(6601, Rev64 , A , Dup2 , B , Zip1_2D , CD), OP(6602, Ext8 , AA, Dup2 , B , Uzp1_4S , CD), OP(6603, Ext12 , AA, Dup2 , B , Zip1_2D , CD), 2510 OP(6610, Dup2 , B , Zip1_2D , AC, _ , _ ), OP(6611, Dup2 , B , Dup1 , A , Uzp2_4S , DC), OP(6612, Zip2_4S , AB, Trn2_4S , AC, Zip1_4S , CD), OP(6613, Ext12 , AA, Dup2 , B , Uzp1_4S , CD), 2511 OP(6620, Dup2 , B , Uzp1_4S , AC, _ , _ ), OP(6621, Ext12 , BA, Dup2 , B , Zip2_2D , CD), OP(6622, Trn1_4S , AB, Zip2_4S , CC, _ , _ ), OP(6623, Rev64 , A , Dup2 , B , Zip2_2D , CD), 2512 OP(6630, Trn1_4S , AB, InsS_3To1, A , InsS_3To2, C ), OP(6631, Dup2 , B , Uzp2_4S , AC, _ , _ ), OP(6632, Dup2 , B , Zip2_2D , AC, _ , _ ), OP(6633, Dup3 , A , Dup2 , B , Uzp2_4S , CD), 2513 OP(6700, Rev64 , B , Dup0 , A , Zip2_2D , DC), OP(6701, Ext8 , BA, Rev64 , C , Ext8 , DD), OP(6702, Uzp1_4S , BA, Zip2_2D , CB, Rev64 , D ), OP(6703, Ext4 , AA, Rev64 , B , Zip2_2D , CD), 2514 OP(6710, Rev64 , B , InsD_0To0, A , _ , _ ), OP(6711, Rev64 , B , Dup1 , A , Zip2_2D , DC), OP(6712, Ext12 , BA, Zip2_2D , CB, Rev64 , D ), OP(6713, Uzp2_4S , BA, Zip2_2D , CB, Rev64 , D ), 2515 OP(6720, Uzp1_4S , AB, InsS_3To2, B , _ , _ ), OP(6721, Ext12 , BA, Rev64 , B , Zip2_2D , CD), OP(6722, Rev64 , B , Dup2 , A , Zip2_2D , DC), OP(6723, Zip2_2D , AB, Rev64 , C , _ , _ ), 2516 OP(6730, Ext4 , AA, Zip2_2D , CB, Rev64 , D ), OP(6731, Dup3 , B , Uzp1_4S , CB, Uzp2_4S , AD), OP(6732, Rev64 , B , Zip2_2D , AC, _ , _ ), OP(6733, Rev64 , B , Dup3 , A , Zip2_2D , DC), 2517 OP(7400, Uzp1_4S , AB, InsS_3To3, B , InsS_0To1, C ), OP(7401, Ext12 , BB, Zip1_2D , AC, Rev64 , D ), OP(7402, Ext4 , AB, Trn2_4S , CB, InsS_0To1, A ), OP(7403, Rev64 , B , InsS_0To0, A , Ext12 , AC), 2518 OP(7410, Zip1_2D , AB, InsS_3To3, B , _ , _ ), OP(7411, Trn2_4S , AB, InsS_0To2, B , InsS_0To1, C ), OP(7412, Ext4 , AB, Trn2_4S , CB, InsS_0To1, C ), OP(7413, Ext4 , AB, Zip2_4S , CB, InsS_0To1, C ), 2519 OP(7420, Uzp1_4S , AB, InsS_3To3, B , _ , _ ), OP(7421, Ext12 , BA, Trn1_4S , BC, Ext8 , CD), OP(7422, Ext4 , AB, Trn2_4S , CB, InsS_0To1, D ), OP(7423, Ext12 , BB, Ext8 , AC, Rev64 , D ), 2520 OP(7430, Uzp2_4S , AB, InsS_0To2, B , InsS_0To0, A ), OP(7431, Uzp2_4S , AB, InsS_0To2, B , _ , _ ), OP(7432, Zip2_2D , AB, InsS_0To2, B , _ , _ ), OP(7433, Trn2_4S , AB, Ext4 , AB, Zip2_4S , DC), 2521 OP(7500, Dup0 , A , Uzp2_4S , CB, _ , _ ), OP(7501, Uzp2_4S , AB, InsS_0To1, A , _ , _ ), OP(7502, Ext4 , AA, Uzp2_4S , CB, _ , _ ), OP(7503, Dup3 , A , Ext4 , CA, Uzp2_4S , DB), 2522 OP(7510, Uzp2_4S , BB, Zip1_2D , AC, _ , _ ), OP(7511, Dup1 , A , Uzp2_4S , CB, _ , _ ), OP(7512, Dup2 , A , Zip1_2D , CA, Uzp2_4S , DB), OP(7513, Ext8 , AA, Uzp2_4S , CB, _ , _ ), 2523 OP(7520, Rev64 , B , Uzp1_4S , AC, _ , _ ), OP(7521, Uzp2_4S , AB, InsS_2To1, A , _ , _ ), OP(7522, Dup2 , A , Uzp2_4S , CB, _ , _ ), OP(7523, Dup3 , A , Uzp1_4S , CA, Uzp2_4S , DB), 2524 OP(7530, Uzp2_4S , AB, InsS_0To0, A , _ , _ ), OP(7531, Uzp2_4S , AB, _ , _ , _ , _ ), OP(7532, Uzp2_4S , BB, Zip2_2D , AC, _ , _ ), OP(7533, Dup3 , A , Uzp2_4S , CB, _ , _ ), 2525 OP(7600, Dup0 , A , Zip2_2D , CB, _ , _ ), OP(7601, Rev64 , A , InsD_1To1, B , _ , _ ), OP(7602, Zip2_2D , AB, InsS_0To1, A , _ , _ ), OP(7603, Ext4 , AA, Zip2_2D , CB, _ , _ ), 2526 OP(7610, Zip2_2D , BB, Zip1_2D , AC, _ , _ ), OP(7611, Dup1 , A , Zip2_2D , CB, _ , _ ), OP(7612, Zip2_2D , AB, InsS_1To1, A , _ , _ ), OP(7613, Dup3 , A , Zip1_4S , CA, Zip2_2D , DB), 2527 OP(7620, Uzp1_4S , BA, Zip2_2D , CB, _ , _ ), OP(7621, Ext12 , BA, Zip2_2D , CB, _ , _ ), OP(7622, Dup2 , A , Zip2_2D , CB, _ , _ ), OP(7623, Rev64 , A , Zip2_2D , CB, _ , _ ), 2528 OP(7630, Zip2_2D , AB, InsS_0To0, A , _ , _ ), OP(7631, Uzp2_4S , BA, Zip2_2D , CB, _ , _ ), OP(7632, Zip2_2D , AB, _ , _ , _ , _ ), OP(7633, Dup3 , A , Zip2_2D , CB, _ , _ ), 2529 OP(7700, Dup3 , B , Dup0 , A , Uzp2_4S , DC), OP(7701, Rev64 , A , Dup3 , B , Zip1_2D , CD), OP(7702, Ext8 , AA, Dup3 , B , Uzp1_4S , CD), OP(7703, Ext12 , AA, Dup3 , B , Zip1_2D , CD), 2530 OP(7710, Dup3 , B , Zip1_2D , AC, _ , _ ), OP(7711, Dup3 , B , Dup1 , A , Uzp2_4S , DC), OP(7712, Trn2_4S , BA, InsS_2To0, A , InsS_2To3, C ), OP(7713, Ext12 , AA, Dup3 , B , Uzp1_4S , CD), 2531 OP(7720, Dup3 , B , Uzp1_4S , AC, _ , _ ), OP(7721, Ext12 , BA, Dup3 , B , Zip2_2D , CD), OP(7722, Dup3 , B , Dup2 , A , Uzp2_4S , DC), OP(7723, Rev64 , A , Dup3 , B , Zip2_2D , CD), 2532 OP(7730, Uzp2_4S , AB, InsS_0To0, A , InsS_3To2, C ), OP(7731, Dup3 , B , Uzp2_4S , AC, _ , _ ), OP(7732, Dup3 , B , Zip2_2D , AC, _ , _ ), OP(7733, Trn2_4S , AB, Zip2_4S , CC, _ , _ ) 2533 }; 2534 2535 #undef OP 2536 2537 static void emit_interleaved_shuffle32_impl(UniCompiler& uc, const Vec& dst, const Vec& src1, const Vec& src2, uint32_t imm) { 2538 ASMJIT_ASSERT((imm & 0xFCFCFCFC) == 0); 2539 2540 if (src1.id() == src2.id()) 2541 return emit_swizzle32_impl(uc, dst, src1, imm); 2542 2543 BackendCompiler* cc = uc.cc; 2544 2545 uint32_t table_index = ((imm & 0x03000000) >> (24 - 6)) | 2546 ((imm & 0x00030000) >> (16 - 4)) | 2547 ((imm & 0x00000300) >> (8 - 2)) | (imm & 0x00000003); 2548 2549 // By default the cost matrix tries to avoid inserting A to DST to minimize the number of instructions 2550 // in case that `dst == a`, which is common. However, we have a also a table that penalizes inserting B 2551 // to DST, so select the best version. 2552 InterleavedShuffle32Ops ops; 2553 if (dst.id() != src2.id()) 2554 ops = interleaved_shuffle32_ops[table_index]; 2555 else 2556 ops = interleaved_shuffle32_ops_dst_same_as_b[table_index]; 2557 2558 Vec regs[5] = { src1, src2, Vec(), Vec(), Vec() }; 2559 2560 uint32_t count = ops.count(); 2561 uint32_t src_use_mask[4] {}; 2562 uint32_t dst_index = count - 1; 2563 2564 { 2565 uint32_t i = count; 2566 uint32_t all_use_mask = 0; 2567 2568 while (i != 0) { 2569 i--; 2570 InterleavedShuffle32Ops::Op op = ops.op(i); 2571 2572 // Calculate masks that follow. 2573 all_use_mask |= 1u << op.src_a(); 2574 all_use_mask |= 1u << op.src_b(); 2575 src_use_mask[i] = all_use_mask; 2576 2577 // Calculate the last operation where we need a new destination (inserts insert to the last). 2578 if (op.is_ins_op()) { 2579 // The last destination would be the previous operation (if not another insert). 2580 dst_index = i - 1; 2581 } 2582 } 2583 } 2584 2585 uint32_t op_index = 2; 2586 Vec final_dst; 2587 2588 for (uint32_t i = 0; i < count; i++) { 2589 InterleavedShuffle32Ops::Op op = ops.op(i); 2590 2591 if (i > 0 && !op.is_ins_op()) { 2592 op_index++; 2593 } 2594 2595 // Should not be counted if it's invalid - so let's verify it here... 2596 ASMJIT_ASSERT(op.is_valid()); 2597 2598 Vec op_dst; 2599 if (i < dst_index) { 2600 if (regs[op_index].is_valid()) { 2601 op_dst = regs[op_index]; 2602 } 2603 else { 2604 op_dst = uc.new_similar_reg(dst, "@shuf_tmp_%u", op_index - 2); 2605 } 2606 } 2607 else { 2608 if (!final_dst.is_valid()) { 2609 uint32_t msk = 0; 2610 if (dst.id() == src1.id()) { 2611 msk = 0x1u; 2612 } 2613 else if (dst.id() == src2.id()) { 2614 msk = 0x2u; 2615 } 2616 2617 if (src_use_mask[i + 1] & msk) { 2618 // In this case the destination is in conflict with one of the source registers. We have to 2619 // create a new virtual register and then move it to the real `dst` to not mess up the shuffle. 2620 ASMJIT_ASSERT(!regs[op_index].is_valid()); 2621 final_dst = uc.new_similar_reg(dst, "@shuf_dst"); 2622 } 2623 else { 2624 // Perfect - the destination is not in conflict with any source register. 2625 final_dst = dst; 2626 } 2627 } 2628 op_dst = final_dst; 2629 } 2630 2631 uint32_t a_index = op.src_a(); 2632 uint32_t b_index = op.src_b(); 2633 2634 switch (op.op()) { 2635 case InterleavedShuffle32Ops::Op::kOpInsS: { 2636 uint32_t src_lane = op.ins_src(); 2637 uint32_t dst_lane = op.ins_dst(); 2638 cc->mov(op_dst.s(dst_lane), regs[a_index].s(src_lane)); 2639 break; 2640 } 2641 2642 case InterleavedShuffle32Ops::Op::kOpInsD: { 2643 uint32_t src_lane = op.ins_src(); 2644 uint32_t dst_lane = op.ins_dst(); 2645 cc->mov(op_dst.d(dst_lane), regs[a_index].d(src_lane)); 2646 break; 2647 } 2648 2649 case InterleavedShuffle32Ops::Op::kOpPerm: { 2650 static constexpr uint16_t perm_inst[8] = { 2651 Inst::kIdZip1_v, 2652 Inst::kIdZip2_v, 2653 Inst::kIdUzp1_v, 2654 Inst::kIdUzp2_v, 2655 Inst::kIdTrn1_v, 2656 Inst::kIdTrn2_v, 2657 Inst::kIdExt_v, 2658 Inst::kIdExt_v 2659 }; 2660 2661 InstId inst_id = perm_inst[op.perm_op()]; 2662 2663 if (inst_id == Inst::kIdExt_v) 2664 cc->ext(op_dst.b16(), regs[a_index].b16(), regs[b_index].b16(), op.perm_ext_imm()); 2665 else if (op.perm_s4()) 2666 cc->emit(inst_id, op_dst.s4(), regs[a_index].s4(), regs[b_index].s4()); 2667 else 2668 cc->emit(inst_id, op_dst.d2(), regs[a_index].d2(), regs[b_index].d2()); 2669 break; 2670 } 2671 2672 case InterleavedShuffle32Ops::Op::kOpRev64: { 2673 cc->rev64(op_dst.s4(), regs[a_index].s4()); 2674 break; 2675 } 2676 2677 case InterleavedShuffle32Ops::Op::kOpDup: { 2678 uint32_t idx = op.dup_idx(); 2679 cc->dup(op_dst.s4(), regs[a_index].s(idx)); 2680 break; 2681 } 2682 2683 default: 2684 ASMJIT_NOT_REACHED(); 2685 } 2686 2687 regs[op_index] = op_dst; 2688 } 2689 2690 vec_mov(uc, dst, final_dst); 2691 } 2692 2693 // ujit::UniCompiler - Vector Instructions - OpArray Iterator 2694 // ========================================================== 2695 2696 template<typename T> 2697 class OpArrayIter { 2698 public: 2699 const T& _op; 2700 2701 ASMJIT_INLINE_NODEBUG OpArrayIter(const T& op) noexcept : _op(op) {} 2702 ASMJIT_INLINE_NODEBUG const T& op() const noexcept { return _op; } 2703 ASMJIT_INLINE_NODEBUG void next() noexcept {} 2704 }; 2705 2706 template<> 2707 class OpArrayIter<OpArray> { 2708 public: 2709 const OpArray& _opArray; 2710 size_t _i {}; 2711 size_t _n {}; 2712 2713 ASMJIT_INLINE_NODEBUG OpArrayIter(const OpArray& op_array) noexcept : _opArray(op_array), _i(0), _n(op_array.size()) {} 2714 ASMJIT_INLINE_NODEBUG const Operand_& op() const noexcept { return _opArray[_i]; } 2715 ASMJIT_INLINE_NODEBUG void next() noexcept { if (++_i >= _n) _i = 0; } 2716 }; 2717 2718 template<typename Src> 2719 static ASMJIT_INLINE void emit_2v_t(UniCompiler& uc, UniOpVV op, const OpArray& dst_, const Src& src_) { 2720 size_t n = dst_.size(); 2721 OpArrayIter<Src> src(src_); 2722 2723 for (size_t i = 0; i < n; i++) { 2724 uc.emit_2v(op, dst_[i], src.op()); 2725 src.next(); 2726 } 2727 } 2728 2729 template<typename Src> 2730 static ASMJIT_INLINE void emit_2vi_t(UniCompiler& uc, UniOpVVI op, const OpArray& dst_, const Src& src_, uint32_t imm) { 2731 size_t n = dst_.size(); 2732 OpArrayIter<Src> src(src_); 2733 2734 for (size_t i = 0; i < n; i++) { 2735 uc.emit_2vi(op, dst_[i], src.op(), imm); 2736 src.next(); 2737 } 2738 } 2739 2740 template<typename Src1, typename Src2> 2741 static ASMJIT_INLINE void emit_3v_t(UniCompiler& uc, UniOpVVV op, const OpArray& dst_, const Src1& src1_, const Src2& src2_) { 2742 size_t n = dst_.size(); 2743 OpArrayIter<Src1> src1(src1_); 2744 OpArrayIter<Src2> src2(src2_); 2745 2746 for (size_t i = 0; i < n; i++) { 2747 uc.emit_3v(op, dst_[i], src1.op(), src2.op()); 2748 src1.next(); 2749 src2.next(); 2750 } 2751 } 2752 2753 template<typename Src1, typename Src2> 2754 static ASMJIT_INLINE void emit_3vi_t(UniCompiler& uc, UniOpVVVI op, const OpArray& dst_, const Src1& src1_, const Src2& src2_, uint32_t imm) { 2755 size_t n = dst_.size(); 2756 OpArrayIter<Src1> src1(src1_); 2757 OpArrayIter<Src2> src2(src2_); 2758 2759 for (size_t i = 0; i < n; i++) { 2760 uc.emit_3vi(op, dst_[i], src1.op(), src2.op(), imm); 2761 src1.next(); 2762 src2.next(); 2763 } 2764 } 2765 2766 template<typename Src1, typename Src2, typename Src3> 2767 static ASMJIT_INLINE void emit_4v_t(UniCompiler& uc, UniOpVVVV op, const OpArray& dst_, const Src1& src1_, const Src2& src2_, const Src3& src3_) { 2768 size_t n = dst_.size(); 2769 OpArrayIter<Src1> src1(src1_); 2770 OpArrayIter<Src2> src2(src2_); 2771 OpArrayIter<Src3> src3(src3_); 2772 2773 for (size_t i = 0; i < n; i++) { 2774 uc.emit_4v(op, dst_[i], src1.op(), src2.op(), src3.op()); 2775 src1.next(); 2776 src2.next(); 2777 src3.next(); 2778 } 2779 } 2780 2781 // ujit::UniCompiler - Vector Instructions - Emit 2V 2782 // ================================================= 2783 2784 void UniCompiler::emit_2v(UniOpVV op, const Operand_& dst_, const Operand_& src_) { 2785 ASMJIT_ASSERT(dst_.is_vec()); 2786 2787 Vec dst(dst_.as<Vec>()); 2788 2789 UniOpVInfo op_info = opcode_info_2v[size_t(op)]; 2790 InstId inst_id = op_info.inst_id; 2791 2792 switch (op) { 2793 case UniOpVV::kMov: { 2794 if (src_.is_vec()) { 2795 Vec src = src_.as<Vec>(); 2796 if (dst.size() < 16 || src.size() < 16) 2797 cc->mov(dst.b8(), src.b8()); 2798 else 2799 cc->mov(dst.b16(), src.b16()); 2800 return; 2801 } 2802 2803 vec_mov(*this, dst, src_); 2804 return; 2805 } 2806 2807 case UniOpVV::kMovU64: { 2808 dst = dst.d(); 2809 Vec src = as_vec(*this, src_, dst); 2810 2811 cc->mov(dst.b8(), src.b8()); 2812 return; 2813 } 2814 2815 case UniOpVV::kBroadcastU8Z: 2816 case UniOpVV::kBroadcastU16Z: 2817 case UniOpVV::kBroadcastU8: 2818 case UniOpVV::kBroadcastU16: 2819 case UniOpVV::kBroadcastU32: 2820 case UniOpVV::kBroadcastF32: 2821 case UniOpVV::kBroadcastU64: 2822 case UniOpVV::kBroadcastF64: { 2823 ElementSize element_size = ElementSize(op_info.dst_element); 2824 vec_set_type(dst, element_size); 2825 2826 Operand src(src_); 2827 2828 if (src.is_mem()) { 2829 // NOTE: ld1r instruction is pretty limited - it offers only `[base]` or `[base + offset|index]@` (post-index) 2830 // addressing. This means that it's sometimes just better to emit a regular load folowed by dup than to actually 2831 // use ld1r. 2832 Mem& m = src.as<Mem>(); 2833 if (!m.has_index() && !m.has_offset()) { 2834 cc->ld1r(dst, m); 2835 return; 2836 } 2837 2838 v_load_iany(dst, src.as<Mem>(), size_t(1) << uint32_t(op_info.src_element), Alignment(1)); 2839 src = dst; 2840 } 2841 2842 if (src.is_vec()) { 2843 Vec& src_vec = src.as<Vec>(); 2844 vec_set_type_and_index(src_vec, element_size, 0); 2845 cc->dup(dst, src_vec); 2846 return; 2847 } 2848 else if (src.is_gp()) { 2849 Gp& src_gp = src.as<Gp>(); 2850 if (element_size <= ElementSize::k32) 2851 src_gp = src_gp.r32(); 2852 else 2853 src_gp = src_gp.r64(); 2854 cc->dup(dst, src_gp); 2855 return; 2856 } 2857 2858 ASMJIT_NOT_REACHED(); 2859 } 2860 2861 case UniOpVV::kBroadcastV128_U32: 2862 case UniOpVV::kBroadcastV128_U64: 2863 case UniOpVV::kBroadcastV128_F32: 2864 case UniOpVV::kBroadcastV128_F64: 2865 // 128-bit broadcast is an alias of mov as we have only 128-bit vectors... 2866 v_mov(dst, src_); 2867 return; 2868 2869 case UniOpVV::kBroadcastV256_U32: 2870 case UniOpVV::kBroadcastV256_U64: 2871 case UniOpVV::kBroadcastV256_F32: 2872 case UniOpVV::kBroadcastV256_F64: 2873 // Unsupported as NEON has only 128-bit vectors. 2874 ASMJIT_NOT_REACHED(); 2875 2876 case UniOpVV::kAbsI8: 2877 case UniOpVV::kAbsI16: 2878 case UniOpVV::kAbsI32: 2879 case UniOpVV::kAbsI64: 2880 case UniOpVV::kNotU32: 2881 case UniOpVV::kNotU64: 2882 case UniOpVV::kNotF32: 2883 case UniOpVV::kNotF64: { 2884 Vec src = as_vec(*this, src_, dst); 2885 2886 vec_set_type(dst, op_info.dst_element); 2887 vec_set_type(src, op_info.src_element); 2888 2889 cc->emit(inst_id, dst, src); 2890 return; 2891 } 2892 2893 case UniOpVV::kCvtI8ToI32: 2894 case UniOpVV::kCvtU8ToU32: { 2895 Vec src = as_vec(*this, src_, dst); 2896 cc->emit(inst_id, dst.h8(), src.b8(), 0); 2897 cc->emit(inst_id, dst.s4(), dst.h4(), 0); 2898 return; 2899 } 2900 2901 case UniOpVV::kCvtI8LoToI16: 2902 case UniOpVV::kCvtI8HiToI16: 2903 case UniOpVV::kCvtU8LoToU16: 2904 case UniOpVV::kCvtU8HiToU16: 2905 case UniOpVV::kCvtI16LoToI32: 2906 case UniOpVV::kCvtI16HiToI32: 2907 case UniOpVV::kCvtU16LoToU32: 2908 case UniOpVV::kCvtU16HiToU32: 2909 case UniOpVV::kCvtI32LoToI64: 2910 case UniOpVV::kCvtI32HiToI64: 2911 case UniOpVV::kCvtU32LoToU64: 2912 case UniOpVV::kCvtU32HiToU64: { 2913 vec_set_type(dst, op_info.dst_element); 2914 2915 Vec src; 2916 2917 if (op_info.src_part == VecPart::kLo) { 2918 src = as_vec(*this, src_, dst, 8); 2919 src = src.v64(); 2920 } 2921 else if (src_.is_vec()) { 2922 src = src_.as<Vec>(); 2923 } 2924 else { 2925 Mem m(src_.as<Mem>()); 2926 m.add_offset(8); 2927 src = vec_from_mem(*this, m, dst, 8); 2928 src = src.v64(); 2929 2930 // Since we have loaded from memory, we want to use the low-part variant of the instruction. 2931 inst_id = opcode_info_2v[size_t(op) - 1u].inst_id; 2932 } 2933 2934 vec_set_type(src, op_info.src_element); 2935 cc->emit(inst_id, dst, src, 0); 2936 return; 2937 } 2938 2939 case UniOpVV::kRcpF32: 2940 case UniOpVV::kRcpF64: { 2941 // Intrinsic. 2942 const void* one_ptr = op_info.src_element == ElementSize::k32 ? static_cast<const void*>(&ct().f32_1) : static_cast<const void*>(&ct().f64_1); 2943 2944 Vec one = simd_vec_const(one_ptr, Bcst::kNA, dst); 2945 Vec src = as_vec(*this, src_, dst); 2946 2947 vec_set_type(dst, op_info.dst_element); 2948 vec_set_type(one, op_info.dst_element); 2949 vec_set_type(src, op_info.dst_element); 2950 2951 cc->fdiv(dst, one, src); 2952 return; 2953 } 2954 2955 case UniOpVV::kAbsF32S: 2956 case UniOpVV::kNegF32S: 2957 case UniOpVV::kTruncF32S: 2958 case UniOpVV::kFloorF32S: 2959 case UniOpVV::kCeilF32S: 2960 case UniOpVV::kRoundEvenF32S: 2961 case UniOpVV::kRoundHalfAwayF32S: 2962 case UniOpVV::kSqrtF32S: { 2963 dst.set_signature(RegTraits<RegType::kVec32>::kSignature); 2964 Vec src = as_vec(*this, src_, dst); 2965 2966 cc->emit(inst_id, dst, src); 2967 return; 2968 } 2969 2970 case UniOpVV::kAbsF64S: 2971 case UniOpVV::kNegF64S: 2972 case UniOpVV::kTruncF64S: 2973 case UniOpVV::kFloorF64S: 2974 case UniOpVV::kCeilF64S: 2975 case UniOpVV::kRoundEvenF64S: 2976 case UniOpVV::kRoundHalfAwayF64S: 2977 case UniOpVV::kSqrtF64S: { 2978 dst.set_signature(RegTraits<RegType::kVec64>::kSignature); 2979 Vec src = as_vec(*this, src_, dst); 2980 2981 cc->emit(inst_id, dst, src); 2982 return; 2983 } 2984 2985 case UniOpVV::kRoundHalfUpF32S: 2986 case UniOpVV::kRoundHalfUpF64S: 2987 case UniOpVV::kRoundHalfUpF32: 2988 case UniOpVV::kRoundHalfUpF64: { 2989 // Intrinsic. 2990 const void* one_ptr = 2991 op_info.src_element == ElementSize::k32 2992 ? static_cast<const void*>(&ct().f32_0_5_minus_1ulp) 2993 : static_cast<const void*>(&ct().f64_0_5_minus_1ulp); 2994 2995 Vec one = simd_vec_const(one_ptr, Bcst::kNA, dst); 2996 Vec src = as_vec(*this, src_, dst); 2997 2998 if (op == UniOpVV::kRoundHalfUpF32S) { 2999 dst.set_signature(RegTraits<RegType::kVec32>::kSignature); 3000 src.set_signature(RegTraits<RegType::kVec32>::kSignature); 3001 one.set_signature(RegTraits<RegType::kVec32>::kSignature); 3002 } 3003 else if (op == UniOpVV::kRoundHalfUpF64S) { 3004 dst.set_signature(RegTraits<RegType::kVec64>::kSignature); 3005 src.set_signature(RegTraits<RegType::kVec64>::kSignature); 3006 one.set_signature(RegTraits<RegType::kVec64>::kSignature); 3007 } 3008 else { 3009 vec_set_type(dst, op_info.dst_element); 3010 vec_set_type(one, op_info.src_element); 3011 vec_set_type(src, op_info.src_element); 3012 } 3013 3014 cc->fadd(dst, src, one); 3015 cc->frintm(dst, dst); 3016 3017 return; 3018 } 3019 3020 case UniOpVV::kAbsF32: 3021 case UniOpVV::kAbsF64: 3022 case UniOpVV::kNegF32: 3023 case UniOpVV::kNegF64: 3024 case UniOpVV::kTruncF32: 3025 case UniOpVV::kTruncF64: 3026 case UniOpVV::kFloorF32: 3027 case UniOpVV::kFloorF64: 3028 case UniOpVV::kCeilF32: 3029 case UniOpVV::kCeilF64: 3030 case UniOpVV::kRoundEvenF32: 3031 case UniOpVV::kRoundEvenF64: 3032 case UniOpVV::kRoundHalfAwayF32: 3033 case UniOpVV::kRoundHalfAwayF64: 3034 case UniOpVV::kSqrtF32: 3035 case UniOpVV::kSqrtF64: 3036 case UniOpVV::kCvtI32ToF32: 3037 case UniOpVV::kCvtRoundF32ToI32: 3038 case UniOpVV::kCvtTruncF32ToI32: { 3039 Vec src = as_vec(*this, src_, dst); 3040 3041 vec_set_type(dst, op_info.dst_element); 3042 vec_set_type(src, op_info.src_element); 3043 3044 cc->emit(inst_id, dst, src); 3045 return; 3046 } 3047 3048 case UniOpVV::kCvtF32ToF64S: 3049 case UniOpVV::kCvtF64ToF32S: { 3050 Vec src = as_vec(*this, src_, dst); 3051 3052 vec_set_vec_type(dst, op_info.dst_element); 3053 vec_set_vec_type(src, op_info.src_element); 3054 3055 cc->fcvt(dst, src); 3056 return; 3057 } 3058 3059 case UniOpVV::kCvtF32HiToF64: 3060 if (src_.is_mem()) { 3061 Vec src = as_vec(*this, src_.as<Mem>().clone_adjusted(8), dst, 8).v64(); 3062 3063 vec_set_type(dst, op_info.dst_element); 3064 vec_set_type(src, op_info.src_element); 3065 3066 cc->emit(opcode_info_2v[size_t(op) - 1u].inst_id, dst, src); 3067 return; 3068 } 3069 [[fallthrough]]; 3070 3071 case UniOpVV::kCvtF32LoToF64: { 3072 Vec src = as_vec(*this, src_, dst, 8); 3073 3074 if (op_info.src_part == VecPart::kLo) { 3075 src = src.v64(); 3076 } 3077 3078 vec_set_type(dst, op_info.dst_element); 3079 vec_set_type(src, op_info.src_element); 3080 3081 cc->emit(inst_id, dst, src); 3082 return; 3083 } 3084 3085 case UniOpVV::kCvtI32HiToF64: 3086 if (src_.is_mem()) { 3087 Vec src = as_vec(*this, src_.as<Mem>().clone_adjusted(8), dst, 8).v64(); 3088 3089 vec_set_type(dst, op_info.dst_element); 3090 vec_set_type(src, op_info.src_element); 3091 3092 cc->emit(opcode_info_2v[size_t(op) - 1u].inst_id, dst, src, 0); 3093 cc->scvtf(dst.d2(), dst.d2()); 3094 return; 3095 } 3096 [[fallthrough]]; 3097 3098 case UniOpVV::kCvtI32LoToF64: { 3099 Vec src = as_vec(*this, src_, dst, 8); 3100 3101 if (op_info.src_part == VecPart::kLo) { 3102 src = src.v64(); 3103 } 3104 3105 vec_set_type(dst, op_info.dst_element); 3106 vec_set_type(src, op_info.src_element); 3107 3108 cc->emit(inst_id, dst, src, 0); 3109 cc->scvtf(dst.d2(), dst.d2()); 3110 return; 3111 } 3112 3113 case UniOpVV::kCvtF64ToF32Lo: 3114 case UniOpVV::kCvtF64ToF32Hi: { 3115 dst = dst.q(); 3116 Vec src = as_vec(*this, src_, dst); 3117 3118 if (op_info.dst_part == VecPart::kLo) { 3119 dst = dst.d(); 3120 } 3121 3122 vec_set_type(dst, op_info.dst_element); 3123 vec_set_type(src, op_info.src_element); 3124 3125 cc->emit(inst_id, dst, src); 3126 return; 3127 } 3128 3129 case UniOpVV::kCvtTruncF64ToI32Lo: 3130 case UniOpVV::kCvtRoundF64ToI32Lo: 3131 case UniOpVV::kCvtTruncF64ToI32Hi: 3132 case UniOpVV::kCvtRoundF64ToI32Hi: { 3133 dst = dst.q(); 3134 3135 Vec src = as_vec(*this, src_, dst); 3136 Vec tmp = new_similar_reg(dst, "@tmp"); 3137 3138 cc->emit(inst_id, tmp.d2(), src.d2()); 3139 3140 if (op_info.dst_part == VecPart::kLo) { 3141 cc->sqxtn(dst.s2(), tmp.d2()); 3142 } 3143 else { 3144 cc->sqxtn2(dst.s4(), tmp.d2()); 3145 } 3146 3147 return; 3148 } 3149 3150 default: 3151 ASMJIT_NOT_REACHED(); 3152 } 3153 } 3154 3155 void UniCompiler::emit_2v(UniOpVV op, const OpArray& dst_, const Operand_& src_) { emit_2v_t(*this, op, dst_, src_); } 3156 void UniCompiler::emit_2v(UniOpVV op, const OpArray& dst_, const OpArray& src_) { emit_2v_t(*this, op, dst_, src_); } 3157 3158 // ujit::UniCompiler - Vector Instructions - Emit 2VI 3159 // ================================================== 3160 3161 void UniCompiler::emit_2vi(UniOpVVI op, const Operand_& dst_, const Operand_& src_, uint32_t imm) { 3162 ASMJIT_ASSERT(dst_.is_vec()); 3163 3164 Vec dst(dst_.as<Vec>()); 3165 3166 UniOpVInfo op_info = opcode_info_2vi[size_t(op)]; 3167 InstId inst_id = op_info.inst_id; 3168 3169 switch (op) { 3170 case UniOpVVI::kSllbU128: 3171 case UniOpVVI::kSrlbU128: { 3172 ASMJIT_ASSERT(imm < 16); 3173 3174 Vec src = as_vec(*this, src_, dst); 3175 3176 // If the shift is used to extract a high 64-bit element and zero the rest of the register. 3177 if (op == UniOpVVI::kSrlbU128 && imm == 8) { 3178 cc->dup(dst.d(), src.d(1)); 3179 return; 3180 } 3181 3182 // If the shift is used to extract the last 32-bit element and zero the rest of the register. 3183 if (op == UniOpVVI::kSrlbU128 && imm == 12) { 3184 cc->dup(dst.s(), src.s(3)); 3185 return; 3186 } 3187 3188 Vec zero = simd_vec_zero(dst); 3189 vec_set_type(dst, ElementSize::k8); 3190 vec_set_type(src, ElementSize::k8); 3191 vec_set_type(zero, ElementSize::k8); 3192 3193 if (op == UniOpVVI::kSllbU128) 3194 cc->ext(dst, zero, src, 16u - imm); 3195 else 3196 cc->ext(dst, src, zero, imm); 3197 return; 3198 } 3199 3200 case UniOpVVI::kSwizzleU16x4: 3201 case UniOpVVI::kSwizzleLoU16x4: 3202 case UniOpVVI::kSwizzleHiU16x4: { 3203 Vec src = as_vec(*this, src_, dst); 3204 3205 uint8_t pred_data[16] = { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF }; 3206 3207 uint32_t d = (imm >> 23) & (0x3 << 1); 3208 uint32_t c = (imm >> 15) & (0x3 << 1); 3209 uint32_t b = (imm >> 7) & (0x3 << 1); 3210 uint32_t a = (imm << 1) & (0x3 << 1); 3211 3212 if (op != UniOpVVI::kSwizzleHiU16x4) { 3213 pred_data[ 0] = uint8_t(a); 3214 pred_data[ 1] = uint8_t(a + 1u); 3215 pred_data[ 2] = uint8_t(b); 3216 pred_data[ 3] = uint8_t(b + 1u); 3217 pred_data[ 4] = uint8_t(c); 3218 pred_data[ 5] = uint8_t(c + 1u); 3219 pred_data[ 6] = uint8_t(d); 3220 pred_data[ 7] = uint8_t(d + 1u); 3221 } 3222 3223 if (op != UniOpVVI::kSwizzleLoU16x4) { 3224 pred_data[ 8] = uint8_t(a + 8u); 3225 pred_data[ 9] = uint8_t(a + 9u); 3226 pred_data[10] = uint8_t(b + 8u); 3227 pred_data[11] = uint8_t(b + 9u); 3228 pred_data[12] = uint8_t(c + 8u); 3229 pred_data[13] = uint8_t(c + 9u); 3230 pred_data[14] = uint8_t(d + 8u); 3231 pred_data[15] = uint8_t(d + 9u); 3232 } 3233 3234 Vec pred = simd_const_16b(pred_data); 3235 cc->tbl(dst.b16(), src.b16(), pred.b16()); 3236 return; 3237 } 3238 3239 case UniOpVVI::kSwizzleU32x4: 3240 case UniOpVVI::kSwizzleF32x4: { 3241 Vec src = as_vec(*this, src_, dst); 3242 emit_swizzle32_impl(*this, dst, src, imm); 3243 return; 3244 } 3245 3246 case UniOpVVI::kSwizzleU64x2: 3247 case UniOpVVI::kSwizzleF64x2: { 3248 Vec src = as_vec(*this, src_, dst); 3249 3250 // Use `dup` to broadcast one 64-bit elements. 3251 if (Swizzle2{imm} == swizzle(0, 0) || 3252 Swizzle2{imm} == swizzle(1, 1)) { 3253 uint32_t idx = imm & 0x1; 3254 cc->dup(dst.d2(), src.d(idx)); 3255 return; 3256 } 3257 3258 // Use `ext` to swap two 64-bit elements. 3259 if (Swizzle2{imm} == swizzle(0, 1)) { 3260 cc->ext(dst.b16(), src.b16(), src.b16(), 8); 3261 return; 3262 } 3263 3264 // NOP... 3265 if (Swizzle2{imm} == swizzle(1, 0)) { 3266 cc->mov(dst.b16(), src.b16()); 3267 return; 3268 } 3269 3270 ASMJIT_NOT_REACHED(); 3271 } 3272 3273 case UniOpVVI::kSwizzleF64x4: 3274 case UniOpVVI::kSwizzleU64x4: 3275 case UniOpVVI::kExtractV128_I32: 3276 case UniOpVVI::kExtractV128_I64: 3277 case UniOpVVI::kExtractV128_F32: 3278 case UniOpVVI::kExtractV128_F64: 3279 case UniOpVVI::kExtractV256_I32: 3280 case UniOpVVI::kExtractV256_I64: 3281 case UniOpVVI::kExtractV256_F32: 3282 case UniOpVVI::kExtractV256_F64: 3283 // Unsupported as NEON has only 128-bit vectors. 3284 ASMJIT_NOT_REACHED(); 3285 3286 default: { 3287 Vec src = as_vec(*this, src_, dst); 3288 3289 if (op_info.dst_part == VecPart::kLo) dst = dst.d(); 3290 if (op_info.src_part == VecPart::kLo) src = src.d(); 3291 3292 vec_set_type(dst, op_info.dst_element); 3293 vec_set_type(src, op_info.src_element); 3294 3295 cc->emit(inst_id, dst, src, imm); 3296 return; 3297 } 3298 } 3299 } 3300 3301 void UniCompiler::emit_2vi(UniOpVVI op, const OpArray& dst_, const Operand_& src_, uint32_t imm) { emit_2vi_t(*this, op, dst_, src_, imm); } 3302 void UniCompiler::emit_2vi(UniOpVVI op, const OpArray& dst_, const OpArray& src_, uint32_t imm) { emit_2vi_t(*this, op, dst_, src_, imm); } 3303 3304 // ujit::UniCompiler - Vector Instructions - Emit 2VS 3305 // ================================================== 3306 3307 void UniCompiler::emit_2vs(UniOpVR op, const Operand_& dst_, const Operand_& src_, uint32_t idx) { 3308 UniOpVInfo op_info = opcode_info_2vs[size_t(op)]; 3309 3310 switch (op) { 3311 case UniOpVR::kMov: { 3312 ASMJIT_ASSERT(dst_.is_reg()); 3313 ASMJIT_ASSERT(src_.is_reg()); 3314 3315 if (dst_.is_gp() && src_.is_vec()) { 3316 if (dst_.as<Reg>().size() == 4) 3317 cc->mov(dst_.as<Gp>(), src_.as<Vec>().s(0)); 3318 else 3319 cc->mov(dst_.as<Gp>(), src_.as<Vec>().d(0)); 3320 return; 3321 } 3322 3323 if (dst_.is_vec() && src_.is_gp()) { 3324 if (src_.as<Reg>().size() == 4) 3325 cc->fmov(dst_.as<Vec>().s(), src_.as<Gp>()); 3326 else 3327 cc->fmov(dst_.as<Vec>().d(), src_.as<Gp>()); 3328 return; 3329 } 3330 3331 ASMJIT_NOT_REACHED(); 3332 } 3333 3334 case UniOpVR::kMovU32: 3335 case UniOpVR::kMovU64: { 3336 ASMJIT_ASSERT(dst_.is_reg()); 3337 ASMJIT_ASSERT(src_.is_reg()); 3338 3339 if (dst_.is_gp() && src_.is_vec()) { 3340 if (op == UniOpVR::kMovU32) 3341 cc->mov(dst_.as<Gp>().r32(), src_.as<Vec>().s(0)); 3342 else 3343 cc->mov(dst_.as<Gp>().r64(), src_.as<Vec>().d(0)); 3344 return; 3345 } 3346 3347 if (dst_.is_vec() && src_.is_gp()) { 3348 if (op == UniOpVR::kMovU32) 3349 cc->fmov(dst_.as<Vec>().s(), src_.as<Gp>().r32()); 3350 else 3351 cc->fmov(dst_.as<Vec>().d(), src_.as<Gp>().r64()); 3352 return; 3353 } 3354 3355 ASMJIT_NOT_REACHED(); 3356 } 3357 3358 case UniOpVR::kInsertU8: 3359 case UniOpVR::kInsertU16: 3360 case UniOpVR::kInsertU32: 3361 case UniOpVR::kInsertU64: { 3362 ASMJIT_ASSERT(dst_.is_vec()); 3363 ASMJIT_ASSERT(src_.is_gp()); 3364 3365 Vec dst(dst_.as<Vec>()); 3366 Gp src(src_.as<Gp>()); 3367 3368 vec_set_type_and_index(dst, op_info.dst_element, idx); 3369 src.set_signature(op == UniOpVR::kInsertU64 ? RegTraits<RegType::kGp64>::kSignature : RegTraits<RegType::kGp32>::kSignature); 3370 3371 cc->mov(dst, src); 3372 return; 3373 } 3374 3375 case UniOpVR::kExtractU8: 3376 case UniOpVR::kExtractU16: 3377 case UniOpVR::kExtractU32: 3378 case UniOpVR::kExtractU64: { 3379 ASMJIT_ASSERT(dst_.is_gp()); 3380 ASMJIT_ASSERT(src_.is_vec()); 3381 3382 Gp dst(dst_.as<Gp>()); 3383 Vec src(src_.as<Vec>()); 3384 3385 dst.set_signature(op == UniOpVR::kExtractU64 ? RegTraits<RegType::kGp64>::kSignature : RegTraits<RegType::kGp32>::kSignature); 3386 vec_set_type_and_index(src, op_info.dst_element, idx); 3387 3388 cc->mov(dst, src); 3389 return; 3390 } 3391 3392 case UniOpVR::kCvtIntToF32: 3393 case UniOpVR::kCvtIntToF64: { 3394 ASMJIT_ASSERT(dst_.is_vec()); 3395 ASMJIT_ASSERT(src_.is_gp()); 3396 3397 Vec dst(dst_.as<Vec>()); 3398 vec_set_vec_type(dst, op_info.dst_element); 3399 cc->scvtf(dst, src_.as<Gp>()); 3400 return; 3401 } 3402 3403 case UniOpVR::kCvtTruncF32ToInt: 3404 case UniOpVR::kCvtRoundF32ToInt: 3405 case UniOpVR::kCvtTruncF64ToInt: 3406 case UniOpVR::kCvtRoundF64ToInt: { 3407 ASMJIT_ASSERT(dst_.is_gp()); 3408 ASMJIT_ASSERT(src_.is_vec()); 3409 3410 Vec src(src_.as<Vec>()); 3411 vec_set_vec_type(src, op_info.src_element); 3412 cc->emit(op_info.inst_id, dst_, src); 3413 return; 3414 } 3415 3416 default: 3417 ASMJIT_NOT_REACHED(); 3418 } 3419 } 3420 3421 // ujit::UniCompiler - Vector Instructions - Emit 2VM 3422 // ================================================== 3423 3424 void UniCompiler::emit_vm(UniOpVM op, const Vec& dst_, const Mem& src_, Alignment alignment, uint32_t idx) { 3425 ASMJIT_ASSERT(dst_.is_vec()); 3426 ASMJIT_ASSERT(src_.is_mem()); 3427 3428 Support::maybe_unused(alignment); 3429 3430 Vec dst(dst_); 3431 Mem src(src_); 3432 UniOpVMInfo op_info = opcode_info_2vm[size_t(op)]; 3433 3434 switch (op) { 3435 case UniOpVM::kLoad8: 3436 case UniOpVM::kLoad16_U16: 3437 case UniOpVM::kLoad32_U32: 3438 case UniOpVM::kLoad32_F32: 3439 case UniOpVM::kLoad64_U32: 3440 case UniOpVM::kLoad64_U64: 3441 case UniOpVM::kLoad64_F32: 3442 case UniOpVM::kLoad64_F64: 3443 case UniOpVM::kLoad128_U32: 3444 case UniOpVM::kLoad128_U64: 3445 case UniOpVM::kLoad128_F32: 3446 case UniOpVM::kLoad128_F64: { 3447 vec_load_mem(*this, dst, src, op_info.mem_size); 3448 return; 3449 } 3450 3451 case UniOpVM::kLoadN_U32: 3452 case UniOpVM::kLoadN_U64: 3453 case UniOpVM::kLoadN_F32: 3454 case UniOpVM::kLoadN_F64: { 3455 vec_load_mem(*this, dst.q(), src, 16); 3456 return; 3457 } 3458 3459 case UniOpVM::kLoadCvtN_U8ToU64: 3460 case UniOpVM::kLoadCvt16_U8ToU64: { 3461 Gp tmp = new_gp32("@tmp"); 3462 cc->ldrh(tmp, src); 3463 cc->mov(dst.b(), tmp); 3464 cc->lsr(tmp, tmp, 8); 3465 cc->mov(dst.b(8), tmp); 3466 return; 3467 } 3468 3469 case UniOpVM::kLoadCvt32_I8ToI16: 3470 case UniOpVM::kLoadCvt32_U8ToU16: 3471 case UniOpVM::kLoadCvt32_I8ToI32: 3472 case UniOpVM::kLoadCvt32_U8ToU32: 3473 case UniOpVM::kLoadCvt32_I16ToI32: 3474 case UniOpVM::kLoadCvt32_U16ToU32: 3475 case UniOpVM::kLoadCvt32_I32ToI64: 3476 case UniOpVM::kLoadCvt32_U32ToU64: 3477 case UniOpVM::kLoadCvt64_I8ToI16: 3478 case UniOpVM::kLoadCvt64_U8ToU16: 3479 case UniOpVM::kLoadCvt64_I16ToI32: 3480 case UniOpVM::kLoadCvt64_U16ToU32: 3481 case UniOpVM::kLoadCvt64_I32ToI64: 3482 case UniOpVM::kLoadCvt64_U32ToU64: { 3483 vec_load_mem(*this, dst, src, op_info.mem_size); 3484 emit_2v(UniOpVV(op_info.cvt_op), dst, dst); 3485 return; 3486 } 3487 3488 case UniOpVM::kLoadCvtN_I8ToI16: 3489 case UniOpVM::kLoadCvtN_I8ToI32: 3490 case UniOpVM::kLoadCvtN_U8ToU16: 3491 case UniOpVM::kLoadCvtN_U8ToU32: 3492 case UniOpVM::kLoadCvtN_I16ToI32: 3493 case UniOpVM::kLoadCvtN_U16ToU32: 3494 case UniOpVM::kLoadCvtN_I32ToI64: 3495 case UniOpVM::kLoadCvtN_U32ToU64: { 3496 vec_load_mem(*this, dst, src, dst.size() / 2u); 3497 emit_2v(UniOpVV(op_info.cvt_op), dst, dst); 3498 return; 3499 } 3500 3501 case UniOpVM::kLoadInsertU8: 3502 case UniOpVM::kLoadInsertU16: 3503 case UniOpVM::kLoadInsertU32: 3504 case UniOpVM::kLoadInsertF32: 3505 case UniOpVM::kLoadInsertU64: 3506 case UniOpVM::kLoadInsertF32x2: 3507 case UniOpVM::kLoadInsertF64: { 3508 if (!src.has_index() && !src.has_offset()) { 3509 vec_set_type_and_index(dst, op_info.element, idx); 3510 cc->ld1(dst, src); 3511 } 3512 else { 3513 Vec tmp = new_similar_reg(dst, "@tmp"); 3514 v_load_iany(tmp, src, op_info.mem_size, Alignment(1)); 3515 3516 vec_set_type_and_index(dst, op_info.element, idx); 3517 vec_set_type_and_index(tmp, op_info.element, 0); 3518 cc->mov(dst, tmp); 3519 } 3520 return; 3521 } 3522 3523 case UniOpVM::kLoad256_U32: 3524 case UniOpVM::kLoad256_U64: 3525 case UniOpVM::kLoad256_F32: 3526 case UniOpVM::kLoad256_F64: 3527 case UniOpVM::kLoad512_U32: 3528 case UniOpVM::kLoad512_U64: 3529 case UniOpVM::kLoad512_F32: 3530 case UniOpVM::kLoad512_F64: 3531 case UniOpVM::kLoadCvt32_U8ToU64: 3532 case UniOpVM::kLoadCvt64_U8ToU64: 3533 case UniOpVM::kLoadCvt64_I8ToI32: 3534 case UniOpVM::kLoadCvt64_U8ToU32: 3535 case UniOpVM::kLoadCvt128_I8ToI32: 3536 case UniOpVM::kLoadCvt128_U8ToU32: 3537 case UniOpVM::kLoadCvt128_I8ToI16: 3538 case UniOpVM::kLoadCvt128_U8ToU16: 3539 case UniOpVM::kLoadCvt128_I16ToI32: 3540 case UniOpVM::kLoadCvt128_U16ToU32: 3541 case UniOpVM::kLoadCvt128_I32ToI64: 3542 case UniOpVM::kLoadCvt128_U32ToU64: 3543 case UniOpVM::kLoadCvt256_I8ToI16: 3544 case UniOpVM::kLoadCvt256_U8ToU16: 3545 case UniOpVM::kLoadCvt256_I16ToI32: 3546 case UniOpVM::kLoadCvt256_U16ToU32: 3547 case UniOpVM::kLoadCvt256_I32ToI64: 3548 case UniOpVM::kLoadCvt256_U32ToU64: 3549 // Unsupported as NEON has only 128-bit vectors. 3550 ASMJIT_NOT_REACHED(); 3551 3552 default: 3553 ASMJIT_NOT_REACHED(); 3554 } 3555 } 3556 3557 void UniCompiler::emit_vm(UniOpVM op, const OpArray& dst_, const Mem& src_, Alignment alignment, uint32_t idx) { 3558 Support::maybe_unused(alignment); 3559 3560 size_t i = 0; 3561 size_t n = dst_.size(); 3562 3563 if (!n) 3564 return; 3565 3566 Mem src(src_); 3567 UniOpVMInfo op_info = opcode_info_2vm[size_t(op)]; 3568 3569 uint32_t overridden_mem_size = op_info.mem_size; 3570 uint32_t mem_size = overridden_mem_size ? overridden_mem_size : dst_[0].as<Vec>().size(); 3571 3572 if (op <= UniOpVM::kLoadN_F64 && !src.has_index() && !src.has_offset() && mem_size >= 4) { 3573 while (i + 2 <= n) { 3574 const Vec& dst0 = dst_[i + 0].as<Vec>(); 3575 const Vec& dst1 = dst_[i + 1].as<Vec>(); 3576 3577 if (mem_size == 4) 3578 cc->ldp(dst0.s(), dst1.s(), src); 3579 else if (mem_size == 8) 3580 cc->ldp(dst0.d(), dst1.d(), src); 3581 else 3582 cc->ldp(dst0.q(), dst1.q(), src); 3583 3584 src.add_offset_lo32(int32_t(mem_size * 2)); 3585 3586 i += 2; 3587 } 3588 } 3589 3590 while (i < n) { 3591 ASMJIT_ASSERT(dst_[i].is_vec()); 3592 3593 const Vec& dst = dst_[i].as<Vec>(); 3594 mem_size = dst.size(); 3595 3596 emit_vm(op, dst, src, Alignment(1), idx); 3597 3598 src.add_offset_lo32(int32_t(mem_size)); 3599 i++; 3600 } 3601 } 3602 3603 void UniCompiler::emit_mv(UniOpMV op, const Mem& dst_, const Vec& src_, Alignment alignment, uint32_t idx) { 3604 ASMJIT_ASSERT(dst_.is_mem()); 3605 ASMJIT_ASSERT(src_.is_vec()); 3606 3607 Support::maybe_unused(alignment); 3608 3609 Mem dst(dst_); 3610 Vec src(src_); 3611 3612 switch (op) { 3613 case UniOpMV::kStore8: { 3614 cc->str(src.b(), dst); 3615 return; 3616 } 3617 3618 case UniOpMV::kStore16_U16: { 3619 cc->str(src.h(), dst); 3620 return; 3621 } 3622 3623 case UniOpMV::kStore32_U32: 3624 case UniOpMV::kStore32_F32: { 3625 cc->str(src.s(), dst); 3626 return; 3627 } 3628 3629 case UniOpMV::kStore64_U32: 3630 case UniOpMV::kStore64_U64: 3631 case UniOpMV::kStore64_F32: 3632 case UniOpMV::kStore64_F64: { 3633 cc->str(src.d(), dst); 3634 return; 3635 } 3636 3637 case UniOpMV::kStore128_U32: 3638 case UniOpMV::kStore128_U64: 3639 case UniOpMV::kStore128_F32: 3640 case UniOpMV::kStore128_F64: { 3641 cc->str(src.q(), dst); 3642 return; 3643 } 3644 3645 case UniOpMV::kStoreN_U32: 3646 case UniOpMV::kStoreN_U64: 3647 case UniOpMV::kStoreN_F32: 3648 case UniOpMV::kStoreN_F64: { 3649 cc->str(src, dst); 3650 return; 3651 } 3652 3653 /* 3654 case UniOpMV::kStoreCvtz64_U16ToU8: 3655 case UniOpMV::kStoreCvtz64_U32ToU16: 3656 case UniOpMV::kStoreCvtz64_U64ToU32: 3657 case UniOpMV::kStoreCvts64_I16ToI8: 3658 case UniOpMV::kStoreCvts64_I16ToU8: 3659 case UniOpMV::kStoreCvts64_U16ToU8: 3660 case UniOpMV::kStoreCvts64_I32ToI16: 3661 case UniOpMV::kStoreCvts64_U32ToU16: 3662 case UniOpMV::kStoreCvts64_I64ToI32: 3663 case UniOpMV::kStoreCvts64_U64ToU32: 3664 case UniOpMV::kStoreCvtzN_U16ToU8: 3665 case UniOpMV::kStoreCvtzN_U32ToU16: 3666 case UniOpMV::kStoreCvtzN_U64ToU32: 3667 case UniOpMV::kStoreCvtsN_I16ToI8: 3668 case UniOpMV::kStoreCvtsN_I16ToU8: 3669 case UniOpMV::kStoreCvtsN_U16ToU8: 3670 case UniOpMV::kStoreCvtsN_I32ToI16: 3671 case UniOpMV::kStoreCvtsN_U32ToU16: 3672 case UniOpMV::kStoreCvtsN_I64ToI32: 3673 case UniOpMV::kStoreCvtsN_U64ToU32: 3674 UNIMPLEMENTED(); 3675 return; 3676 */ 3677 3678 case UniOpMV::kStore256_U32: 3679 case UniOpMV::kStore256_U64: 3680 case UniOpMV::kStore256_F32: 3681 case UniOpMV::kStore256_F64: 3682 case UniOpMV::kStore512_U32: 3683 case UniOpMV::kStore512_U64: 3684 case UniOpMV::kStore512_F32: 3685 case UniOpMV::kStore512_F64: 3686 /* 3687 case UniOpMV::kStoreCvtz128_U16ToU8: 3688 case UniOpMV::kStoreCvtz128_U32ToU16: 3689 case UniOpMV::kStoreCvtz128_U64ToU32: 3690 case UniOpMV::kStoreCvts128_I16ToI8: 3691 case UniOpMV::kStoreCvts128_I16ToU8: 3692 case UniOpMV::kStoreCvts128_U16ToU8: 3693 case UniOpMV::kStoreCvts128_I32ToI16: 3694 case UniOpMV::kStoreCvts128_U32ToU16: 3695 case UniOpMV::kStoreCvts128_I64ToI32: 3696 case UniOpMV::kStoreCvts128_U64ToU32: 3697 case UniOpMV::kStoreCvtz256_U16ToU8: 3698 case UniOpMV::kStoreCvtz256_U32ToU16: 3699 case UniOpMV::kStoreCvtz256_U64ToU32: 3700 case UniOpMV::kStoreCvts256_I16ToI8: 3701 case UniOpMV::kStoreCvts256_I16ToU8: 3702 case UniOpMV::kStoreCvts256_U16ToU8: 3703 case UniOpMV::kStoreCvts256_I32ToI16: 3704 case UniOpMV::kStoreCvts256_U32ToU16: 3705 case UniOpMV::kStoreCvts256_I64ToI32: 3706 case UniOpMV::kStoreCvts256_U64ToU32: 3707 */ 3708 // Unsupported as NEON has only 128-bit vectors. 3709 ASMJIT_NOT_REACHED(); 3710 3711 case UniOpMV::kStoreExtractU16: { 3712 if (idx == 0) { 3713 cc->str(src.h(), dst); 3714 } 3715 else { 3716 cc->st1(src.h(idx), dst); 3717 } 3718 return; 3719 } 3720 3721 case UniOpMV::kStoreExtractU32: { 3722 if (idx == 0) { 3723 cc->str(src.s(), dst); 3724 } 3725 else { 3726 cc->st1(src.s(idx), dst); 3727 } 3728 return; 3729 } 3730 3731 case UniOpMV::kStoreExtractU64: { 3732 if (idx == 0) { 3733 cc->str(src.d(), dst); 3734 } 3735 else { 3736 cc->st1(src.d(idx), dst); 3737 } 3738 return; 3739 } 3740 3741 default: 3742 ASMJIT_NOT_REACHED(); 3743 } 3744 } 3745 3746 void UniCompiler::emit_mv(UniOpMV op, const Mem& dst_, const OpArray& src_, Alignment alignment, uint32_t idx) { 3747 Support::maybe_unused(alignment); 3748 3749 size_t i = 0; 3750 size_t n = src_.size(); 3751 3752 if (!n) 3753 return; 3754 3755 Mem dst(dst_); 3756 UniOpVMInfo op_info = opcode_info_2mv[size_t(op)]; 3757 3758 uint32_t overridden_mem_size = op_info.mem_size; 3759 uint32_t mem_size = overridden_mem_size ? overridden_mem_size : src_[0].as<Vec>().size(); 3760 3761 if (op <= UniOpMV::kStoreN_F64 && mem_size >= 4) { 3762 bool good_offset = (uint32_t(dst.offset_lo32()) & (mem_size - 1)) == 0u; 3763 3764 if (good_offset) { 3765 while (i + 2 <= n) { 3766 const Vec& src0 = src_[i + 0].as<Vec>(); 3767 const Vec& src1 = src_[i + 1].as<Vec>(); 3768 3769 if (mem_size == 4) 3770 cc->stp(src0.s(), src1.s(), dst); 3771 else if (mem_size == 8) 3772 cc->stp(src0.d(), src1.d(), dst); 3773 else 3774 cc->stp(src0.q(), src1.q(), dst); 3775 3776 dst.add_offset_lo32(int32_t(mem_size * 2)); 3777 3778 i += 2; 3779 } 3780 } 3781 } 3782 3783 while (i < n) { 3784 ASMJIT_ASSERT(src_[i].is_vec()); 3785 3786 const Vec& src = src_[i].as<Vec>(); 3787 emit_mv(op, dst, src, Alignment(1), idx); 3788 3789 dst.add_offset_lo32(int32_t(mem_size)); 3790 i++; 3791 }} 3792 3793 // ujit::UniCompiler - Vector Instructions - Emit 3V 3794 // ================================================= 3795 3796 static void emit_3v_op( 3797 UniCompiler& uc, 3798 InstId inst_id, 3799 Vec dst, Vec src1, Operand_ src2_, 3800 FloatMode float_mode, 3801 ElementSize dst_element, VecPart dst_part, 3802 ElementSize src_element, VecPart src_part, 3803 uint32_t reversed) { 3804 3805 Vec src2; 3806 3807 switch (float_mode) { 3808 case FloatMode::kF32S: { 3809 dst = dst.s(); 3810 src1 = src1.s(); 3811 src2 = as_vec(uc, src2_, dst, 4); 3812 break; 3813 } 3814 3815 case FloatMode::kF64S: { 3816 dst = dst.d(); 3817 src1 = src1.d(); 3818 src2 = as_vec(uc, src2_, dst, 8); 3819 break; 3820 } 3821 3822 default: { 3823 src2 = as_vec(uc, src2_, dst); 3824 3825 if (dst_part == VecPart::kLo) { 3826 dst = dst.d(); 3827 } 3828 3829 if (src_part == VecPart::kLo) { 3830 src1 = src1.d(); 3831 src2 = src2.d(); 3832 } 3833 3834 vec_set_type(dst, dst_element); 3835 vec_set_type(src1, src_element); 3836 vec_set_type(src2, src_element); 3837 break; 3838 } 3839 } 3840 3841 BackendCompiler* cc = uc.cc; 3842 if (reversed) 3843 cc->emit(inst_id, dst, src2, src1); 3844 else 3845 cc->emit(inst_id, dst, src1, src2); 3846 } 3847 3848 void UniCompiler::emit_3v(UniOpVVV op, const Operand_& dst_, const Operand_& src1_, const Operand_& src2_) { 3849 ASMJIT_ASSERT(dst_.is_vec()); 3850 ASMJIT_ASSERT(src1_.is_vec()); 3851 3852 Vec dst(dst_.as<Vec>()); 3853 Vec src1(src1_.as<Vec>().clone_as(dst)); 3854 3855 UniOpVInfo op_info = opcode_info_3v[size_t(op)]; 3856 InstId inst_id = op_info.inst_id; 3857 3858 if (is_same_vec(src1, src2_)) { 3859 switch (op_info.same_vec_op) { 3860 case SameVecOp::kZero: { 3861 cc->movi(dst.b16(), 0); 3862 return; 3863 } 3864 3865 case SameVecOp::kOnes: { 3866 cc->movi(dst.b16(), 0xFF); 3867 return; 3868 } 3869 3870 case SameVecOp::kSrc: { 3871 vec_mov(*this, dst, src1); 3872 return; 3873 } 3874 3875 default: 3876 break; 3877 } 3878 } 3879 3880 switch (op) { 3881 // dst = a - (floor(a / b) * b). 3882 case UniOpVVV::kModF32S: 3883 case UniOpVVV::kModF64S: 3884 case UniOpVVV::kModF32: 3885 case UniOpVVV::kModF64: { 3886 Vec src2 = as_vec(*this, src2_, dst, op_info.float_mode); 3887 Vec tmp = new_similar_reg(dst, "@tmp1"); 3888 3889 UniOpVVV sub_op = translate_op(op, UniOpVVV::kModF32S, UniOpVVV::kSubF32S); 3890 UniOpVVV mul_op = translate_op(op, UniOpVVV::kModF32S, UniOpVVV::kMulF32S); 3891 UniOpVVV div_op = translate_op(op, UniOpVVV::kModF32S, UniOpVVV::kDivF32S); 3892 UniOpVV trunc_op = translate_op(op, UniOpVVV::kModF32S, UniOpVV::kTruncF32S); 3893 3894 emit_3v(div_op, tmp, src1, src2); 3895 emit_2v(trunc_op, tmp, tmp); 3896 emit_3v(mul_op, tmp, tmp, src2); 3897 emit_3v(sub_op, dst, src1, tmp); 3898 3899 return; 3900 } 3901 3902 case UniOpVVV::kMulU64: { 3903 Vec src2 = as_vec(*this, src2_, dst); 3904 Vec tmp1 = new_similar_reg(dst, "@tmp1"); 3905 Vec tmp2 = new_similar_reg(dst, "@tmp2"); 3906 Vec tmp3 = new_similar_reg(dst, "@tmp3"); 3907 3908 cc->rev64(tmp1.s4(), src1.s4()); 3909 cc->xtn(tmp2.s2(), src1.d2()); 3910 cc->mul(tmp1.s4(), tmp1.s4(), src2.s4()); 3911 cc->xtn(tmp3.s2(), src2.d2()); 3912 cc->uaddlp(tmp1.d2(), tmp1.s4()); 3913 cc->shl(dst.d2(), tmp1.d2(), 32); 3914 cc->umlal(dst.d2(), tmp2.s2(), tmp3.s2()); 3915 3916 return; 3917 } 3918 3919 case UniOpVVV::kMulhI16: { 3920 Vec src2 = as_vec(*this, src2_, dst); 3921 Vec tmp = new_similar_reg(dst, "@tmp"); 3922 3923 cc->smull(tmp.s4(), src1.h4(), src2.h4()); 3924 cc->smull2(dst.s4(), src1.h8(), src2.h8()); 3925 cc->uzp2(dst.h8(), tmp.h8(), dst.h8()); 3926 return; 3927 } 3928 3929 case UniOpVVV::kMulhU16: { 3930 Vec src2 = as_vec(*this, src2_, dst); 3931 Vec tmp = new_similar_reg(dst, "@tmp"); 3932 3933 cc->umull(tmp.s4(), src1.h4(), src2.h4()); 3934 cc->umull2(dst.s4(), src1.h8(), src2.h8()); 3935 cc->uzp2(dst.h8(), tmp.h8(), dst.h8()); 3936 return; 3937 } 3938 3939 case UniOpVVV::kMulU64_LoU32: { 3940 Vec src2 = as_vec(*this, src2_, dst); 3941 Vec tmp1 = new_similar_reg(dst, "@tmp1"); 3942 Vec tmp2 = new_similar_reg(dst, "@tmp2"); 3943 Vec tmp3 = dst; 3944 3945 if (dst.id() == src1.id() || dst.id() == src2.id()) 3946 tmp3 = new_similar_reg(dst, "@tmp3"); 3947 3948 cc->xtn(tmp1.s2(), src1.d2()); 3949 cc->shl(tmp3.d2(), src2.d2(), 32); 3950 cc->xtn(tmp2.s2(), src2.d2()); 3951 cc->mul(dst.s4(), tmp3.s4(), src1.s4()); 3952 cc->umlal(dst.d2(), tmp1.s2(), tmp2.s2()); 3953 3954 return; 3955 } 3956 3957 case UniOpVVV::kMHAddI16_I32: { 3958 Vec src2 = as_vec(*this, src2_, dst); 3959 3960 Vec al = new_similar_reg(dst, "@al"); 3961 Vec ah = new_similar_reg(dst, "@ah"); 3962 Vec bl = new_similar_reg(dst, "@bl"); 3963 Vec bh = new_similar_reg(dst, "@bh"); 3964 3965 cc->xtn(al.h4(), src1.s4()); 3966 cc->xtn(bl.h4(), src2.s4()); 3967 3968 cc->shrn(ah.h4(), src1.s4(), 16); 3969 cc->shrn(bh.h4(), src2.s4(), 16); 3970 3971 cc->smull(dst.s4(), al.h4(), bl.h4()); 3972 cc->smlal(dst.s4(), ah.h4(), bh.h4()); 3973 3974 return; 3975 } 3976 3977 case UniOpVVV::kMinI64: 3978 case UniOpVVV::kMinU64: 3979 case UniOpVVV::kMaxI64: 3980 case UniOpVVV::kMaxU64: { 3981 Vec src2 = as_vec(*this, src2_, dst); 3982 3983 // Min/Max is commutative, so let's make dst only overlap src1. 3984 if (dst.id() == src2.id()) { 3985 std::swap(src1, src2); 3986 } 3987 3988 bool dst_overlaps_src = dst.id() == src1.id(); 3989 3990 Vec tmp = dst; 3991 if (dst_overlaps_src) { 3992 tmp = new_similar_reg(dst, "@tmp"); 3993 } 3994 3995 // Let's emit a nicer sequence for u64 maximum. 3996 if (op == UniOpVVV::kMaxU64) { 3997 cc->uqsub(tmp.d2(), src1.d2(), src2.d2()); 3998 cc->add(dst.d2(), tmp.d2(), src2.d2()); 3999 return; 4000 } 4001 4002 cc->emit(inst_id, tmp.d2(), src1.d2(), src2.d2()); 4003 4004 if (op_info.imm) 4005 v_blendv_u8(dst, src2, src1, tmp); 4006 else 4007 v_blendv_u8(dst, src1, src2, tmp); 4008 4009 return; 4010 } 4011 4012 case UniOpVVV::kCmpNeF32S: 4013 case UniOpVVV::kCmpNeF64S: 4014 case UniOpVVV::kCmpNeF32: 4015 case UniOpVVV::kCmpNeF64: { 4016 emit_3v_op(*this, inst_id, dst, src1, src2_, op_info.float_mode, op_info.dst_element, op_info.dst_part, op_info.src_element, op_info.src_part, 0); 4017 vec_neg(*this, dst, dst, op_info.float_mode); 4018 return; 4019 } 4020 4021 case UniOpVVV::kCmpOrdF32S: 4022 case UniOpVVV::kCmpOrdF64S: 4023 case UniOpVVV::kCmpOrdF32: 4024 case UniOpVVV::kCmpOrdF64: 4025 case UniOpVVV::kCmpUnordF32S: 4026 case UniOpVVV::kCmpUnordF64S: 4027 case UniOpVVV::kCmpUnordF32: 4028 case UniOpVVV::kCmpUnordF64: { 4029 if (is_same_vec(src1, src2_)) { 4030 emit_3v_op(*this, Inst::kIdFcmeq_v, dst, src1, src1, op_info.float_mode, op_info.dst_element, op_info.dst_part, op_info.src_element, op_info.src_part, 0); 4031 } 4032 else { 4033 // This takes advantage of the following: 4034 // 4035 // When FPCR.AH is 0, the behavior is as follows (ASSUMED): 4036 // 4037 // - Negative zero compares less than positive zero. 4038 // - When FPCR.DN is 0, if either element is a NaN, the result is a quiet NaN. 4039 // - When FPCR.DN is 1, if either element is a NaN, the result is Default NaN. 4040 // 4041 // When FPCR.AH is 1, the behavior is as follows (USED FOR X86 EMULATION - NOT ASSUMED): 4042 // 4043 // - If both elements are zeros, regardless of the sign of either zero, the result is the second element. 4044 // - If either element is a NaN, regardless of the value of FPCR.DN, the result is the second element. 4045 Vec src2 = as_vec(*this, src2_, dst, op_info.float_mode); 4046 emit_3v_op(*this, Inst::kIdFmin_v, dst, src1, src2, op_info.float_mode, op_info.dst_element, op_info.dst_part, op_info.src_element, op_info.src_part, 0); 4047 emit_3v_op(*this, Inst::kIdFcmeq_v, dst, dst, dst, op_info.float_mode, op_info.dst_element, op_info.dst_part, op_info.src_element, op_info.src_part, 0); 4048 } 4049 4050 if (op_info.imm) 4051 vec_neg(*this, dst, dst, op_info.float_mode); 4052 4053 return; 4054 } 4055 4056 case UniOpVVV::kHAddF64: { 4057 Vec tmp = new_similar_reg(dst, "@tmp"); 4058 Vec src2 = as_vec(*this, src2_, dst); 4059 4060 if (src1.id() == src2.id()) { 4061 cc->ext(tmp.b16(), src1.b16(), src1.b16(), 8); 4062 cc->fadd(dst.d2(), src1.d2(), tmp.d2()); 4063 } 4064 else { 4065 cc->zip1(tmp.d2(), src1.d2(), src2.d2()); 4066 cc->zip2(dst.d2(), src1.d2(), src2.d2()); 4067 cc->fadd(dst.d2(), dst.d2(), tmp.d2()); 4068 } 4069 return; 4070 } 4071 4072 case UniOpVVV::kCombineLoHiU64: 4073 case UniOpVVV::kCombineLoHiF64: { 4074 // Intrinsic - dst = {src1.u64[0], src2.64[1]} - combining low part of src1 and high part of src1. 4075 Vec src2 = as_vec(*this, src2_, dst); 4076 4077 vec_set_type(dst, ElementSize::k8); 4078 vec_set_type(src1, ElementSize::k8); 4079 vec_set_type(src2, ElementSize::k8); 4080 4081 // `EXT dst, a, b, #n` -> `dst = b:a >> #n*8` 4082 cc->ext(dst, src2, src1, 8); 4083 return; 4084 } 4085 4086 case UniOpVVV::kCombineHiLoU64: 4087 case UniOpVVV::kCombineHiLoF64: { 4088 // Intrinsic - dst = {src1.u64[1], src2.64[0]} - combining high part of src1 and low part of src2. 4089 Vec src2 = as_vec(*this, src2_, dst); 4090 4091 if (is_same_vec(dst, src1)) { 4092 if (is_same_vec(dst, src2)) 4093 return; 4094 cc->mov(dst.d(0), src2.d(0)); 4095 } 4096 else if (is_same_vec(dst, src2)) { 4097 cc->mov(dst.d(1), src1.d(1)); 4098 } 4099 else { 4100 cc->mov(dst.b16(), src1.b16()); 4101 cc->mov(dst.d(0), src2.d(0)); 4102 } 4103 4104 return; 4105 } 4106 4107 case UniOpVVV::kPacksI16_I8: 4108 case UniOpVVV::kPacksI16_U8: 4109 case UniOpVVV::kPacksI32_I16: 4110 case UniOpVVV::kPacksI32_U16: { 4111 static constexpr uint16_t pack_lo_inst[4] = { Inst::kIdSqxtn_v , Inst::kIdSqxtun_v , Inst::kIdSqxtn_v , Inst::kIdSqxtun_v }; 4112 static constexpr uint16_t pack_hi_inst[4] = { Inst::kIdSqxtn2_v, Inst::kIdSqxtun2_v, Inst::kIdSqxtn2_v, Inst::kIdSqxtun2_v }; 4113 4114 size_t id = size_t(op) - size_t(UniOpVVV::kPacksI16_I8); 4115 4116 Vec src2 = as_vec(*this, src2_, dst); 4117 vec_set_type(src1, op_info.src_element); 4118 vec_set_type(src2, op_info.src_element); 4119 4120 if (src1.id() == src2.id()) { 4121 Vec dst_d = dst.d(); 4122 4123 vec_set_type(dst, op_info.dst_element); 4124 vec_set_type(dst_d, op_info.dst_element); 4125 4126 cc->emit(pack_lo_inst[id], dst_d, src1); 4127 cc->mov(dst.d(1), dst.d(0)); 4128 } 4129 else { 4130 Vec tmp = dst; 4131 if (dst.id() == src1.id() || dst.id() == src2.id()) { 4132 tmp = new_similar_reg(dst, "@tmp"); 4133 } 4134 4135 a64::Vec tmp_d = tmp.d(); 4136 4137 vec_set_type(tmp, op_info.dst_element); 4138 vec_set_type(tmp_d, op_info.dst_element); 4139 4140 cc->emit(pack_lo_inst[id], tmp_d, src1); 4141 cc->emit(pack_hi_inst[id], tmp, src2); 4142 4143 if (dst.id() != tmp.id()) { 4144 cc->mov(dst.b16(), tmp.b16()); 4145 } 4146 } 4147 return; 4148 } 4149 4150 default: { 4151 emit_3v_op(*this, inst_id, dst, src1, src2_, op_info.float_mode, op_info.dst_element, op_info.dst_part, op_info.src_element, op_info.src_part, op_info.reverse); 4152 return; 4153 } 4154 } 4155 } 4156 4157 void UniCompiler::emit_3v(UniOpVVV op, const OpArray& dst_, const Operand_& src1_, const OpArray& src2_) { emit_3v_t(*this, op, dst_, src1_, src2_); } 4158 void UniCompiler::emit_3v(UniOpVVV op, const OpArray& dst_, const OpArray& src1_, const Operand_& src2_) { emit_3v_t(*this, op, dst_, src1_, src2_); } 4159 void UniCompiler::emit_3v(UniOpVVV op, const OpArray& dst_, const OpArray& src1_, const OpArray& src2_) { emit_3v_t(*this, op, dst_, src1_, src2_); } 4160 4161 // ujit::UniCompiler - Vector Instructions - Emit 3VI 4162 // ================================================== 4163 4164 void UniCompiler::emit_3vi(UniOpVVVI op, const Operand_& dst_, const Operand_& src1_, const Operand_& src2_, uint32_t imm) { 4165 ASMJIT_ASSERT(dst_.is_vec()); 4166 ASMJIT_ASSERT(src1_.is_vec()); 4167 4168 Vec dst(dst_.as<Vec>()); 4169 Vec src1(src1_.as<Vec>().clone_as(dst)); 4170 4171 // Not used at the moment - maybe the info is not needed in this case. 4172 UniOpVInfo op_info = opcode_info_3vi[size_t(op)]; 4173 Support::maybe_unused(op_info); 4174 4175 switch (op) { 4176 case UniOpVVVI::kAlignr_U128: { 4177 ASMJIT_ASSERT(imm < 16); 4178 4179 if (imm == 0) { 4180 vec_mov(*this, dst, src2_); 4181 return; 4182 } 4183 4184 Vec src2 = as_vec(*this, src2_, dst); 4185 vec_set_type(dst, ElementSize::k8); 4186 vec_set_type(src1, ElementSize::k8); 4187 vec_set_type(src2, ElementSize::k8); 4188 cc->ext(dst, src2, src1, imm); 4189 return; 4190 } 4191 4192 case UniOpVVVI::kInterleaveShuffleU32x4: 4193 case UniOpVVVI::kInterleaveShuffleF32x4: { 4194 ASMJIT_ASSERT((imm & 0xFCFCFCFC) == 0); 4195 4196 Vec src2 = as_vec(*this, src2_, dst); 4197 emit_interleaved_shuffle32_impl(*this, dst, src1, src2, imm); 4198 return; 4199 } 4200 4201 case UniOpVVVI::kInterleaveShuffleU64x2: 4202 case UniOpVVVI::kInterleaveShuffleF64x2: { 4203 ASMJIT_ASSERT((imm & 0xFFFCFEFE) == 0); 4204 4205 Vec src2 = as_vec(*this, src2_, dst); 4206 4207 if (src1.id() == src2.id()) { 4208 v_swizzle_u64x2(dst, src1, Swizzle2{imm}); 4209 return; 4210 } 4211 4212 if (Swizzle2{imm} == swizzle(0, 0)) 4213 cc->zip1(dst.d2(), src1.d2(), src2.d2()); 4214 else if (Swizzle2{imm} == swizzle(1, 1)) 4215 cc->zip2(dst.d2(), src1.d2(), src2.d2()); 4216 else if (Swizzle2{imm} == swizzle(1, 0)) 4217 v_combine_hi_lo_u64(dst, src2, src1); 4218 else 4219 v_combine_lo_hi_u64(dst, src2, src1); 4220 4221 return; 4222 } 4223 4224 case UniOpVVVI::kInsertV128_U32: 4225 case UniOpVVVI::kInsertV128_F32: 4226 case UniOpVVVI::kInsertV128_U64: 4227 case UniOpVVVI::kInsertV128_F64: 4228 case UniOpVVVI::kInsertV256_U32: 4229 case UniOpVVVI::kInsertV256_F32: 4230 case UniOpVVVI::kInsertV256_U64: 4231 case UniOpVVVI::kInsertV256_F64: 4232 // Unsupported as NEON has only 128-bit vectors. 4233 ASMJIT_NOT_REACHED(); 4234 4235 default: 4236 ASMJIT_NOT_REACHED(); 4237 } 4238 } 4239 4240 void UniCompiler::emit_3vi(UniOpVVVI op, const OpArray& dst_, const Operand_& src1_, const OpArray& src2_, uint32_t imm) { emit_3vi_t(*this, op, dst_, src1_, src2_, imm); } 4241 void UniCompiler::emit_3vi(UniOpVVVI op, const OpArray& dst_, const OpArray& src1_, const Operand_& src2_, uint32_t imm) { emit_3vi_t(*this, op, dst_, src1_, src2_, imm); } 4242 void UniCompiler::emit_3vi(UniOpVVVI op, const OpArray& dst_, const OpArray& src1_, const OpArray& src2_, uint32_t imm) { emit_3vi_t(*this, op, dst_, src1_, src2_, imm); } 4243 4244 // ujit::UniCompiler - Vector Instructions - Emit 4V 4245 // ================================================= 4246 4247 void UniCompiler::emit_4v(UniOpVVVV op, const Operand_& dst_, const Operand_& src1_, const Operand_& src2_, const Operand_& src3_) { 4248 ASMJIT_ASSERT(dst_.is_vec()); 4249 ASMJIT_ASSERT(src1_.is_vec()); 4250 4251 Vec dst(dst_.as<Vec>()); 4252 Vec src1(src1_.as<Vec>().clone_as(dst)); 4253 4254 UniOpVInfo op_info = opcode_info_4v[size_t(op)]; 4255 InstId inst_id = op_info.inst_id; 4256 4257 switch (op) { 4258 case UniOpVVVV::kBlendV_U8: { 4259 Vec src2 = as_vec(*this, src2_, dst); 4260 Vec src3 = as_vec(*this, src3_, dst); 4261 4262 vec_set_type(dst, op_info.dst_element); 4263 vec_set_type(src1, op_info.src_element); 4264 vec_set_type(src2, op_info.src_element); 4265 vec_set_type(src3, op_info.src_element); 4266 4267 // We can pick between these depending on register arrangement: 4268 // - BSL (bitwise select) 4269 // - BIT (bitwise insert if true) 4270 // - BIF (bitwise insert if false) 4271 4272 if (dst.id() == src1.id()) { 4273 cc->bit(dst, src2, src3); 4274 return; 4275 } 4276 4277 if (dst.id() == src2.id()) { 4278 cc->bif(dst, src1, src3); 4279 return; 4280 } 4281 4282 vec_mov(*this, dst, src3); 4283 cc->bsl(dst, src2, src1); 4284 return; 4285 } 4286 4287 case UniOpVVVV::kMAddF32S: 4288 case UniOpVVVV::kMAddF64S: 4289 case UniOpVVVV::kMSubF32S: 4290 case UniOpVVVV::kMSubF64S: 4291 case UniOpVVVV::kNMAddF32S: 4292 case UniOpVVVV::kNMAddF64S: 4293 case UniOpVVVV::kNMSubF32S: 4294 case UniOpVVVV::kNMSubF64S: { 4295 Vec src2; 4296 Vec src3; 4297 4298 if (op_info.float_mode == FloatMode::kF32S) { 4299 dst = dst.s(); 4300 src1 = src1.s(); 4301 src2 = as_vec(*this, src2_, dst, 4); 4302 src3 = as_vec(*this, src3_, dst, 4); 4303 } 4304 else { 4305 dst = dst.d(); 4306 src1 = src1.d(); 4307 src2 = as_vec(*this, src2_, dst, 8); 4308 src3 = as_vec(*this, src3_, dst, 8); 4309 } 4310 4311 cc->emit(inst_id, dst, src1, src2, src3); 4312 return; 4313 } 4314 4315 case UniOpVVVV::kMAddU16: 4316 case UniOpVVVV::kMAddU32: 4317 case UniOpVVVV::kMAddF32: 4318 case UniOpVVVV::kMAddF64: 4319 case UniOpVVVV::kMSubF32: 4320 case UniOpVVVV::kMSubF64: 4321 case UniOpVVVV::kNMAddF32: 4322 case UniOpVVVV::kNMAddF64: 4323 case UniOpVVVV::kNMSubF32: 4324 case UniOpVVVV::kNMSubF64: { 4325 Vec src2 = as_vec(*this, src2_, dst); 4326 Vec src3; 4327 4328 bool negate_acc = op_info.imm != 0; 4329 bool dst_overlaps = dst.id() == src1.id() || dst.id() == src2.id(); 4330 bool destructible = is_same_vec(dst, src3_) || !src3_.is_reg(); 4331 4332 if (!dst_overlaps && src3_.is_mem()) { 4333 vec_load_mem(*this, dst, src3_.as<Mem>(), dst.size()); 4334 src3 = dst; 4335 } 4336 else { 4337 src3 = as_vec(*this, src3_, dst); 4338 } 4339 4340 vec_set_type(dst, op_info.dst_element); 4341 vec_set_type(src1, op_info.src_element); 4342 vec_set_type(src2, op_info.src_element); 4343 vec_set_type(src3, op_info.src_element); 4344 4345 if (destructible) { 4346 if (negate_acc) 4347 cc->fneg(src3, src3); 4348 4349 cc->emit(inst_id, src3, src1, src2); 4350 4351 if (dst.id() != src3.id()) 4352 cc->mov(dst, src3); 4353 return; 4354 } 4355 4356 Vec tmp = dst; 4357 4358 if (dst_overlaps) { 4359 tmp = new_similar_reg(dst, "@tmp"); 4360 vec_set_type(tmp, op_info.dst_element); 4361 } 4362 4363 if (negate_acc) 4364 cc->fneg(tmp, src3); 4365 else 4366 cc->mov(tmp, src3); 4367 4368 cc->emit(inst_id, tmp, src1, src2); 4369 4370 if (dst.id() != tmp.id()) 4371 cc->mov(dst, tmp); 4372 return; 4373 } 4374 4375 default: { 4376 ASMJIT_NOT_REACHED(); 4377 } 4378 } 4379 } 4380 4381 void UniCompiler::emit_4v(UniOpVVVV op, const OpArray& dst_, const Operand_& src1_, const Operand_& src2_, const OpArray& src3_) { emit_4v_t(*this, op, dst_, src1_, src2_, src3_); } 4382 void UniCompiler::emit_4v(UniOpVVVV op, const OpArray& dst_, const Operand_& src1_, const OpArray& src2_, const Operand& src3_) { emit_4v_t(*this, op, dst_, src1_, src2_, src3_); } 4383 void UniCompiler::emit_4v(UniOpVVVV op, const OpArray& dst_, const Operand_& src1_, const OpArray& src2_, const OpArray& src3_) { emit_4v_t(*this, op, dst_, src1_, src2_, src3_); } 4384 void UniCompiler::emit_4v(UniOpVVVV op, const OpArray& dst_, const OpArray& src1_, const Operand_& src2_, const Operand& src3_) { emit_4v_t(*this, op, dst_, src1_, src2_, src3_); } 4385 void UniCompiler::emit_4v(UniOpVVVV op, const OpArray& dst_, const OpArray& src1_, const Operand_& src2_, const OpArray& src3_) { emit_4v_t(*this, op, dst_, src1_, src2_, src3_); } 4386 void UniCompiler::emit_4v(UniOpVVVV op, const OpArray& dst_, const OpArray& src1_, const OpArray& src2_, const Operand& src3_) { emit_4v_t(*this, op, dst_, src1_, src2_, src3_); } 4387 void UniCompiler::emit_4v(UniOpVVVV op, const OpArray& dst_, const OpArray& src1_, const OpArray& src2_, const OpArray& src3_) { emit_4v_t(*this, op, dst_, src1_, src2_, src3_); } 4388 4389 ASMJIT_END_SUB_NAMESPACE 4390 4391 #endif