a64assembler.cpp (171013B)
1 // This file is part of AsmJit project <https://asmjit.com> 2 // 3 // See <asmjit/core.h> or LICENSE.md for license and copyright information 4 // SPDX-License-Identifier: Zlib 5 6 #include "../core/api-build_p.h" 7 #if !defined(ASMJIT_NO_AARCH64) 8 9 #include "../core/codewriter_p.h" 10 #include "../core/cpuinfo.h" 11 #include "../core/emitterutils_p.h" 12 #include "../core/formatter.h" 13 #include "../core/logger.h" 14 #include "../core/misc_p.h" 15 #include "../core/support.h" 16 17 #include "../arm/armformatter_p.h" 18 #include "../arm/armutils.h" 19 #include "../arm/a64assembler.h" 20 #include "../arm/a64emithelper_p.h" 21 #include "../arm/a64instdb_p.h" 22 23 ASMJIT_BEGIN_SUB_NAMESPACE(a64) 24 25 // a64::Assembler - Utils 26 // ====================== 27 28 static ASMJIT_INLINE_CONSTEXPR uint32_t diff(RegType a, RegType b) noexcept { return uint32_t(a) - uint32_t(b); } 29 static ASMJIT_INLINE_CONSTEXPR uint32_t diff(VecElementType element_type, VecElementType base_type) noexcept { return uint32_t(element_type) - uint32_t(base_type); } 30 31 // a64::Assembler - Cond 32 // ===================== 33 34 static inline uint32_t cond_code_to_opcode_field(uint32_t cond) noexcept { return (uint32_t(cond) - 2u) & 0xFu; } 35 36 // a64::Assembler - Bits 37 // ===================== 38 39 template<typename T> 40 static inline constexpr uint32_t B(const T& index) noexcept { return uint32_t(1u) << uint32_t(index); } 41 42 static constexpr uint32_t kSP = Gp::kIdSp; 43 static constexpr uint32_t kZR = Gp::kIdZr; 44 static constexpr uint32_t kWX = InstDB::kWX; 45 46 // a64::Assembler - ShiftOpToLdStOptMap 47 // ==================================== 48 49 // Table that maps ShiftOp to OPT part in LD/ST (register) opcode. 50 #define VALUE(index) index == uint32_t(ShiftOp::kUXTW) ? 2u : \ 51 index == uint32_t(ShiftOp::kLSL) ? 3u : \ 52 index == uint32_t(ShiftOp::kSXTW) ? 6u : \ 53 index == uint32_t(ShiftOp::kSXTX) ? 7u : 0xFF 54 static const uint8_t shift_op_to_ld_st_opt_map[] = { ASMJIT_LOOKUP_TABLE_16(VALUE, 0) }; 55 #undef VALUE 56 57 // a64::Assembler - ExtendOpToRegType 58 // ================================== 59 60 static inline RegType extend_option_to_reg_type(uint32_t option) noexcept { 61 uint32_t pred = (uint32_t(RegType::kGp32) << (0x0 * 4)) | // 0b000 - UXTB. 62 (uint32_t(RegType::kGp32) << (0x1 * 4)) | // 0b001 - UXTH. 63 (uint32_t(RegType::kGp32) << (0x2 * 4)) | // 0b010 - UXTW. 64 (uint32_t(RegType::kGp64) << (0x3 * 4)) | // 0b011 - UXTX|LSL. 65 (uint32_t(RegType::kGp32) << (0x4 * 4)) | // 0b100 - SXTB. 66 (uint32_t(RegType::kGp32) << (0x5 * 4)) | // 0b101 - SXTH. 67 (uint32_t(RegType::kGp32) << (0x6 * 4)) | // 0b110 - SXTW. 68 (uint32_t(RegType::kGp64) << (0x7 * 4)) ; // 0b111 - SXTX. 69 return RegType((pred >> (option * 4u)) & 0xFu); 70 } 71 72 // asmjit::a64::Assembler - SizeOp 73 // =============================== 74 75 //! Struct that contains Size (2 bits), Q flag, and S (scalar) flag. These values 76 //! are used to encode Q, Size, and Scalar fields in an opcode. 77 struct SizeOp { 78 //! \name Constants 79 //! \{ 80 81 static inline constexpr uint8_t k128BitShift = 0; 82 static inline constexpr uint8_t kScalarShift = 1; 83 static inline constexpr uint8_t kSizeShift = 2; 84 85 static inline constexpr uint8_t kQ = uint8_t(1u << k128BitShift); 86 static inline constexpr uint8_t kS = uint8_t(1u << kScalarShift); 87 88 static inline constexpr uint8_t k00 = uint8_t(0 << kSizeShift); 89 static inline constexpr uint8_t k01 = uint8_t(1 << kSizeShift); 90 static inline constexpr uint8_t k10 = uint8_t(2 << kSizeShift); 91 static inline constexpr uint8_t k11 = uint8_t(3 << kSizeShift); 92 93 static inline constexpr uint8_t k00Q = k00 | kQ; 94 static inline constexpr uint8_t k01Q = k01 | kQ; 95 static inline constexpr uint8_t k10Q = k10 | kQ; 96 static inline constexpr uint8_t k11Q = k11 | kQ; 97 98 static inline constexpr uint8_t k00S = k00 | kS; 99 static inline constexpr uint8_t k01S = k01 | kS; 100 static inline constexpr uint8_t k10S = k10 | kS; 101 static inline constexpr uint8_t k11S = k11 | kS; 102 103 static inline constexpr uint8_t kInvalid = 0xFFu; 104 105 // Masks used by SizeOpMap. 106 static inline constexpr uint8_t kSzQ = (0x3u << kSizeShift) | kQ; 107 static inline constexpr uint8_t kSzS = (0x3u << kSizeShift) | kS; 108 static inline constexpr uint8_t kSzQS = (0x3u << kSizeShift) | kQ | kS; 109 110 //! \} 111 112 //! \name Members 113 //! \{ 114 115 uint8_t value; 116 117 //! \} 118 119 //! \name Accessors 120 //! \{ 121 122 inline bool is_valid() const noexcept { return value != kInvalid; } 123 inline void make_invalid() noexcept { value = kInvalid; } 124 125 inline uint32_t q() const noexcept { return (value >> k128BitShift) & 0x1u; } 126 inline uint32_t qs() const noexcept { return ((value >> k128BitShift) | (value >> kScalarShift)) & 0x1u; } 127 inline uint32_t scalar() const noexcept { return (value >> kScalarShift) & 0x1u; } 128 inline uint32_t size() const noexcept { return (value >> kSizeShift) & 0x3u; } 129 130 inline void decrement_size() noexcept { 131 ASMJIT_ASSERT(size() > 0); 132 value = uint8_t(value - (1u << kSizeShift)); 133 } 134 135 //! \} 136 }; 137 138 struct SizeOpTable { 139 enum TableId : uint8_t { 140 kTableBin = 0, 141 kTableAny, 142 kCount 143 }; 144 145 // 40 elements for each combination. 146 SizeOp array[(uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8) + 1) * 8]; 147 }; 148 149 #define VALUE_BIN(x) { \ 150 x == (((uint32_t(RegType::kVec64 ) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k00 : \ 151 x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k00Q : \ 152 x == (((uint32_t(RegType::kVec64 ) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00 : \ 153 x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00Q : SizeOp::kInvalid \ 154 } 155 156 #define VALUE_ANY(x) { \ 157 x == (((uint32_t(RegType::kVec8) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k00S : \ 158 x == (((uint32_t(RegType::kVec16) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k01S : \ 159 x == (((uint32_t(RegType::kVec32) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k10S : \ 160 x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k11S : \ 161 x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00 : \ 162 x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00Q : \ 163 x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kH )) ? SizeOp::k01 : \ 164 x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kH )) ? SizeOp::k01Q : \ 165 x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kS )) ? SizeOp::k10 : \ 166 x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kS )) ? SizeOp::k10Q : \ 167 x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kD )) ? SizeOp::k11S : \ 168 x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kD )) ? SizeOp::k11Q : SizeOp::kInvalid \ 169 } 170 171 static const SizeOpTable size_op_table[SizeOpTable::kCount] = { 172 {{ ASMJIT_LOOKUP_TABLE_40(VALUE_BIN, 0) }}, 173 {{ ASMJIT_LOOKUP_TABLE_40(VALUE_ANY, 0) }} 174 }; 175 176 #undef VALUE_ANY 177 #undef VALUE_BIN 178 179 struct SizeOpMap { 180 uint8_t table_id; 181 uint8_t size_op_mask; 182 uint16_t accept_mask; 183 }; 184 185 static const constexpr SizeOpMap size_op_map[InstDB::kVO_Count] = { 186 { // kVO_V_B: 187 SizeOpTable::kTableBin, SizeOp::kQ , uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q)) 188 }, 189 190 { // kVO_V_BH: 191 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q) | B(SizeOp::k01) | B(SizeOp::k01Q)) 192 }, 193 194 { // kVO_V_BH_4S: 195 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q) | B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k10Q)) 196 }, 197 198 { // kVO_V_BHS: 199 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q) | B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k10) | B(SizeOp::k10Q)) 200 }, 201 202 { // kVO_V_BHS_D2: 203 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q) | B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k10) | B(SizeOp::k10Q) | B(SizeOp::k11Q)) 204 }, 205 206 { // kVO_V_HS: 207 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k10) | B(SizeOp::k10Q)) 208 }, 209 210 { // kVO_V_S: 211 SizeOpTable::kTableAny, SizeOp::kQ , uint16_t(B(SizeOp::k10) | B(SizeOp::k10Q)) 212 }, 213 214 { // kVO_V_B8H4: 215 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k01)) 216 }, 217 218 { // kVO_V_B8H4S2: 219 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k01) | B(SizeOp::k10)) 220 }, 221 222 { // kVO_V_B8D1: 223 SizeOpTable::kTableAny, SizeOp::kSzQ , uint16_t(B(SizeOp::k00) | B(SizeOp::k11S)) 224 }, 225 226 { // kVO_V_H4S2: 227 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k01) | B(SizeOp::k10)) 228 }, 229 230 { // kVO_V_B16: 231 SizeOpTable::kTableBin, SizeOp::kQ , uint16_t(B(SizeOp::k00Q)) 232 }, 233 234 { // kVO_V_B16H8: 235 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00Q) | B(SizeOp::k01Q)) 236 }, 237 238 { // kVO_V_B16H8S4: 239 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00Q) | B(SizeOp::k01Q) | B(SizeOp::k10Q)) 240 }, 241 242 { // kVO_V_B16D2: 243 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00Q) | B(SizeOp::k11Q)) 244 }, 245 246 { // kVO_V_H8S4: 247 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k01Q) | B(SizeOp::k10Q)) 248 }, 249 250 { // kVO_V_S4: 251 SizeOpTable::kTableAny, 0 , uint16_t(B(SizeOp::k10Q)) 252 }, 253 254 { // kVO_V_D2: 255 SizeOpTable::kTableAny, 0 , uint16_t(B(SizeOp::k11Q)) 256 }, 257 258 { // kVO_SV_BHS: 259 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q) | B(SizeOp::k00S) | B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k01S) | B(SizeOp::k10) | B(SizeOp::k10Q) | B(SizeOp::k10S)) 260 }, 261 262 { // kVO_SV_B8H4S2: 263 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00S) | B(SizeOp::k01) | B(SizeOp::k01S) | B(SizeOp::k10) | B(SizeOp::k10S)) 264 }, 265 266 { // kVO_SV_HS: 267 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k01S) | B(SizeOp::k10) | B(SizeOp::k10Q) | B(SizeOp::k10S)) 268 }, 269 270 { // kVO_V_Any: 271 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q) | B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k10) | B(SizeOp::k10Q) | B(SizeOp::k11S) | B(SizeOp::k11Q)) 272 }, 273 274 { // kVO_SV_Any: 275 SizeOpTable::kTableAny, SizeOp::kSzQS, uint16_t(B(SizeOp::k00) | B(SizeOp::k00Q) | B(SizeOp::k00S) | 276 B(SizeOp::k01) | B(SizeOp::k01Q) | B(SizeOp::k01S) | 277 B(SizeOp::k10) | B(SizeOp::k10Q) | B(SizeOp::k10S) | 278 B(SizeOp::k11) | B(SizeOp::k11Q) | B(SizeOp::k11S)) 279 } 280 }; 281 282 static const Operand_& significant_simd_op(const Operand_& o0, const Operand_& o1, uint32_t inst_flags) noexcept { 283 return !(inst_flags & InstDB::kInstFlagLong) ? o0 : o1; 284 } 285 286 static inline SizeOp element_type_to_size_op(uint32_t vec_op_type, RegType reg_type, VecElementType element_type) noexcept { 287 // Instruction data or Assembler is wrong if this triggers an assertion failure. 288 ASMJIT_ASSERT(vec_op_type < InstDB::kVO_Count); 289 // ElementType uses 3 bits in the operand signature, it should never overflow. 290 ASMJIT_ASSERT(uint32_t(element_type) <= 0x7u); 291 292 const SizeOpMap& map = size_op_map[vec_op_type]; 293 const SizeOpTable& table = size_op_table[map.table_id]; 294 295 size_t index = (Support::min<uint32_t>(diff(reg_type, RegType::kVec8), diff(RegType::kVec128, RegType::kVec8) + 1) << 3) | uint32_t(element_type); 296 SizeOp op = table.array[index]; 297 SizeOp modified_op { uint8_t(op.value & map.size_op_mask) }; 298 299 if (!Support::bit_test(map.accept_mask, op.value)) { 300 modified_op.make_invalid(); 301 } 302 303 return modified_op; 304 } 305 306 // a64::Assembler - Immediate Encoding Utilities (Integral) 307 // ======================================================== 308 309 using Utils::LogicalImm; 310 311 struct HalfWordImm { 312 uint32_t hw; 313 uint32_t inv; 314 uint32_t imm; 315 }; 316 317 struct LMHImm { 318 uint32_t lm; 319 uint32_t h; 320 uint32_t max_rm_id; 321 }; 322 323 static inline uint32_t count_zero_half_words_64(uint64_t imm) noexcept { 324 return uint32_t((imm & 0x000000000000FFFFu) == 0) + 325 uint32_t((imm & 0x00000000FFFF0000u) == 0) + 326 uint32_t((imm & 0x0000FFFF00000000u) == 0) + 327 uint32_t((imm & 0xFFFF000000000000u) == 0) ; 328 } 329 330 static uint32_t encode_mov_sequence_32(uint32_t out[2], uint32_t imm, uint32_t rd, uint32_t x) noexcept { 331 ASMJIT_ASSERT(rd <= 31); 332 333 uint32_t kMovZ = 0b01010010100000000000000000000000 | (x << 31); 334 uint32_t kMovN = 0b00010010100000000000000000000000; 335 uint32_t kMovK = 0b01110010100000000000000000000000; 336 337 if ((imm & 0xFFFF0000u) == 0x00000000u) { 338 out[0] = kMovZ | (0 << 21) | ((imm & 0xFFFFu) << 5) | rd; 339 return 1; 340 } 341 342 if ((imm & 0xFFFF0000u) == 0xFFFF0000u) { 343 out[0] = kMovN | (0 << 21) | ((~imm & 0xFFFFu) << 5) | rd; 344 return 1; 345 } 346 347 if ((imm & 0x0000FFFFu) == 0x00000000u) { 348 out[0] = kMovZ | (1 << 21) | ((imm >> 16) << 5) | rd; 349 return 1; 350 } 351 352 if ((imm & 0x0000FFFFu) == 0x0000FFFFu) { 353 out[0] = kMovN | (1 << 21) | ((~imm >> 16) << 5) | rd; 354 return 1; 355 } 356 357 out[0] = kMovZ | (0 << 21) | ((imm & 0xFFFFu) << 5) | rd; 358 out[1] = kMovK | (1 << 21) | ((imm >> 16) << 5) | rd; 359 return 2; 360 } 361 362 static uint32_t encode_mov_sequence_64(uint32_t out[4], uint64_t imm, uint32_t rd, uint32_t x) noexcept { 363 ASMJIT_ASSERT(rd <= 31); 364 365 uint32_t kMovZ = 0b11010010100000000000000000000000; 366 uint32_t kMovN = 0b10010010100000000000000000000000; 367 uint32_t kMovK = 0b11110010100000000000000000000000; 368 369 if (imm <= 0xFFFFFFFFu) 370 return encode_mov_sequence_32(out, uint32_t(imm), rd, x); 371 372 uint32_t zhw = count_zero_half_words_64( imm); 373 uint32_t ohw = count_zero_half_words_64(~imm); 374 375 if (zhw >= ohw) { 376 uint32_t op = kMovZ; 377 uint32_t count = 0; 378 379 for (uint32_t hw_index = 0; hw_index < 4; hw_index++, imm >>= 16) { 380 uint32_t hw_imm = uint32_t(imm & 0xFFFFu); 381 if (hw_imm == 0) { 382 continue; 383 } 384 385 out[count++] = op | (hw_index << 21) | (hw_imm << 5) | rd; 386 op = kMovK; 387 } 388 389 // This should not happen - zero should be handled by encode_mov_sequence_32(). 390 ASMJIT_ASSERT(count > 0); 391 392 return count; 393 } 394 else { 395 uint32_t op = kMovN; 396 uint32_t count = 0; 397 uint32_t neg_mask = 0xFFFFu; 398 399 for (uint32_t hw_index = 0; hw_index < 4; hw_index++, imm >>= 16) { 400 uint32_t hw_imm = uint32_t(imm & 0xFFFFu); 401 if (hw_imm == 0xFFFFu) { 402 continue; 403 } 404 405 out[count++] = op | (hw_index << 21) | ((hw_imm ^ neg_mask) << 5) | rd; 406 op = kMovK; 407 neg_mask = 0; 408 } 409 410 if (count == 0) { 411 out[count++] = kMovN | ((0xFFFF ^ neg_mask) << 5) | rd; 412 } 413 414 return count; 415 } 416 } 417 418 static inline bool encode_lmh(uint32_t size_field, uint32_t element_index, Out<LMHImm> out) noexcept { 419 if (size_field != 1 && size_field != 2) 420 return false; 421 422 uint32_t h_shift = 3u - size_field; 423 uint32_t lm_shift = size_field - 1u; 424 uint32_t max_element_index = 15u >> size_field; 425 426 out->h = element_index >> h_shift; 427 out->lm = (element_index << lm_shift) & 0x3u; 428 out->max_rm_id = (8u << size_field) - 1; 429 430 return element_index <= max_element_index; 431 } 432 433 // a64::Assembler - Opcode 434 // ======================= 435 436 //! Helper class to store and manipulate ARM opcode. 437 struct Opcode { 438 uint32_t v; 439 440 enum Bits : uint32_t { 441 kN = (1u << 22), 442 kQ = (1u << 30), 443 kX = (1u << 31) 444 }; 445 446 // -------------------------------------------------------------------------- 447 // [Opcode Builder] 448 // -------------------------------------------------------------------------- 449 450 inline uint32_t get() const noexcept { return v; } 451 inline void reset(uint32_t value) noexcept { v = value; } 452 453 inline bool has_q() const noexcept { return (v & kQ) != 0; } 454 inline bool has_x() const noexcept { return (v & kX) != 0; } 455 456 template<typename T> 457 inline Opcode& add_imm(T value, uint32_t bit_index) noexcept { return operator|=(uint32_t(value) << bit_index); } 458 459 template<typename T> 460 inline Opcode& xor_imm(T value, uint32_t bit_index) noexcept { return operator^=(uint32_t(value) << bit_index); } 461 462 template<typename T, typename Condition> 463 inline Opcode& add_if(T value, const Condition& condition) noexcept { return operator|=(condition ? uint32_t(value) : uint32_t(0)); } 464 465 inline Opcode& add_logical_imm(const LogicalImm& logical_imm) noexcept { 466 add_imm(logical_imm.n, 22); 467 add_imm(logical_imm.r, 16); 468 add_imm(logical_imm.s, 10); 469 return *this; 470 } 471 472 inline Opcode& add_reg(uint32_t id, uint32_t bit_index) noexcept { return operator|=((id & 31u) << bit_index); } 473 inline Opcode& add_reg(const Operand_& op, uint32_t bit_index) noexcept { return add_reg(op.id(), bit_index); } 474 475 inline Opcode& operator=(uint32_t x) noexcept { v = x; return *this; } 476 inline Opcode& operator&=(uint32_t x) noexcept { v &= x; return *this; } 477 inline Opcode& operator|=(uint32_t x) noexcept { v |= x; return *this; } 478 inline Opcode& operator^=(uint32_t x) noexcept { v ^= x; return *this; } 479 480 inline uint32_t operator&(uint32_t x) const noexcept { return v & x; } 481 inline uint32_t operator|(uint32_t x) const noexcept { return v | x; } 482 inline uint32_t operator^(uint32_t x) const noexcept { return v ^ x; } 483 }; 484 485 // a64::Assembler - Signature Utilities 486 // ==================================== 487 488 // TODO: [ARM] Deprecate match_signature. 489 static inline bool match_signature(const Operand_& o0, const Operand_& o1, uint32_t inst_flags) noexcept { 490 if (!(inst_flags & (InstDB::kInstFlagLong | InstDB::kInstFlagNarrow))) 491 return o0.signature() == o1.signature(); 492 493 // TODO: [ARM] Something smart to validate this. 494 return true; 495 } 496 497 static inline bool match_signature(const Operand_& o0, const Operand_& o1, const Operand_& o2, uint32_t inst_flags) noexcept { 498 return match_signature(o0, o1, inst_flags) && o1.signature() == o2.signature(); 499 } 500 501 static inline bool match_signature(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3, uint32_t inst_flags) noexcept { 502 return match_signature(o0, o1, inst_flags) && o1.signature() == o2.signature() && o2.signature() == o3.signature(); 503 } 504 505 // Memory must be either: 506 // 1. Absolute address, which will be converted to relative. 507 // 2. Relative displacement (Label). 508 // 3. Base register + either offset or index. 509 static inline bool check_mem_base_index_rel(const Mem& mem) noexcept { 510 // Allowed base types (Nothing, Label, and Gp64). 511 constexpr uint32_t kBaseMask = B(0) | B(RegType::kLabelTag) | B(RegType::kGp64); 512 // Allowed index types (Nothing, Gp32, and Gp64). 513 constexpr uint32_t kIndexMask = B(0) | B(RegType::kGp32) | B(RegType::kGp64) ; 514 515 RegType base_type = mem.base_type(); 516 RegType index_type = mem.index_type(); 517 518 if (!Support::bit_test(kBaseMask, base_type)) { 519 return false; 520 } 521 522 if (base_type > RegType::kLabelTag) { 523 // Index allows either Gp32 or Gp64. 524 if (!Support::bit_test(kIndexMask, index_type)) { 525 return false; 526 } 527 528 if (index_type == RegType::kNone) { 529 return true; 530 } 531 else { 532 return !mem.has_offset(); 533 } 534 } 535 else { 536 // No index register allowed if this is a PC relative address (literal). 537 return index_type == RegType::kNone; 538 } 539 } 540 541 struct EncodeFpOpcodeBits { 542 uint32_t size_mask; 543 uint32_t mask[3]; 544 }; 545 546 static inline bool pick_fp_opcode(const Vec& reg, uint32_t s_op, uint32_t s_hf, uint32_t v_op, uint32_t v_hf, Opcode* opcode, uint32_t* sz_out) noexcept { 547 static constexpr uint32_t kQBitIndex = 30; 548 549 static const EncodeFpOpcodeBits sz_bits_table[InstDB::kHF_Count] = { 550 { B(2) | B(1) , { 0u , 0u, B(22) } }, 551 { B(2) | B(1) | B(0), { 0u , 0u, 0u } }, 552 { B(2) | B(1) | B(0), { B(23) | B(22) , 0u, B(22) } }, 553 { B(2) | B(1) | B(0), { B(22) | B(20) | B(19) , 0u, B(22) } }, 554 { B(2) | B(1) | B(0), { B(22) | B(21) | B(15) | B(14), 0u, B(22) } }, 555 { B(2) | B(1) | B(0), { B(23) , 0u, B(22) } } 556 }; 557 558 if (!reg.has_element_type()) { 559 // Scalar operation [HSD]. 560 uint32_t sz = diff(reg.reg_type(), RegType::kVec16); 561 if (sz > 2u || !Support::bit_test(sz_bits_table[s_hf].size_mask, sz)) { 562 return false; 563 } 564 565 opcode->reset(sz_bits_table[s_hf].mask[sz] ^ s_op); 566 *sz_out = sz; 567 return s_op != 0; 568 } 569 else { 570 // Vector operation [HSD]. 571 uint32_t q = diff(reg.reg_type(), RegType::kVec64); 572 uint32_t sz = diff(reg.element_type(), VecElementType::kH); 573 574 if (q > 1u || sz > 2u || !Support::bit_test(sz_bits_table[v_hf].size_mask, sz)) { 575 return false; 576 } 577 578 opcode->reset(sz_bits_table[v_hf].mask[sz] ^ (v_op | (q << kQBitIndex))); 579 *sz_out = sz; 580 return v_op != 0; 581 } 582 } 583 584 static inline bool pick_fp_opcode(const Vec& reg, uint32_t s_op, uint32_t s_hf, uint32_t v_op, uint32_t v_hf, Opcode* opcode) noexcept { 585 uint32_t sz; 586 return pick_fp_opcode(reg, s_op, s_hf, v_op, v_hf, opcode, &sz); 587 } 588 589 // a64::Assembler - Operand Checks 590 // =============================== 591 592 // Checks whether all operands have the same signature. 593 static inline bool check_signature(const Operand_& o0, const Operand_& o1) noexcept { 594 return o0.signature() == o1.signature(); 595 } 596 597 static inline bool check_signature(const Operand_& o0, const Operand_& o1, const Operand_& o2) noexcept { 598 return o0.signature() == o1.signature() && 599 o1.signature() == o2.signature(); 600 } 601 602 static inline bool check_signature(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) noexcept { 603 return o0.signature() == o1.signature() && 604 o1.signature() == o2.signature() && 605 o2.signature() == o3.signature(); 606 } 607 608 // Checks whether the register is GP register of the allowed types. 609 // 610 // Allowed is a 2-bit mask, where the first bits allows Gp32 and the second bit allows Gp64. These bits are usually 611 // stored within the instruction, but could be also hardcoded in the assembler for instructions where GP types are 612 // not selectable. 613 static inline bool check_gp_type(const Operand_& op, uint32_t allowed) noexcept { 614 RegType type = op.as<Reg>().reg_type(); 615 return Support::bit_test(allowed << uint32_t(RegType::kGp32), type); 616 } 617 618 static inline bool check_gp_type(const Operand_& op, uint32_t allowed, uint32_t* x) noexcept { 619 // NOTE: We set 'x' to one only when Gp32 is allowed, otherwise the X is part 620 // of the opcode and we cannot set it. This is why this works without requiring 621 // additional logic. 622 RegType type = op.as<Reg>().reg_type(); 623 *x = diff(type, RegType::kGp32) & allowed; 624 return Support::bit_test(allowed << uint32_t(RegType::kGp32), type); 625 } 626 627 static inline bool check_gp_type(const Operand_& o0, const Operand_& o1, uint32_t allowed, uint32_t* x) noexcept { 628 return check_gp_type(o0, allowed, x) && check_signature(o0, o1); 629 } 630 631 static inline bool check_gp_type(const Operand_& o0, const Operand_& o1, const Operand_& o2, uint32_t allowed, uint32_t* x) noexcept { 632 return check_gp_type(o0, allowed, x) && check_signature(o0, o1, o2); 633 } 634 635 static inline bool check_gp_id(const Operand_& op, uint32_t hi_id = kZR) noexcept { 636 uint32_t id = op.as<Reg>().id(); 637 return id < 31u || id == hi_id; 638 } 639 640 static inline bool check_gp_id(const Operand_& o0, const Operand_& o1, uint32_t hi_id = kZR) noexcept { 641 uint32_t id0 = o0.as<Reg>().id(); 642 uint32_t id1 = o1.as<Reg>().id(); 643 644 return (id0 < 31u || id0 == hi_id) && (id1 < 31u || id1 == hi_id); 645 } 646 647 static inline bool check_gp_id(const Operand_& o0, const Operand_& o1, const Operand_& o2, uint32_t hi_id = kZR) noexcept { 648 uint32_t id0 = o0.as<Reg>().id(); 649 uint32_t id1 = o1.as<Reg>().id(); 650 uint32_t id2 = o2.as<Reg>().id(); 651 652 return (id0 < 31u || id0 == hi_id) && (id1 < 31u || id1 == hi_id) && (id2 < 31u || id2 == hi_id); 653 } 654 655 static inline bool check_vec_id(const Operand_& op) noexcept { 656 uint32_t id = op.as<Reg>().id(); 657 return id <= 31u; 658 } 659 660 static inline bool check_vec_id(const Operand_& o0, const Operand_& o1) noexcept { 661 uint32_t id0 = o0.as<Reg>().id(); 662 uint32_t id1 = o1.as<Reg>().id(); 663 664 return (id0 | id1) <= 31u; 665 } 666 667 /* Unused at the moment. 668 static inline bool check_vec_id(const Operand_& o0, const Operand_& o1, const Operand_& o2) noexcept { 669 uint32_t id0 = o0.as<Reg>().id(); 670 uint32_t id1 = o1.as<Reg>().id(); 671 uint32_t id2 = o2.as<Reg>().id(); 672 673 return (id0 | id1 | id2) <= 31u; 674 } 675 676 static inline bool check_vec_id(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) noexcept { 677 uint32_t id0 = o0.as<Reg>().id(); 678 uint32_t id1 = o1.as<Reg>().id(); 679 uint32_t id2 = o2.as<Reg>().id(); 680 uint32_t id3 = o3.as<Reg>().id(); 681 682 return (id0 | id1 | id2 | id3) <= 31u; 683 } 684 */ 685 686 static inline bool check_mem_base(const Mem& mem) noexcept { 687 return mem.base_type() == RegType::kGp64 && mem.base_id() <= 31; 688 } 689 690 static inline bool check_even(const Operand_& o0, const Operand_& o1) noexcept { 691 return ((o0.id() | o1.id()) & 1) == 0; 692 } 693 694 static inline bool check_consecutive(const Operand_& o0, const Operand_& o1) noexcept { 695 return ((o0.id() + 1u) & 0x1Fu) == o1.id(); 696 } 697 698 static inline bool check_consecutive(const Operand_& o0, const Operand_& o1, const Operand_& o2) noexcept { 699 return ((o0.id() + 1u) & 0x1Fu) == o1.id() && 700 ((o0.id() + 2u) & 0x1Fu) == o2.id(); 701 } 702 703 static inline bool check_consecutive(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) noexcept { 704 return ((o0.id() + 1u) & 0x1Fu) == o1.id() && 705 ((o0.id() + 2u) & 0x1Fu) == o2.id() && 706 ((o0.id() + 3u) & 0x1Fu) == o3.id(); 707 } 708 709 // a64::Assembler - CheckReg 710 // ========================= 711 712 #define V(index) (index == uint32_t(RegType::kGp32) ? Gp::kIdZr : \ 713 index == uint32_t(RegType::kGp64) ? Gp::kIdZr : \ 714 index == uint32_t(RegType::kVec8) ? 31u : \ 715 index == uint32_t(RegType::kVec16) ? 31u : \ 716 index == uint32_t(RegType::kVec32) ? 31u : \ 717 index == uint32_t(RegType::kVec64) ? 31u : \ 718 index == uint32_t(RegType::kVec128) ? 31u : 0) 719 static const Support::Array<uint8_t, 32> common_hi_reg_id_of_type_table = {{ 720 ASMJIT_LOOKUP_TABLE_32(V, 0) 721 }}; 722 #undef V 723 724 static inline bool check_valid_regs(const Operand_& o0) noexcept { 725 return bool(unsigned(o0.id() < 31) | unsigned(o0.id() == common_hi_reg_id_of_type_table[o0.as<Reg>().reg_type()])); 726 } 727 728 static inline bool check_valid_regs(const Operand_& o0, const Operand_& o1) noexcept { 729 return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == common_hi_reg_id_of_type_table[o0.as<Reg>().reg_type()])) & 730 (unsigned(o1.id() < 31) | unsigned(o1.id() == common_hi_reg_id_of_type_table[o1.as<Reg>().reg_type()]))); 731 } 732 733 static inline bool check_valid_regs(const Operand_& o0, const Operand_& o1, const Operand_& o2) noexcept { 734 return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == common_hi_reg_id_of_type_table[o0.as<Reg>().reg_type()])) & 735 (unsigned(o1.id() < 31) | unsigned(o1.id() == common_hi_reg_id_of_type_table[o1.as<Reg>().reg_type()])) & 736 (unsigned(o2.id() < 31) | unsigned(o2.id() == common_hi_reg_id_of_type_table[o2.as<Reg>().reg_type()]))); 737 } 738 739 static inline bool check_valid_regs(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) noexcept { 740 return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == common_hi_reg_id_of_type_table[o0.as<Reg>().reg_type()])) & 741 (unsigned(o1.id() < 31) | unsigned(o1.id() == common_hi_reg_id_of_type_table[o1.as<Reg>().reg_type()])) & 742 (unsigned(o2.id() < 31) | unsigned(o2.id() == common_hi_reg_id_of_type_table[o2.as<Reg>().reg_type()])) & 743 (unsigned(o3.id() < 31) | unsigned(o3.id() == common_hi_reg_id_of_type_table[o3.as<Reg>().reg_type()]))); 744 } 745 746 // a64::Assembler - Construction & Destruction 747 // =========================================== 748 749 Assembler::Assembler(CodeHolder* code) noexcept : BaseAssembler() { 750 _arch_mask = uint64_t(1) << uint32_t(Arch::kAArch64); 751 init_emitter_funcs(this); 752 753 if (code) { 754 code->attach(this); 755 } 756 } 757 758 Assembler::~Assembler() noexcept {} 759 760 // a64::Assembler - Emit 761 // ===================== 762 763 #define ENC_OPS1(OP0) \ 764 (uint32_t(OperandType::k##OP0)) 765 766 #define ENC_OPS2(OP0, OP1) \ 767 (uint32_t(OperandType::k##OP0) + \ 768 (uint32_t(OperandType::k##OP1) << 3)) 769 770 #define ENC_OPS3(OP0, OP1, OP2) \ 771 (uint32_t(OperandType::k##OP0) + \ 772 (uint32_t(OperandType::k##OP1) << 3) + \ 773 (uint32_t(OperandType::k##OP2) << 6)) 774 775 #define ENC_OPS4(OP0, OP1, OP2, OP3) \ 776 (uint32_t(OperandType::k##OP0) + \ 777 (uint32_t(OperandType::k##OP1) << 3) + \ 778 (uint32_t(OperandType::k##OP2) << 6) + \ 779 (uint32_t(OperandType::k##OP3) << 9)) 780 781 Error Assembler::_emit(InstId inst_id, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* op_ext) { 782 // Logging/Validation/Error. 783 constexpr InstOptions kRequiresSpecialHandling = InstOptions::kReserved; 784 785 Error err; 786 CodeWriter writer(this); 787 788 // Combine all instruction options and also check whether the instruction 789 // is valid. All options that require special handling (including invalid 790 // instruction) are handled by the next branch. 791 InstOptions options = InstOptions(inst_id - 1 >= Inst::_kIdCount - 1) | InstOptions((size_t)(_buffer_end - writer.cursor()) < 4) | inst_options() | forced_inst_options(); 792 793 CondCode inst_cc = BaseInst::extract_arm_cond_code(inst_id); 794 inst_id = inst_id & uint32_t(InstIdParts::kRealId); 795 796 if (inst_id >= Inst::_kIdCount) { 797 inst_id = 0; 798 } 799 800 const InstDB::InstInfo* inst_info = &InstDB::_inst_info_table[inst_id]; 801 uint32_t encoding_index = inst_info->_encoding_data_index; 802 803 Opcode opcode; 804 uint32_t isign4; 805 uint32_t inst_flags; 806 807 const Operand_& o3 = op_ext[EmitterUtils::kOp3]; 808 const Operand_* rm_rel = nullptr; 809 810 uint32_t multiple_op_data[4]; 811 uint32_t multiple_op_count; 812 813 // These are only used when instruction uses a relative displacement. 814 OffsetFormat offset_format; // Offset format. 815 uint64_t offset_value; // Offset value (if known). 816 817 if (ASMJIT_UNLIKELY(Support::test(options, kRequiresSpecialHandling))) { 818 if (ASMJIT_UNLIKELY(!_code)) { 819 return report_error(make_error(Error::kNotInitialized)); 820 } 821 822 // Unknown instruction. 823 if (ASMJIT_UNLIKELY(inst_id == 0)) { 824 goto InvalidInstruction; 825 } 826 827 // Condition code can only be used with 'B' instruction. 828 if (ASMJIT_UNLIKELY(inst_cc != CondCode::kAL && inst_id != Inst::kIdB)) { 829 goto InvalidInstruction; 830 } 831 832 // Grow request, happens rarely. 833 err = writer.ensure_space(this, 4); 834 if (ASMJIT_UNLIKELY(err != Error::kOk)) { 835 goto Failed; 836 } 837 838 #ifndef ASMJIT_NO_VALIDATION 839 // Strict validation. 840 if (has_diagnostic_option(DiagnosticOptions::kValidateAssembler)) { 841 Operand_ op_array[Globals::kMaxOpCount]; 842 EmitterUtils::op_array_from_emit_args(op_array, o0, o1, o2, op_ext); 843 844 err = _funcs.validate(BaseInst(inst_id, options, _extra_reg), op_array, Globals::kMaxOpCount, ValidationFlags::kNone); 845 if (ASMJIT_UNLIKELY(err != Error::kOk)) { 846 goto Failed; 847 } 848 } 849 #endif 850 } 851 852 // Signature of the first 4 operands. 853 isign4 = (uint32_t(o0.op_type()) ) + 854 (uint32_t(o1.op_type()) << 3) + 855 (uint32_t(o2.op_type()) << 6) + 856 (uint32_t(o3.op_type()) << 9); 857 inst_flags = inst_info->flags(); 858 859 switch (inst_info->_encoding) { 860 // ------------------------------------------------------------------------ 861 // [Base - Universal] 862 // ------------------------------------------------------------------------ 863 864 case InstDB::kEncodingBaseOp: { 865 const InstDB::EncodingData::BaseOp& op_data = InstDB::EncodingData::baseOp[encoding_index]; 866 867 if (isign4 == 0) { 868 opcode.reset(op_data.opcode); 869 goto EmitOp; 870 } 871 872 break; 873 } 874 875 case InstDB::kEncodingBaseOpX16: { 876 const InstDB::EncodingData::BaseOpX16& op_data = InstDB::EncodingData::baseOpX16[encoding_index]; 877 878 if (isign4 == ENC_OPS1(Reg) && o0.as<Reg>().is_gp64(16)) { 879 opcode.reset(op_data.opcode); 880 goto EmitOp; 881 } 882 883 break; 884 } 885 886 case InstDB::kEncodingBaseOpImm: { 887 const InstDB::EncodingData::BaseOpImm& op_data = InstDB::EncodingData::baseOpImm[encoding_index]; 888 889 if (isign4 == ENC_OPS1(Imm)) { 890 uint64_t imm = o0.as<Imm>().value_as<uint64_t>(); 891 uint32_t immMax = 1u << op_data.imm_bits; 892 893 if (imm >= immMax) 894 goto InvalidImmediate; 895 896 opcode.reset(op_data.opcode); 897 opcode.add_imm(imm, op_data.imm_offset); 898 goto EmitOp; 899 } 900 901 break; 902 } 903 904 case InstDB::kEncodingBaseR: { 905 const InstDB::EncodingData::BaseR& op_data = InstDB::EncodingData::baseR[encoding_index]; 906 907 if (isign4 == ENC_OPS1(Reg)) { 908 if (!check_gp_type(o0, op_data.reg_type)) 909 goto InvalidInstruction; 910 911 if (!check_gp_id(o0, op_data.reg_hi_id)) 912 goto InvalidPhysId; 913 914 opcode.reset(op_data.opcode); 915 opcode.add_reg(o0, op_data.r_shift); 916 goto EmitOp; 917 } 918 919 break; 920 } 921 922 case InstDB::kEncodingBaseRR: { 923 const InstDB::EncodingData::BaseRR& op_data = InstDB::EncodingData::baseRR[encoding_index]; 924 925 if (isign4 == ENC_OPS2(Reg, Reg)) { 926 uint32_t x; 927 if (!check_gp_type(o0, op_data.a_type, &x)) 928 goto InvalidInstruction; 929 930 if (!check_gp_type(o1, op_data.b_type)) 931 goto InvalidInstruction; 932 933 if (op_data.uniform && !check_signature(o0, o1)) 934 goto InvalidInstruction; 935 936 if (!check_gp_id(o0, op_data.a_hi_id)) 937 goto InvalidPhysId; 938 939 if (!check_gp_id(o1, op_data.b_hi_id)) 940 goto InvalidPhysId; 941 942 opcode.reset(op_data.opcode); 943 opcode.add_imm(x, 31); 944 opcode.add_reg(o1, op_data.b_shift); 945 opcode.add_reg(o0, op_data.a_shift); 946 goto EmitOp; 947 } 948 949 break; 950 } 951 952 case InstDB::kEncodingBaseRRR: { 953 const InstDB::EncodingData::BaseRRR& op_data = InstDB::EncodingData::baseRRR[encoding_index]; 954 955 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 956 uint32_t x; 957 if (!check_gp_type(o0, op_data.a_type, &x)) 958 goto InvalidInstruction; 959 960 if (!check_gp_type(o1, op_data.b_type)) 961 goto InvalidInstruction; 962 963 if (!check_gp_type(o2, op_data.c_type)) 964 goto InvalidInstruction; 965 966 if (op_data.uniform && !check_signature(o0, o1, o2)) 967 goto InvalidInstruction; 968 969 if (!check_gp_id(o0, op_data.a_hi_id)) 970 goto InvalidPhysId; 971 972 if (!check_gp_id(o1, op_data.b_hi_id)) 973 goto InvalidPhysId; 974 975 if (!check_gp_id(o2, op_data.c_hi_id)) 976 goto InvalidPhysId; 977 978 opcode.reset(op_data.opcode()); 979 opcode.add_imm(x, 31); 980 opcode.add_reg(o2, 16); 981 opcode.add_reg(o1, 5); 982 opcode.add_reg(o0, 0); 983 goto EmitOp; 984 } 985 986 break; 987 } 988 989 case InstDB::kEncodingBaseRRRR: { 990 const InstDB::EncodingData::BaseRRRR& op_data = InstDB::EncodingData::baseRRRR[encoding_index]; 991 992 if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) { 993 uint32_t x; 994 if (!check_gp_type(o0, op_data.a_type, &x)) 995 goto InvalidInstruction; 996 997 if (!check_gp_type(o1, op_data.b_type)) 998 goto InvalidInstruction; 999 1000 if (!check_gp_type(o2, op_data.c_type)) 1001 goto InvalidInstruction; 1002 1003 if (!check_gp_type(o3, op_data.d_type)) 1004 goto InvalidInstruction; 1005 1006 if (op_data.uniform && !check_signature(o0, o1, o2, o3)) 1007 goto InvalidInstruction; 1008 1009 if (!check_gp_id(o0, op_data.a_hi_id)) 1010 goto InvalidPhysId; 1011 1012 if (!check_gp_id(o1, op_data.b_hi_id)) 1013 goto InvalidPhysId; 1014 1015 if (!check_gp_id(o2, op_data.c_hi_id)) 1016 goto InvalidPhysId; 1017 1018 if (!check_gp_id(o3, op_data.d_hi_id)) 1019 goto InvalidPhysId; 1020 1021 opcode.reset(op_data.opcode()); 1022 opcode.add_imm(x, 31); 1023 opcode.add_reg(o2, 16); 1024 opcode.add_reg(o3, 10); 1025 opcode.add_reg(o1, 5); 1026 opcode.add_reg(o0, 0); 1027 goto EmitOp; 1028 } 1029 1030 break; 1031 } 1032 1033 case InstDB::kEncodingBaseRRII: { 1034 const InstDB::EncodingData::BaseRRII& op_data = InstDB::EncodingData::baseRRII[encoding_index]; 1035 1036 if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) { 1037 if (!check_gp_type(o0, op_data.a_type)) 1038 goto InvalidInstruction; 1039 1040 if (!check_gp_type(o1, op_data.b_type)) 1041 goto InvalidInstruction; 1042 1043 if (!check_gp_id(o0, op_data.a_hi_id)) 1044 goto InvalidPhysId; 1045 1046 if (!check_gp_id(o1, op_data.b_hi_id)) 1047 goto InvalidPhysId; 1048 1049 if (o2.as<Imm>().value_as<uint64_t>() >= Support::bit_mask<uint32_t>(op_data.a_imm_size + op_data.a_imm_discard_lsb) || 1050 o3.as<Imm>().value_as<uint64_t>() >= Support::bit_mask<uint32_t>(op_data.b_imm_size + op_data.b_imm_discard_lsb)) 1051 goto InvalidImmediate; 1052 1053 uint32_t a_imm = o2.as<Imm>().value_as<uint32_t>() >> op_data.a_imm_discard_lsb; 1054 uint32_t b_imm = o3.as<Imm>().value_as<uint32_t>() >> op_data.b_imm_discard_lsb; 1055 1056 if ((a_imm << op_data.a_imm_discard_lsb) != o2.as<Imm>().value_as<uint32_t>() || 1057 (b_imm << op_data.b_imm_discard_lsb) != o3.as<Imm>().value_as<uint32_t>()) 1058 goto InvalidImmediate; 1059 1060 opcode.reset(op_data.opcode()); 1061 opcode.add_imm(a_imm, op_data.a_imm_offset); 1062 opcode.add_imm(b_imm, op_data.b_imm_offset); 1063 opcode.add_reg(o1, 5); 1064 opcode.add_reg(o0, 0); 1065 goto EmitOp; 1066 } 1067 1068 break; 1069 } 1070 1071 // ------------------------------------------------------------------------ 1072 // [Base - Mov] 1073 // ------------------------------------------------------------------------ 1074 1075 case InstDB::kEncodingBaseMov: { 1076 // MOV is a pseudo instruction that uses various instructions depending on its signature. 1077 uint32_t x = diff(o0.as<Reg>().reg_type(), RegType::kGp32); 1078 if (x > 1) 1079 goto InvalidInstruction; 1080 1081 if (isign4 == ENC_OPS2(Reg, Reg)) { 1082 if (!o0.as<Reg>().is_gp()) 1083 goto InvalidInstruction; 1084 1085 if (!check_signature(o0, o1)) 1086 goto InvalidInstruction; 1087 1088 bool has_sp = o0.as<Gp>().is_sp() || o1.as<Gp>().is_sp(); 1089 if (has_sp) { 1090 // Cannot be combined with ZR. 1091 if (!check_gp_id(o0, o1, kSP)) 1092 goto InvalidPhysId; 1093 1094 // MOV Rd, Rm -> ADD Rd, Rn, #0. 1095 opcode.reset(0b00010001000000000000000000000000); 1096 opcode.add_imm(x, 31); 1097 opcode.add_reg(o1, 5); 1098 opcode.add_reg(o0, 0); 1099 goto EmitOp; 1100 } 1101 else { 1102 if (!check_gp_id(o0, o1, kZR)) 1103 goto InvalidPhysId; 1104 1105 // MOV Rd, Rm -> ORR Rd, <ZR>, Rm. 1106 opcode.reset(0b00101010000000000000001111100000); 1107 opcode.add_imm(x, 31); 1108 opcode.add_reg(o1, 16); 1109 opcode.add_reg(o0, 0); 1110 goto EmitOp; 1111 } 1112 } 1113 1114 if (isign4 == ENC_OPS2(Reg, Imm)) { 1115 if (!o0.as<Reg>().is_gp()) 1116 goto InvalidInstruction; 1117 1118 uint64_t imm_value = o1.as<Imm>().value_as<uint64_t>(); 1119 if (!x) 1120 imm_value &= 0xFFFFFFFFu; 1121 1122 // Prefer a single MOVN/MOVZ instruction over a logical instruction. 1123 multiple_op_count = encode_mov_sequence_64(multiple_op_data, imm_value, o0.id() & 31, x); 1124 if (multiple_op_count == 1 && !o0.as<Gp>().is_sp()) { 1125 opcode.reset(multiple_op_data[0]); 1126 goto EmitOp; 1127 } 1128 1129 // Logical instructions use 13-bit immediate pattern encoded as N:ImmR:ImmS. 1130 LogicalImm logical_imm; 1131 if (!o0.as<Gp>().is_zr()) { 1132 if (Utils::encode_logical_imm(imm_value, x ? 64 : 32, Out(logical_imm))) { 1133 if (!check_gp_id(o0, kSP)) 1134 goto InvalidPhysId; 1135 1136 opcode.reset(0b00110010000000000000001111100000); 1137 opcode.add_imm(x, 31); 1138 opcode.add_logical_imm(logical_imm); 1139 opcode.add_reg(o0, 0); 1140 goto EmitOp; 1141 } 1142 } 1143 1144 if (!check_gp_id(o0, kZR)) 1145 goto InvalidPhysId; 1146 1147 goto EmitOp_Multiple; 1148 } 1149 1150 break; 1151 } 1152 1153 case InstDB::kEncodingBaseMovKNZ: { 1154 const InstDB::EncodingData::BaseMovKNZ& op_data = InstDB::EncodingData::baseMovKNZ[encoding_index]; 1155 1156 uint32_t x = diff(o0.as<Reg>().reg_type(), RegType::kGp32); 1157 if (x > 1) 1158 goto InvalidInstruction; 1159 1160 if (!check_gp_id(o0, kZR)) 1161 goto InvalidPhysId; 1162 1163 opcode.reset(op_data.opcode); 1164 opcode.add_imm(x, 31); 1165 1166 if (isign4 == ENC_OPS2(Reg, Imm)) { 1167 uint64_t imm16 = o1.as<Imm>().value_as<uint64_t>(); 1168 if (imm16 > 0xFFFFu) 1169 goto InvalidImmediate; 1170 1171 opcode.add_imm(imm16, 5); 1172 opcode.add_reg(o0, 0); 1173 goto EmitOp; 1174 } 1175 1176 if (isign4 == ENC_OPS3(Reg, Imm, Imm)) { 1177 uint64_t imm16 = o1.as<Imm>().value_as<uint64_t>(); 1178 uint32_t shift_type = o2.as<Imm>().predicate(); 1179 uint64_t shiftValue = o2.as<Imm>().value_as<uint64_t>(); 1180 1181 if (imm16 > 0xFFFFu || shiftValue > 48 || shift_type != uint32_t(ShiftOp::kLSL)) 1182 goto InvalidImmediate; 1183 1184 // Convert shift value to 'hw' field. 1185 uint32_t hw = uint32_t(shiftValue) >> 4; 1186 if ((hw << 4) != uint32_t(shiftValue)) 1187 goto InvalidImmediate; 1188 1189 opcode.add_imm(hw, 21); 1190 opcode.add_imm(imm16, 5); 1191 opcode.add_reg(o0, 0); 1192 1193 if (!x && hw > 1u) 1194 goto InvalidImmediate; 1195 1196 goto EmitOp; 1197 } 1198 1199 break; 1200 } 1201 1202 // ------------------------------------------------------------------------ 1203 // [Base - Adr] 1204 // ------------------------------------------------------------------------ 1205 1206 case InstDB::kEncodingBaseAdr: { 1207 const InstDB::EncodingData::BaseAdr& op_data = InstDB::EncodingData::baseAdr[encoding_index]; 1208 1209 if (isign4 == ENC_OPS2(Reg, Label) || isign4 == ENC_OPS2(Reg, Imm)) { 1210 if (!o0.as<Reg>().is_gp64()) 1211 goto InvalidInstruction; 1212 1213 if (!check_gp_id(o0, kZR)) 1214 goto InvalidPhysId; 1215 1216 opcode.reset(op_data.opcode()); 1217 opcode.add_reg(o0, 0); 1218 offset_format.reset_to_imm_value(op_data.offset_type, 4, 5, 21, 0); 1219 1220 if (inst_id == Inst::kIdAdrp) 1221 offset_format._imm_discard_lsb = 12; 1222 1223 rm_rel = &o1; 1224 goto EmitOp_Rel; 1225 } 1226 1227 break; 1228 } 1229 1230 // ------------------------------------------------------------------------ 1231 // [Base - Arithmetic and Logical] 1232 // ------------------------------------------------------------------------ 1233 1234 case InstDB::kEncodingBaseAddSub: { 1235 const InstDB::EncodingData::BaseAddSub& op_data = InstDB::EncodingData::baseAddSub[encoding_index]; 1236 1237 uint32_t x; 1238 if (!check_gp_type(o0, o1, kWX, &x)) 1239 goto InvalidInstruction; 1240 1241 if (isign4 == ENC_OPS3(Reg, Reg, Imm) || isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) { 1242 opcode.reset(uint32_t(op_data.immediate_op) << 24); 1243 1244 // ADD | SUB (immediate) - ZR is not allowed. 1245 // ADDS|SUBS (immediate) - ZR allowed in Rd, SP allowed in Rn. 1246 uint32_t a_hi_id = opcode.get() & B(29) ? kZR : kSP; 1247 uint32_t b_hi_id = kSP; 1248 1249 if (!check_gp_id(o0, a_hi_id) || !check_gp_id(o1, b_hi_id)) 1250 goto InvalidPhysId; 1251 1252 // ADD|SUB (immediate) use 12-bit immediate optionally shifted by 'LSL #12'. 1253 uint64_t imm = o2.as<Imm>().value_as<uint64_t>(); 1254 uint32_t shift = 0; 1255 1256 if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) { 1257 if (o3.as<Imm>().predicate() != uint32_t(ShiftOp::kLSL)) 1258 goto InvalidImmediate; 1259 1260 if (o3.as<Imm>().value() != 0 && o3.as<Imm>().value() != 12) 1261 goto InvalidImmediate; 1262 1263 shift = uint32_t(o3.as<Imm>().value() != 0); 1264 } 1265 1266 // Accept immediate value of '0x00XXX000' by setting 'shift' to 12. 1267 if (imm > 0xFFFu) { 1268 if (shift || (imm & ~uint64_t(0xFFFu << 12)) != 0) 1269 goto InvalidImmediate; 1270 shift = 1; 1271 imm >>= 12; 1272 } 1273 1274 opcode.add_imm(x, 31); 1275 opcode.add_imm(shift, 22); 1276 opcode.add_imm(imm, 10); 1277 opcode.add_reg(o1, 5); 1278 opcode.add_reg(o0, 0); 1279 goto EmitOp; 1280 } 1281 1282 if (isign4 == ENC_OPS3(Reg, Reg, Reg) || isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) { 1283 uint32_t op_size = x ? 64 : 32; 1284 uint64_t shift = 0; 1285 uint32_t shift_type = uint32_t(ShiftOp::kLSL); 1286 1287 if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) { 1288 shift_type = o3.as<Imm>().predicate(); 1289 shift = o3.as<Imm>().value_as<uint64_t>(); 1290 } 1291 1292 if (!check_gp_id(o2, kZR)) 1293 goto InvalidPhysId; 1294 1295 // Shift operation - LSL, LSR, ASR. 1296 if (shift_type <= uint32_t(ShiftOp::kASR)) { 1297 bool has_sp = o0.as<Gp>().is_sp() || o1.as<Gp>().is_sp(); 1298 if (!has_sp) { 1299 if (!check_signature(o1, o2)) { 1300 goto InvalidInstruction; 1301 } 1302 1303 if (!check_gp_id(o0, o1, kZR)) { 1304 goto InvalidPhysId; 1305 } 1306 1307 if (shift >= op_size) { 1308 goto InvalidImmediate; 1309 } 1310 1311 opcode.reset(uint32_t(op_data.shifted_op) << 21); 1312 opcode.add_imm(x, 31); 1313 opcode.add_imm(shift_type, 22); 1314 opcode.add_reg(o2, 16); 1315 opcode.add_imm(shift, 10); 1316 opcode.add_reg(o1, 5); 1317 opcode.add_reg(o0, 0); 1318 goto EmitOp; 1319 } 1320 1321 // SP register can only be used with LSL or Extend. 1322 if (shift_type != uint32_t(ShiftOp::kLSL)) { 1323 goto InvalidImmediate; 1324 } 1325 1326 shift_type = x ? uint32_t(ShiftOp::kUXTX) : uint32_t(ShiftOp::kUXTW); 1327 } 1328 1329 // Extend operation - UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX. 1330 opcode.reset(uint32_t(op_data.extended_op) << 21); 1331 shift_type -= uint32_t(ShiftOp::kUXTB); 1332 1333 if (shift_type > 7 || shift > 4) { 1334 goto InvalidImmediate; 1335 } 1336 1337 if (!(opcode.get() & B(29))) { 1338 // ADD|SUB (extend) - ZR is not allowed. 1339 if (!check_gp_id(o0, o1, kSP)) 1340 goto InvalidPhysId; 1341 } 1342 else { 1343 // ADDS|SUBS (extend) - ZR allowed in Rd, SP allowed in Rn. 1344 if (!check_gp_id(o0, kZR) || !check_gp_id(o1, kSP)) 1345 goto InvalidPhysId; 1346 } 1347 1348 // Validate whether the register operands match extend option. 1349 if (o2.as<Reg>().reg_type() != extend_option_to_reg_type(shift_type) || o1.as<Reg>().reg_type() < o2.as<Reg>().reg_type()) { 1350 goto InvalidInstruction; 1351 } 1352 1353 opcode.add_imm(x, 31); 1354 opcode.add_reg(o2, 16); 1355 opcode.add_imm(shift_type, 13); 1356 opcode.add_imm(shift, 10); 1357 opcode.add_reg(o1, 5); 1358 opcode.add_reg(o0, 0); 1359 goto EmitOp; 1360 } 1361 1362 break; 1363 } 1364 1365 case InstDB::kEncodingBaseLogical: { 1366 const InstDB::EncodingData::BaseLogical& op_data = InstDB::EncodingData::baseLogical[encoding_index]; 1367 1368 uint32_t x; 1369 if (!check_gp_type(o0, o1, kWX, &x)) 1370 goto InvalidInstruction; 1371 1372 if (!check_signature(o0, o1)) 1373 goto InvalidInstruction; 1374 1375 uint32_t op_size = x ? 64 : 32; 1376 1377 if (isign4 == ENC_OPS3(Reg, Reg, Imm) && op_data.immediate_op != 0) { 1378 opcode.reset(uint32_t(op_data.immediate_op) << 23); 1379 1380 // AND|ANDS|BIC|BICS|ORR|EOR (immediate) uses a LogicalImm format described by N:R:S values. 1381 uint64_t imm_mask = Support::lsb_mask<uint64_t>(op_size); 1382 uint64_t imm_value = o2.as<Imm>().value_as<uint64_t>(); 1383 1384 if (op_data.negate_imm) 1385 imm_value ^= imm_mask; 1386 1387 // Logical instructions use 13-bit immediate pattern encoded as N:ImmS:ImmR. 1388 LogicalImm logical_imm; 1389 if (!Utils::encode_logical_imm(imm_value & imm_mask, op_size, Out(logical_imm))) 1390 goto InvalidImmediate; 1391 1392 // AND|BIC|ORR|EOR (immediate) can have SP on destination, but ANDS|BICS (immediate) cannot. 1393 uint32_t kOpANDS = 0x3 << 29; 1394 bool isANDS = (opcode.get() & kOpANDS) == kOpANDS; 1395 1396 if (!check_gp_id(o0, isANDS ? kZR : kSP) || !check_gp_id(o1, kZR)) 1397 goto InvalidPhysId; 1398 1399 opcode.add_imm(x, 31); 1400 opcode.add_logical_imm(logical_imm); 1401 opcode.add_reg(o1, 5); 1402 opcode.add_reg(o0, 0); 1403 goto EmitOp; 1404 } 1405 1406 if (!check_signature(o1, o2)) 1407 goto InvalidInstruction; 1408 1409 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 1410 if (!check_gp_id(o0, o1, o2, kZR)) 1411 goto InvalidPhysId; 1412 1413 opcode.reset(uint32_t(op_data.shifted_op) << 21); 1414 opcode.add_imm(x, 31); 1415 opcode.add_reg(o2, 16); 1416 opcode.add_reg(o1, 5); 1417 opcode.add_reg(o0, 0); 1418 goto EmitOp; 1419 } 1420 1421 if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) { 1422 if (!check_gp_id(o0, o1, o2, kZR)) 1423 goto InvalidPhysId; 1424 1425 uint32_t shift_type = o3.as<Imm>().predicate(); 1426 uint64_t op_shift = o3.as<Imm>().value_as<uint64_t>(); 1427 1428 if (shift_type > 0x3 || op_shift >= op_size) 1429 goto InvalidImmediate; 1430 1431 opcode.reset(uint32_t(op_data.shifted_op) << 21); 1432 opcode.add_imm(x, 31); 1433 opcode.add_imm(shift_type, 22); 1434 opcode.add_reg(o2, 16); 1435 opcode.add_imm(op_shift, 10); 1436 opcode.add_reg(o1, 5); 1437 opcode.add_reg(o0, 0); 1438 goto EmitOp; 1439 } 1440 1441 break; 1442 } 1443 1444 case InstDB::kEncodingBaseCmpCmn: { 1445 const InstDB::EncodingData::BaseCmpCmn& op_data = InstDB::EncodingData::baseCmpCmn[encoding_index]; 1446 1447 uint32_t x; 1448 if (!check_gp_type(o0, kWX, &x)) 1449 goto InvalidInstruction; 1450 1451 if (isign4 == ENC_OPS2(Reg, Imm)) { 1452 // CMN|CMP (immediate) - ZR is not allowed. 1453 if (!check_gp_id(o0, kSP)) 1454 goto InvalidPhysId; 1455 1456 // CMN|CMP (immediate) use 12-bit immediate optionally shifted by 'LSL #12'. 1457 const Imm& imm12 = o1.as<Imm>(); 1458 uint32_t imm_shift = 0; 1459 uint64_t imm_value = imm12.value_as<uint64_t>(); 1460 1461 if (imm_value > 0xFFFu) { 1462 if ((imm_value & ~uint64_t(0xFFFu << 12)) != 0) 1463 goto InvalidImmediate; 1464 imm_shift = 1; 1465 imm_value >>= 12; 1466 } 1467 1468 opcode.reset(uint32_t(op_data.immediate_op) << 24); 1469 opcode.add_imm(x, 31); 1470 opcode.add_imm(imm_shift, 22); 1471 opcode.add_imm(imm_value, 10); 1472 opcode.add_reg(o0, 5); 1473 opcode.add_reg(Gp::kIdZr, 0); 1474 goto EmitOp; 1475 } 1476 1477 if (isign4 == ENC_OPS2(Reg, Reg) || isign4 == ENC_OPS3(Reg, Reg, Imm)) { 1478 uint32_t op_size = x ? 64 : 32; 1479 uint32_t shift_type = 0; 1480 uint64_t shift_value = 0; 1481 1482 if (isign4 == ENC_OPS3(Reg, Reg, Imm)) { 1483 shift_type = o2.as<Imm>().predicate(); 1484 shift_value = o2.as<Imm>().value_as<uint64_t>(); 1485 } 1486 1487 bool has_sp = o0.as<Gp>().is_sp() || o1.as<Gp>().is_sp(); 1488 1489 // Shift operation - LSL, LSR, ASR. 1490 if (shift_type <= uint32_t(ShiftOp::kASR)) { 1491 if (!has_sp) { 1492 if (!check_signature(o0, o1)) { 1493 goto InvalidInstruction; 1494 } 1495 1496 if (shift_value >= op_size) { 1497 goto InvalidImmediate; 1498 } 1499 1500 opcode.reset(uint32_t(op_data.shifted_op) << 21); 1501 opcode.add_imm(x, 31); 1502 opcode.add_imm(shift_type, 22); 1503 opcode.add_reg(o1, 16); 1504 opcode.add_imm(shift_value, 10); 1505 opcode.add_reg(o0, 5); 1506 opcode.add_reg(Gp::kIdZr, 0); 1507 goto EmitOp; 1508 } 1509 1510 // SP register can only be used with LSL or Extend. 1511 if (shift_type != uint32_t(ShiftOp::kLSL)) 1512 goto InvalidImmediate; 1513 1514 shift_type = x ? uint32_t(ShiftOp::kUXTX) : uint32_t(ShiftOp::kUXTW); 1515 } 1516 1517 // Extend operation - UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX. 1518 shift_type -= uint32_t(ShiftOp::kUXTB); 1519 if (shift_type > 7 || shift_value > 4) { 1520 goto InvalidImmediate; 1521 } 1522 1523 // Validate whether the register operands match extend option. 1524 if (o1.as<Reg>().reg_type() != extend_option_to_reg_type(shift_type) || o0.as<Reg>().reg_type() < o1.as<Reg>().reg_type()) { 1525 goto InvalidInstruction; 1526 } 1527 1528 opcode.reset(uint32_t(op_data.extended_op) << 21); 1529 opcode.add_imm(x, 31); 1530 opcode.add_reg(o1, 16); 1531 opcode.add_imm(shift_type, 13); 1532 opcode.add_imm(shift_value, 10); 1533 opcode.add_reg(o0, 5); 1534 opcode.add_reg(Gp::kIdZr, 0); 1535 goto EmitOp; 1536 } 1537 1538 break; 1539 } 1540 1541 case InstDB::kEncodingBaseMvnNeg: { 1542 const InstDB::EncodingData::BaseMvnNeg& op_data = InstDB::EncodingData::baseMvnNeg[encoding_index]; 1543 1544 uint32_t x; 1545 if (!check_gp_type(o0, o1, kWX, &x)) 1546 goto InvalidInstruction; 1547 1548 opcode.reset(op_data.opcode); 1549 opcode.add_imm(x, 31); 1550 opcode.add_reg(o1, 16); 1551 opcode.add_reg(o0, 0); 1552 1553 if (isign4 == ENC_OPS2(Reg, Reg)) { 1554 if (!check_gp_id(o0, o1, kZR)) 1555 goto InvalidPhysId; 1556 1557 goto EmitOp; 1558 } 1559 1560 if (isign4 == ENC_OPS3(Reg, Reg, Imm)) { 1561 if (!check_gp_id(o0, o1, kZR)) 1562 goto InvalidPhysId; 1563 1564 uint32_t op_size = x ? 64 : 32; 1565 uint32_t shift_type = o2.as<Imm>().predicate(); 1566 uint64_t shift_value = o2.as<Imm>().value_as<uint64_t>(); 1567 1568 if (shift_type > uint32_t(ShiftOp::kROR) || shift_value >= op_size) 1569 goto InvalidImmediate; 1570 1571 opcode.add_imm(shift_type, 22); 1572 opcode.add_imm(shift_value, 10); 1573 goto EmitOp; 1574 } 1575 1576 break; 1577 } 1578 1579 case InstDB::kEncodingBaseTst: { 1580 const InstDB::EncodingData::BaseTst& op_data = InstDB::EncodingData::baseTst[encoding_index]; 1581 1582 uint32_t x; 1583 if (!check_gp_type(o0, kWX, &x)) 1584 goto InvalidInstruction; 1585 1586 uint32_t op_size = x ? 64 : 32; 1587 1588 if (isign4 == ENC_OPS2(Reg, Imm) && op_data.immediate_op != 0) { 1589 if (!check_gp_id(o0, kZR)) 1590 goto InvalidPhysId; 1591 1592 // TST (immediate) uses a LogicalImm format described by N:R:S values. 1593 uint64_t imm_mask = Support::lsb_mask<uint64_t>(op_size); 1594 uint64_t imm_value = o1.as<Imm>().value_as<uint64_t>(); 1595 1596 // Logical instructions use 13-bit immediate pattern encoded as N:ImmS:ImmR. 1597 LogicalImm logical_imm; 1598 if (!Utils::encode_logical_imm(imm_value & imm_mask, op_size, Out(logical_imm))) 1599 goto InvalidImmediate; 1600 1601 opcode.reset(uint32_t(op_data.immediate_op) << 22); 1602 opcode.add_logical_imm(logical_imm); 1603 opcode.add_imm(x, 31); 1604 opcode.add_reg(o0, 5); 1605 opcode.add_reg(Gp::kIdZr, 0); 1606 goto EmitOp; 1607 } 1608 1609 opcode.reset(uint32_t(op_data.shifted_op) << 21); 1610 opcode.add_imm(x, 31); 1611 opcode.add_reg(o1, 16); 1612 opcode.add_reg(o0, 5); 1613 opcode.add_reg(Gp::kIdZr, 0); 1614 1615 if (isign4 == ENC_OPS2(Reg, Reg)) { 1616 if (!check_gp_id(o0, o1, kZR)) 1617 goto InvalidPhysId; 1618 1619 goto EmitOp; 1620 } 1621 1622 if (isign4 == ENC_OPS3(Reg, Reg, Imm)) { 1623 if (!check_gp_id(o0, o1, kZR)) 1624 goto InvalidPhysId; 1625 1626 uint32_t shift_type = o2.as<Imm>().predicate(); 1627 uint64_t op_shift = o2.as<Imm>().value_as<uint64_t>(); 1628 1629 if (shift_type > 0x3 || op_shift >= op_size) 1630 goto InvalidImmediate; 1631 1632 opcode.add_imm(shift_type, 22); 1633 opcode.add_imm(op_shift, 10); 1634 goto EmitOp; 1635 } 1636 1637 break; 1638 } 1639 1640 // ------------------------------------------------------------------------ 1641 // [Base - Bit Manipulation] 1642 // ------------------------------------------------------------------------ 1643 1644 case InstDB::kEncodingBaseBfc: { 1645 const InstDB::EncodingData::BaseBfc& op_data = InstDB::EncodingData::baseBfc[encoding_index]; 1646 1647 if (isign4 == ENC_OPS3(Reg, Imm, Imm)) { 1648 uint32_t x; 1649 if (!check_gp_type(o0, InstDB::kWX, &x)) 1650 goto InvalidInstruction; 1651 1652 if (!check_gp_id(o0)) 1653 goto InvalidPhysId; 1654 1655 uint64_t lsb = o1.as<Imm>().value_as<uint64_t>(); 1656 uint64_t width = o2.as<Imm>().value_as<uint64_t>(); 1657 uint32_t op_size = x ? 64 : 32; 1658 1659 if (lsb >= op_size || width == 0 || width > op_size) 1660 goto InvalidImmediate; 1661 1662 uint32_t lsb32 = Support::neg(uint32_t(lsb)) & (op_size - 1); 1663 uint32_t width32 = uint32_t(width) - 1; 1664 1665 opcode.reset(op_data.opcode); 1666 opcode.add_imm(x, 31); 1667 opcode.add_imm(x, 22); 1668 opcode.add_imm(lsb32, 16); 1669 opcode.add_imm(width32, 10); 1670 opcode.add_reg(o0, 0); 1671 goto EmitOp; 1672 } 1673 1674 break; 1675 } 1676 1677 case InstDB::kEncodingBaseBfi: { 1678 const InstDB::EncodingData::BaseBfi& op_data = InstDB::EncodingData::baseBfi[encoding_index]; 1679 1680 if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) { 1681 uint32_t x; 1682 if (!check_gp_type(o0, InstDB::kWX, &x)) 1683 goto InvalidInstruction; 1684 1685 if (!check_signature(o0, o1)) 1686 goto InvalidInstruction; 1687 1688 if (!check_gp_id(o0, o1)) 1689 goto InvalidPhysId; 1690 1691 uint64_t lsb = o2.as<Imm>().value_as<uint64_t>(); 1692 uint64_t width = o3.as<Imm>().value_as<uint64_t>(); 1693 uint32_t op_size = x ? 64 : 32; 1694 1695 if (lsb >= op_size || width == 0 || width > op_size) 1696 goto InvalidImmediate; 1697 1698 uint32_t imm_l = Support::neg(uint32_t(lsb)) & (op_size - 1); 1699 uint32_t imm_w = uint32_t(width) - 1; 1700 1701 opcode.reset(op_data.opcode); 1702 opcode.add_imm(x, 31); 1703 opcode.add_imm(x, 22); 1704 opcode.add_imm(imm_l, 16); 1705 opcode.add_imm(imm_w, 10); 1706 opcode.add_reg(o1, 5); 1707 opcode.add_reg(o0, 0); 1708 goto EmitOp; 1709 } 1710 1711 break; 1712 } 1713 1714 case InstDB::kEncodingBaseBfm: { 1715 const InstDB::EncodingData::BaseBfm& op_data = InstDB::EncodingData::baseBfm[encoding_index]; 1716 1717 if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) { 1718 uint32_t x; 1719 if (!check_gp_type(o0, InstDB::kWX, &x)) 1720 goto InvalidInstruction; 1721 1722 if (!check_signature(o0, o1)) 1723 goto InvalidInstruction; 1724 1725 if (!check_gp_id(o0, o1)) 1726 goto InvalidPhysId; 1727 1728 uint64_t imm_r = o2.as<Imm>().value_as<uint64_t>(); 1729 uint64_t imm_s = o3.as<Imm>().value_as<uint64_t>(); 1730 uint32_t op_size = x ? 64 : 32; 1731 1732 if ((imm_r | imm_s) >= op_size) 1733 goto InvalidImmediate; 1734 1735 opcode.reset(op_data.opcode); 1736 opcode.add_imm(x, 31); 1737 opcode.add_imm(x, 22); 1738 opcode.add_imm(imm_r, 16); 1739 opcode.add_imm(imm_s, 10); 1740 opcode.add_reg(o1, 5); 1741 opcode.add_reg(o0, 0); 1742 goto EmitOp; 1743 } 1744 1745 break; 1746 } 1747 1748 case InstDB::kEncodingBaseBfx: { 1749 const InstDB::EncodingData::BaseBfx& op_data = InstDB::EncodingData::baseBfx[encoding_index]; 1750 1751 if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) { 1752 uint32_t x; 1753 if (!check_gp_type(o0, InstDB::kWX, &x)) 1754 goto InvalidInstruction; 1755 1756 if (!check_signature(o0, o1)) 1757 goto InvalidInstruction; 1758 1759 if (!check_gp_id(o0, o1)) 1760 goto InvalidPhysId; 1761 1762 uint64_t lsb = o2.as<Imm>().value_as<uint64_t>(); 1763 uint64_t width = o3.as<Imm>().value_as<uint64_t>(); 1764 uint32_t op_size = x ? 64 : 32; 1765 1766 if (lsb >= op_size || width == 0 || width > op_size) 1767 goto InvalidImmediate; 1768 1769 uint32_t lsb32 = uint32_t(lsb); 1770 uint32_t width32 = lsb32 + uint32_t(width) - 1u; 1771 1772 if (width32 >= op_size) 1773 goto InvalidImmediate; 1774 1775 opcode.reset(op_data.opcode); 1776 opcode.add_imm(x, 31); 1777 opcode.add_imm(x, 22); 1778 opcode.add_imm(lsb32, 16); 1779 opcode.add_imm(width32, 10); 1780 opcode.add_reg(o1, 5); 1781 opcode.add_reg(o0, 0); 1782 goto EmitOp; 1783 } 1784 1785 break; 1786 } 1787 1788 case InstDB::kEncodingBaseExtend: { 1789 const InstDB::EncodingData::BaseExtend& op_data = InstDB::EncodingData::baseExtend[encoding_index]; 1790 1791 if (isign4 == ENC_OPS2(Reg, Reg)) { 1792 uint32_t x; 1793 if (!check_gp_type(o0, op_data.reg_type, &x)) 1794 goto InvalidInstruction; 1795 1796 if (!o1.as<Reg>().is_gp32()) 1797 goto InvalidInstruction; 1798 1799 if (!check_gp_id(o0, o1)) 1800 goto InvalidPhysId; 1801 1802 opcode.reset(op_data.opcode()); 1803 opcode.add_imm(x, 31); 1804 opcode.add_imm(x, 22); 1805 opcode.add_reg(o1, 5); 1806 opcode.add_reg(o0, 0); 1807 goto EmitOp; 1808 } 1809 1810 break; 1811 } 1812 1813 case InstDB::kEncodingBaseExtract: { 1814 const InstDB::EncodingData::BaseExtract& op_data = InstDB::EncodingData::baseExtract[encoding_index]; 1815 1816 if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) { 1817 uint32_t x; 1818 if (!check_gp_type(o0, kWX, &x)) 1819 goto InvalidInstruction; 1820 1821 if (!check_signature(o0, o1, o2)) 1822 goto InvalidInstruction; 1823 1824 if (!check_gp_id(o0, o1, o2)) 1825 goto InvalidPhysId; 1826 1827 uint64_t lsb = o3.as<Imm>().value_as<uint64_t>(); 1828 uint32_t op_size = x ? 64 : 32; 1829 1830 if (lsb >= op_size) 1831 goto InvalidImmediate; 1832 1833 opcode.reset(op_data.opcode); 1834 opcode.add_imm(x, 31); 1835 opcode.add_imm(x, 22); 1836 opcode.add_reg(o2, 16); 1837 opcode.add_imm(lsb, 10); 1838 opcode.add_reg(o1, 5); 1839 opcode.add_reg(o0, 0); 1840 goto EmitOp; 1841 } 1842 1843 break; 1844 } 1845 1846 case InstDB::kEncodingBaseRev: { 1847 if (isign4 == ENC_OPS2(Reg, Reg)) { 1848 uint32_t x; 1849 if (!check_gp_type(o0, InstDB::kWX, &x)) 1850 goto InvalidInstruction; 1851 1852 if (!check_signature(o0, o1)) 1853 goto InvalidInstruction; 1854 1855 if (!check_gp_id(o0, o1)) 1856 goto InvalidPhysId; 1857 1858 opcode.reset(0b01011010110000000000100000000000); 1859 opcode.add_imm(x, 31); 1860 opcode.add_imm(x, 10); 1861 opcode.add_reg(o1, 5); 1862 opcode.add_reg(o0, 0); 1863 goto EmitOp; 1864 } 1865 1866 break; 1867 } 1868 1869 case InstDB::kEncodingBaseShift: { 1870 const InstDB::EncodingData::BaseShift& op_data = InstDB::EncodingData::baseShift[encoding_index]; 1871 1872 uint32_t x; 1873 if (!check_gp_type(o0, kWX, &x)) 1874 goto InvalidInstruction; 1875 1876 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 1877 if (!check_signature(o0, o1, o2)) 1878 goto InvalidInstruction; 1879 1880 if (!check_gp_id(o0, o1, o2, kZR)) 1881 goto InvalidPhysId; 1882 1883 opcode.reset(op_data.register_op()); 1884 opcode.add_imm(x, 31); 1885 opcode.add_reg(o2, 16); 1886 opcode.add_reg(o1, 5); 1887 opcode.add_reg(o0, 0); 1888 goto EmitOp; 1889 } 1890 1891 if (isign4 == ENC_OPS3(Reg, Reg, Imm) && op_data.immediate_op()) { 1892 if (!check_signature(o0, o1)) 1893 goto InvalidInstruction; 1894 1895 if (!check_gp_id(o0, o1, kZR)) 1896 goto InvalidPhysId; 1897 1898 uint64_t imm_r = o2.as<Imm>().value_as<uint64_t>(); 1899 uint32_t op_size = x ? 64 : 32; 1900 1901 if (imm_r >= op_size) 1902 goto InvalidImmediate; 1903 1904 opcode.reset(op_data.immediate_op()); 1905 opcode.add_imm(x, 31); 1906 opcode.add_imm(x, 22); 1907 opcode.add_reg(o1, 5); 1908 opcode.add_reg(o0, 0); 1909 1910 if (opcode.get() & B(10)) { 1911 // ASR and LSR (immediate) has the same logic. 1912 opcode.add_imm(x, 15); 1913 opcode.add_imm(imm_r, 16); 1914 goto EmitOp; 1915 } 1916 1917 if (op_data.ror == 0) { 1918 // LSL (immediate) is an alias to UBFM 1919 uint32_t ubfm_imm_r = Support::neg(uint32_t(imm_r)) & (op_size - 1); 1920 uint32_t ubfm_imm_s = op_size - 1 - uint32_t(imm_r); 1921 1922 opcode.add_imm(ubfm_imm_r, 16); 1923 opcode.add_imm(ubfm_imm_s, 10); 1924 goto EmitOp; 1925 } 1926 else { 1927 // ROR (immediate) is an alias to EXTR. 1928 opcode.add_imm(imm_r, 10); 1929 opcode.add_reg(o1, 16); 1930 goto EmitOp; 1931 } 1932 } 1933 1934 break; 1935 } 1936 1937 // ------------------------------------------------------------------------ 1938 // [Base - Conditionals] 1939 // ------------------------------------------------------------------------ 1940 1941 case InstDB::kEncodingBaseCCmp: { 1942 const InstDB::EncodingData::BaseCCmp& op_data = InstDB::EncodingData::baseCCmp[encoding_index]; 1943 1944 if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm) || isign4 == ENC_OPS4(Reg, Imm, Imm, Imm)) { 1945 uint32_t x; 1946 if (!check_gp_type(o0, InstDB::kWX, &x)) 1947 goto InvalidInstruction; 1948 1949 if (!check_gp_id(o0, kZR)) 1950 goto InvalidPhysId; 1951 1952 uint64_t nzcv = o2.as<Imm>().value_as<uint64_t>(); 1953 uint64_t cond = o3.as<Imm>().value_as<uint64_t>(); 1954 1955 if ((nzcv | cond) > 0xFu) 1956 goto InvalidImmediate; 1957 1958 opcode.reset(op_data.opcode); 1959 opcode.add_imm(x, 31); 1960 opcode.add_imm(cond_code_to_opcode_field(uint32_t(cond)), 12); 1961 opcode.add_imm(nzcv, 0); 1962 1963 if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) { 1964 // CCMN|CCMP (register) form. 1965 if (!check_signature(o0, o1)) 1966 goto InvalidInstruction; 1967 1968 if (!check_gp_id(o1, kZR)) 1969 goto InvalidPhysId; 1970 1971 opcode.add_reg(o1, 16); 1972 opcode.add_reg(o0, 5); 1973 goto EmitOp; 1974 } 1975 else { 1976 // CCMN|CCMP (immediate) form. 1977 uint64_t imm5 = o1.as<Imm>().value_as<uint64_t>(); 1978 if (imm5 > 0x1F) 1979 goto InvalidImmediate; 1980 1981 opcode.add_imm(1, 11); 1982 opcode.add_imm(imm5, 16); 1983 opcode.add_reg(o0, 5); 1984 goto EmitOp; 1985 } 1986 } 1987 1988 break; 1989 } 1990 1991 case InstDB::kEncodingBaseCInc: { 1992 const InstDB::EncodingData::BaseCInc& op_data = InstDB::EncodingData::baseCInc[encoding_index]; 1993 1994 if (isign4 == ENC_OPS3(Reg, Reg, Imm)) { 1995 uint32_t x; 1996 if (!check_gp_type(o0, o1, InstDB::kWX, &x)) 1997 goto InvalidInstruction; 1998 1999 if (!check_gp_id(o0, o1, kZR)) 2000 goto InvalidPhysId; 2001 2002 uint64_t cond = o2.as<Imm>().value_as<uint64_t>(); 2003 if (cond - 2u > 0xEu) 2004 goto InvalidImmediate; 2005 2006 opcode.reset(op_data.opcode); 2007 opcode.add_imm(x, 31); 2008 opcode.add_reg(o1, 16); 2009 opcode.add_imm(cond_code_to_opcode_field(uint32_t(cond)) ^ 1u, 12); 2010 opcode.add_reg(o1, 5); 2011 opcode.add_reg(o0, 0); 2012 goto EmitOp; 2013 } 2014 2015 break; 2016 } 2017 2018 case InstDB::kEncodingBaseCSel: { 2019 const InstDB::EncodingData::BaseCSel& op_data = InstDB::EncodingData::baseCSel[encoding_index]; 2020 2021 if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) { 2022 uint32_t x; 2023 if (!check_gp_type(o0, o1, o2, InstDB::kWX, &x)) 2024 goto InvalidInstruction; 2025 2026 if (!check_gp_id(o0, o1, o2, kZR)) 2027 goto InvalidPhysId; 2028 2029 uint64_t cond = o3.as<Imm>().value_as<uint64_t>(); 2030 if (cond > 0xFu) 2031 goto InvalidImmediate; 2032 2033 opcode.reset(op_data.opcode); 2034 opcode.add_imm(x, 31); 2035 opcode.add_reg(o2, 16); 2036 opcode.add_imm(cond_code_to_opcode_field(uint32_t(cond)), 12); 2037 opcode.add_reg(o1, 5); 2038 opcode.add_reg(o0, 0); 2039 goto EmitOp; 2040 } 2041 2042 break; 2043 } 2044 2045 case InstDB::kEncodingBaseCSet: { 2046 const InstDB::EncodingData::BaseCSet& op_data = InstDB::EncodingData::baseCSet[encoding_index]; 2047 2048 if (isign4 == ENC_OPS2(Reg, Imm)) { 2049 uint32_t x; 2050 if (!check_gp_type(o0, InstDB::kWX, &x)) 2051 goto InvalidInstruction; 2052 2053 if (!check_gp_id(o0, kZR)) 2054 goto InvalidPhysId; 2055 2056 uint64_t cond = o1.as<Imm>().value_as<uint64_t>(); 2057 if (cond - 2u >= 0xEu) 2058 goto InvalidImmediate; 2059 2060 opcode.reset(op_data.opcode); 2061 opcode.add_imm(x, 31); 2062 opcode.add_imm(cond_code_to_opcode_field(uint32_t(cond)) ^ 1u, 12); 2063 opcode.add_reg(o0, 0); 2064 goto EmitOp; 2065 } 2066 2067 break; 2068 } 2069 2070 // ------------------------------------------------------------------------ 2071 // [Base - Min/Max] 2072 // ------------------------------------------------------------------------ 2073 2074 case InstDB::kEncodingBaseMinMax: { 2075 const InstDB::EncodingData::BaseMinMax& op_data = InstDB::EncodingData::baseMinMax[encoding_index]; 2076 2077 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 2078 uint32_t x; 2079 if (!check_gp_type(o0, InstDB::kWX, &x)) 2080 goto InvalidInstruction; 2081 2082 if (!check_signature(o0, o1, o2)) 2083 goto InvalidInstruction; 2084 2085 opcode.reset(op_data.register_op); 2086 opcode.add_imm(x, 31); 2087 opcode.add_reg(o2, 16); 2088 opcode.add_reg(o1, 5); 2089 opcode.add_reg(o0, 0); 2090 goto EmitOp; 2091 } 2092 2093 if (isign4 == ENC_OPS3(Reg, Reg, Imm)) { 2094 uint32_t x; 2095 if (!check_gp_type(o0, InstDB::kWX, &x)) 2096 goto InvalidInstruction; 2097 2098 if (!check_signature(o0, o1)) 2099 goto InvalidInstruction; 2100 2101 uint64_t imm = o2.as<Imm>().value_as<uint64_t>(); 2102 2103 if (op_data.immediate_op & (1u << 18)) { 2104 // Zero extend imm. 2105 if (!Support::is_uint_n<8>(imm)) { 2106 goto InvalidImmediate; 2107 } 2108 } 2109 else { 2110 // Sign extend imm. 2111 if (!Support::is_int_n<8>(int64_t(imm))) { 2112 goto InvalidImmediate; 2113 } 2114 } 2115 2116 opcode.reset(op_data.immediate_op); 2117 opcode.add_imm(x, 31); 2118 opcode.add_imm(uint32_t(imm & 0xFFu), 10); 2119 opcode.add_reg(o1, 5); 2120 opcode.add_reg(o0, 0); 2121 goto EmitOp; 2122 } 2123 2124 break; 2125 } 2126 2127 // ------------------------------------------------------------------------ 2128 // [Base - Special] 2129 // ------------------------------------------------------------------------ 2130 2131 case InstDB::kEncodingBaseAtDcIcTlbi: { 2132 const InstDB::EncodingData::BaseAtDcIcTlbi& op_data = InstDB::EncodingData::baseAtDcIcTlbi[encoding_index]; 2133 2134 if (isign4 == ENC_OPS1(Imm) || isign4 == ENC_OPS2(Imm, Reg)) { 2135 if (op_data.mandatory_reg && isign4 != ENC_OPS2(Imm, Reg)) 2136 goto InvalidInstruction; 2137 2138 if (o0.as<Imm>().value_as<uint64_t>() > 0x7FFFu) 2139 goto InvalidImmediate; 2140 2141 uint32_t imm = o0.as<Imm>().value_as<uint32_t>(); 2142 if ((imm & op_data.imm_verify_mask) != op_data.imm_verify_data) 2143 goto InvalidImmediate; 2144 2145 uint32_t rt = 31; 2146 if (o1.is_reg()) { 2147 if (!o1.as<Reg>().is_gp64()) 2148 goto InvalidInstruction; 2149 2150 if (!check_gp_id(o1, kZR)) 2151 goto InvalidPhysId; 2152 2153 rt = o1.id() & 31; 2154 } 2155 2156 opcode.reset(0b11010101000010000000000000000000); 2157 opcode.add_imm(imm, 5); 2158 opcode.add_reg(rt, 0); 2159 goto EmitOp; 2160 } 2161 break; 2162 } 2163 2164 case InstDB::kEncodingBaseMrs: { 2165 if (isign4 == ENC_OPS2(Reg, Imm)) { 2166 if (!o0.as<Reg>().is_gp64()) 2167 goto InvalidInstruction; 2168 2169 if (!check_gp_id(o0, kZR)) 2170 goto InvalidPhysId; 2171 2172 if (o1.as<Imm>().value_as<uint64_t>() > 0xFFFFu) 2173 goto InvalidImmediate; 2174 2175 uint32_t imm = o1.as<Imm>().value_as<uint32_t>(); 2176 if (!(imm & B(15))) 2177 goto InvalidImmediate; 2178 2179 opcode.reset(0b11010101001100000000000000000000); 2180 opcode.add_imm(imm, 5); 2181 opcode.add_reg(o0, 0); 2182 goto EmitOp; 2183 } 2184 2185 break; 2186 } 2187 2188 case InstDB::kEncodingBaseMsr: { 2189 if (isign4 == ENC_OPS2(Imm, Reg)) { 2190 if (!o1.as<Reg>().is_gp64()) 2191 goto InvalidInstruction; 2192 2193 if (o0.as<Imm>().value_as<uint64_t>() > 0xFFFFu) 2194 goto InvalidImmediate; 2195 2196 uint32_t imm = o0.as<Imm>().value_as<uint32_t>(); 2197 if (!(imm & B(15))) 2198 goto InvalidImmediate; 2199 2200 if (!check_gp_id(o1, kZR)) 2201 goto InvalidPhysId; 2202 2203 opcode.reset(0b11010101000100000000000000000000); 2204 opcode.add_imm(imm, 5); 2205 opcode.add_reg(o1, 0); 2206 goto EmitOp; 2207 } 2208 2209 if (isign4 == ENC_OPS2(Imm, Imm)) { 2210 if (o0.as<Imm>().value_as<uint64_t>() > 0x1Fu) 2211 goto InvalidImmediate; 2212 2213 if (o1.as<Imm>().value_as<uint64_t>() > 0xFu) 2214 goto InvalidImmediate; 2215 2216 uint32_t op = o0.as<Imm>().value_as<uint32_t>(); 2217 uint32_t crm = o1.as<Imm>().value_as<uint32_t>(); 2218 2219 uint32_t op1 = uint32_t(op) >> 3; 2220 uint32_t op2 = uint32_t(op) & 0x7u; 2221 2222 opcode.reset(0b11010101000000000100000000011111); 2223 opcode.add_imm(op1, 16); 2224 opcode.add_imm(crm, 8); 2225 opcode.add_imm(op2, 5); 2226 goto EmitOp; 2227 } 2228 2229 break; 2230 } 2231 2232 case InstDB::kEncodingBaseSys: { 2233 if (isign4 == ENC_OPS4(Imm, Imm, Imm, Imm)) { 2234 if (o0.as<Imm>().value_as<uint64_t>() > 0x7u || 2235 o1.as<Imm>().value_as<uint64_t>() > 0xFu || 2236 o2.as<Imm>().value_as<uint64_t>() > 0xFu || 2237 o3.as<Imm>().value_as<uint64_t>() > 0x7u) 2238 goto InvalidImmediate; 2239 2240 uint32_t op1 = o0.as<Imm>().value_as<uint32_t>(); 2241 uint32_t crn = o1.as<Imm>().value_as<uint32_t>(); 2242 uint32_t crm = o2.as<Imm>().value_as<uint32_t>(); 2243 uint32_t op2 = o3.as<Imm>().value_as<uint32_t>(); 2244 uint32_t rt = 31; 2245 2246 const Operand_& o4 = op_ext[EmitterUtils::kOp4]; 2247 if (o4.is_reg()) { 2248 if (!o4.as<Reg>().is_gp64()) 2249 goto InvalidInstruction; 2250 2251 if (!check_gp_id(o4, kZR)) 2252 goto InvalidPhysId; 2253 2254 rt = o4.id() & 31; 2255 } 2256 else if (!o4.is_none()) { 2257 goto InvalidInstruction; 2258 } 2259 2260 opcode.reset(0b11010101000010000000000000000000); 2261 opcode.add_imm(op1, 16); 2262 opcode.add_imm(crn, 12); 2263 opcode.add_imm(crm, 8); 2264 opcode.add_imm(op2, 5); 2265 opcode.add_imm(rt, 0); 2266 goto EmitOp; 2267 } 2268 2269 break; 2270 } 2271 2272 // ------------------------------------------------------------------------ 2273 // [Base - Branch] 2274 // ------------------------------------------------------------------------ 2275 2276 case InstDB::kEncodingBaseBranchReg: { 2277 const InstDB::EncodingData::BaseBranchReg& op_data = InstDB::EncodingData::baseBranchReg[encoding_index]; 2278 2279 if (isign4 == ENC_OPS1(Reg)) { 2280 if (!o0.as<Reg>().is_gp64()) 2281 goto InvalidInstruction; 2282 2283 if (!check_gp_id(o0, kZR)) 2284 goto InvalidPhysId; 2285 2286 opcode.reset(op_data.opcode); 2287 opcode.add_reg(o0, 5); 2288 goto EmitOp; 2289 } 2290 2291 break; 2292 } 2293 2294 case InstDB::kEncodingBaseBranchRel: { 2295 const InstDB::EncodingData::BaseBranchRel& op_data = InstDB::EncodingData::baseBranchRel[encoding_index]; 2296 2297 if (isign4 == ENC_OPS1(Label) || isign4 == ENC_OPS1(Imm)) { 2298 opcode.reset(op_data.opcode); 2299 rm_rel = &o0; 2300 2301 // A variation that uses Cond code (or where Cond code is forced like BC.<cond>). 2302 if (inst_cc != CondCode::kAL || Support::bit_test(opcode.v, 30)) { 2303 if (opcode.has_x()) { 2304 // Condition code cannot be applied when the instruction has X bit set (this would be BL instruction). 2305 goto InvalidInstruction; 2306 } 2307 2308 opcode |= B(30); 2309 opcode.add_imm(cond_code_to_opcode_field(uint32_t(inst_cc)), 0); 2310 offset_format.reset_to_imm_value(OffsetType::kSignedOffset, 4, 5, 19, 2); 2311 goto EmitOp_Rel; 2312 } 2313 2314 offset_format.reset_to_imm_value(OffsetType::kSignedOffset, 4, 0, 26, 2); 2315 goto EmitOp_Rel; 2316 } 2317 2318 break; 2319 } 2320 2321 case InstDB::kEncodingBaseBranchCmp: { 2322 const InstDB::EncodingData::BaseBranchCmp& op_data = InstDB::EncodingData::baseBranchCmp[encoding_index]; 2323 2324 if (isign4 == ENC_OPS2(Reg, Label) || isign4 == ENC_OPS2(Reg, Imm)) { 2325 uint32_t x; 2326 if (!check_gp_type(o0, kWX, &x)) 2327 goto InvalidInstruction; 2328 2329 if (!check_gp_id(o0, kZR)) 2330 goto InvalidPhysId; 2331 2332 opcode.reset(op_data.opcode); 2333 opcode.add_imm(x, 31); 2334 opcode.add_reg(o0, 0); 2335 offset_format.reset_to_imm_value(OffsetType::kSignedOffset, 4, 5, 19, 2); 2336 2337 rm_rel = &o1; 2338 goto EmitOp_Rel; 2339 } 2340 2341 break; 2342 } 2343 2344 case InstDB::kEncodingBaseBranchTst: { 2345 const InstDB::EncodingData::BaseBranchTst& op_data = InstDB::EncodingData::baseBranchTst[encoding_index]; 2346 2347 if (isign4 == ENC_OPS3(Reg, Imm, Label) || isign4 == ENC_OPS3(Reg, Imm, Imm)) { 2348 uint32_t x; 2349 if (!check_gp_type(o0, kWX, &x)) 2350 goto InvalidInstruction; 2351 2352 if (!check_gp_id(o0, kZR)) 2353 goto InvalidPhysId; 2354 2355 uint64_t imm = o1.as<Imm>().value_as<uint64_t>(); 2356 2357 opcode.reset(op_data.opcode); 2358 if (imm >= 32) { 2359 if (!x) 2360 goto InvalidImmediate; 2361 opcode.add_imm(x, 31); 2362 imm &= 0x1F; 2363 } 2364 2365 opcode.add_reg(o0, 0); 2366 opcode.add_imm(imm, 19); 2367 offset_format.reset_to_imm_value(OffsetType::kSignedOffset, 4, 5, 14, 2); 2368 2369 rm_rel = &o2; 2370 goto EmitOp_Rel; 2371 } 2372 2373 break; 2374 } 2375 2376 // ------------------------------------------------------------------------ 2377 // [Base - Prefetch] 2378 // ------------------------------------------------------------------------ 2379 2380 case InstDB::kEncodingBasePrfm: { 2381 const InstDB::EncodingData::BasePrfm& op_data = InstDB::EncodingData::basePrfm[encoding_index]; 2382 2383 if (isign4 == ENC_OPS2(Imm, Mem)) { 2384 const Mem& m = o1.as<Mem>(); 2385 rm_rel = &m; 2386 2387 uint32_t imm_shift = 3u; 2388 2389 if (o0.as<Imm>().value_as<uint64_t>() > 0x1Fu) 2390 goto InvalidImmediate; 2391 2392 if (!check_mem_base_index_rel(m)) 2393 goto InvalidAddress; 2394 2395 int64_t offset = m.offset(); 2396 uint32_t prfop = o0.as<Imm>().value_as<uint32_t>(); 2397 2398 if (m.has_base_reg()) { 2399 // [Base {Offset | Index}] 2400 if (m.has_index()) { 2401 uint32_t opt = shift_op_to_ld_st_opt_map[size_t(m.shift_op())]; 2402 if (opt == 0xFF) 2403 goto InvalidAddress; 2404 2405 uint32_t shift = m.shift(); 2406 uint32_t s = shift != 0; 2407 2408 if (s && shift != imm_shift) 2409 goto InvalidAddressScale; 2410 2411 opcode.reset(uint32_t(op_data.register_op) << 21); 2412 opcode.add_imm(opt, 13); 2413 opcode.add_imm(s, 12); 2414 opcode |= B(11); 2415 opcode.add_imm(prfop, 0); 2416 goto EmitOp_MemBaseIndex_Rn5_Rm16; 2417 } 2418 2419 if (!Support::is_int_n<32>(offset)) 2420 goto InvalidDisplacement; 2421 2422 int32_t offset32 = int32_t(offset); 2423 2424 if (m.is_pre_or_post()) 2425 goto InvalidAddress; 2426 2427 uint32_t imm12 = uint32_t(offset32) >> imm_shift; 2428 2429 if (Support::is_uint_n<12>(imm12) && (imm12 << imm_shift) == uint32_t(offset32)) { 2430 opcode.reset(uint32_t(op_data.s_offset_op) << 22); 2431 opcode.add_imm(imm12, 10); 2432 opcode.add_imm(prfop, 0); 2433 goto EmitOp_MemBase_Rn5; 2434 } 2435 2436 if (Support::is_int_n<9>(offset32)) { 2437 opcode.reset(uint32_t(op_data.u_offset_op) << 21); 2438 opcode.add_imm(uint32_t(offset32) & 0x1FFu, 12); 2439 opcode.add_imm(prfop, 0); 2440 goto EmitOp_MemBase_Rn5; 2441 } 2442 2443 goto InvalidAddress; 2444 } 2445 else { 2446 opcode.reset(uint32_t(op_data.literal_op) << 24); 2447 opcode.add_imm(prfop, 0); 2448 offset_format.reset_to_imm_value(OffsetType::kSignedOffset, 4, 5, 19, 2); 2449 goto EmitOp_Rel; 2450 } 2451 } 2452 2453 break; 2454 } 2455 2456 // ------------------------------------------------------------------------ 2457 // [Base - Load / Store] 2458 // ------------------------------------------------------------------------ 2459 2460 case InstDB::kEncodingBaseLdSt: { 2461 const InstDB::EncodingData::BaseLdSt& op_data = InstDB::EncodingData::baseLdSt[encoding_index]; 2462 2463 if (isign4 == ENC_OPS2(Reg, Mem)) { 2464 const Mem& m = o1.as<Mem>(); 2465 rm_rel = &m; 2466 2467 uint32_t x; 2468 if (!check_gp_type(o0, op_data.reg_type, &x)) 2469 goto InvalidInstruction; 2470 2471 if (!check_gp_id(o0, kZR)) 2472 goto InvalidPhysId; 2473 2474 // Instructions that work with either word or dword have the unsigned 2475 // offset shift set to 2 (word), so we set it to 3 (dword) if this is 2476 // X version of the instruction. 2477 uint32_t x_shift_mask = uint32_t(op_data.u_offset_shift == 2); 2478 uint32_t imm_shift = uint32_t(op_data.u_offset_shift) + (x & x_shift_mask); 2479 2480 if (!check_mem_base_index_rel(m)) 2481 goto InvalidAddress; 2482 2483 int64_t offset = m.offset(); 2484 if (m.has_base_reg()) { 2485 // [Base {Offset | Index}] 2486 if (m.has_index()) { 2487 uint32_t opt = shift_op_to_ld_st_opt_map[size_t(m.shift_op())]; 2488 if (opt == 0xFF) 2489 goto InvalidAddress; 2490 2491 uint32_t shift = m.shift(); 2492 uint32_t s = shift != 0; 2493 2494 if (s && shift != imm_shift) 2495 goto InvalidAddressScale; 2496 2497 opcode.reset(uint32_t(op_data.register_op) << 21); 2498 opcode.xor_imm(x, op_data.x_offset); 2499 opcode.add_imm(opt, 13); 2500 opcode.add_imm(s, 12); 2501 opcode |= B(11); 2502 opcode.add_reg(o0, 0); 2503 goto EmitOp_MemBaseIndex_Rn5_Rm16; 2504 } 2505 2506 // Makes it easier to work with the offset especially on 32-bit arch. 2507 if (!Support::is_int_n<32>(offset)) 2508 goto InvalidDisplacement; 2509 int32_t offset32 = int32_t(offset); 2510 2511 if (m.is_pre_or_post()) { 2512 if (!Support::is_int_n<9>(offset32)) 2513 goto InvalidDisplacement; 2514 2515 opcode.reset(uint32_t(op_data.pre_post_op) << 21); 2516 opcode.xor_imm(x, op_data.x_offset); 2517 opcode.add_imm(offset32 & 0x1FF, 12); 2518 opcode.add_imm(m.is_pre_index(), 11); 2519 opcode |= B(10); 2520 opcode.add_reg(o0, 0); 2521 goto EmitOp_MemBase_Rn5; 2522 } 2523 else { 2524 uint32_t imm12 = uint32_t(offset32) >> imm_shift; 2525 2526 // Alternative form of LDUR/STUR and related instructions as described by AArch64 reference manual: 2527 // 2528 // If this instruction is not encodable with scaled unsigned offset, try unscaled signed offset. 2529 if (!Support::is_uint_n<12>(imm12) || (imm12 << imm_shift) != uint32_t(offset32)) { 2530 inst_id = op_data.u_alt_inst_id; 2531 inst_info = &InstDB::_inst_info_table[inst_id]; 2532 encoding_index = inst_info->_encoding_data_index; 2533 goto Case_BaseLdurStur; 2534 } 2535 2536 opcode.reset(uint32_t(op_data.u_offset_op) << 22); 2537 opcode.xor_imm(x, op_data.x_offset); 2538 opcode.add_imm(imm12, 10); 2539 opcode.add_reg(o0, 0); 2540 goto EmitOp_MemBase_Rn5; 2541 } 2542 } 2543 else { 2544 if (!op_data.literal_op) 2545 goto InvalidAddress; 2546 2547 opcode.reset(uint32_t(op_data.literal_op) << 24); 2548 opcode.xor_imm(x, op_data.x_offset); 2549 opcode.add_reg(o0, 0); 2550 offset_format.reset_to_imm_value(OffsetType::kSignedOffset, 4, 5, 19, 2); 2551 goto EmitOp_Rel; 2552 } 2553 } 2554 2555 break; 2556 } 2557 2558 case InstDB::kEncodingBaseLdpStp: { 2559 const InstDB::EncodingData::BaseLdpStp& op_data = InstDB::EncodingData::baseLdpStp[encoding_index]; 2560 2561 if (isign4 == ENC_OPS3(Reg, Reg, Mem)) { 2562 const Mem& m = o2.as<Mem>(); 2563 rm_rel = &m; 2564 2565 uint32_t x; 2566 if (!check_gp_type(o0, o1, op_data.reg_type, &x)) 2567 goto InvalidInstruction; 2568 2569 if (!check_gp_id(o0, o1, kZR)) 2570 goto InvalidPhysId; 2571 2572 if (m.base_type() != RegType::kGp64 || m.has_index()) 2573 goto InvalidAddress; 2574 2575 if (m.is_offset_64bit()) 2576 goto InvalidDisplacement; 2577 2578 uint32_t offset_shift = op_data.offset_shift + x; 2579 int32_t offset32 = m.offset_lo32() >> offset_shift; 2580 2581 // Make sure we didn't lose bits by applying the mandatory offset shift. 2582 if (uint32_t(offset32) << offset_shift != uint32_t(m.offset_lo32())) 2583 goto InvalidDisplacement; 2584 2585 // Offset is encoded as 7-bit immediate. 2586 if (!Support::is_int_n<7>(offset32)) 2587 goto InvalidDisplacement; 2588 2589 if (m.is_pre_or_post() && offset32 != 0) { 2590 if (!op_data.pre_post_op) 2591 goto InvalidAddress; 2592 2593 opcode.reset(uint32_t(op_data.pre_post_op) << 22); 2594 opcode.add_imm(m.is_pre_index(), 24); 2595 } 2596 else { 2597 opcode.reset(uint32_t(op_data.offset_op) << 22); 2598 } 2599 2600 opcode.add_imm(x, op_data.x_offset); 2601 opcode.add_imm(offset32 & 0x7F, 15); 2602 opcode.add_reg(o1, 10); 2603 opcode.add_reg(o0, 0); 2604 goto EmitOp_MemBase_Rn5; 2605 } 2606 2607 break; 2608 } 2609 2610 case InstDB::kEncodingBaseStx: { 2611 const InstDB::EncodingData::BaseStx& op_data = InstDB::EncodingData::baseStx[encoding_index]; 2612 2613 if (isign4 == ENC_OPS3(Reg, Reg, Mem)) { 2614 const Mem& m = o2.as<Mem>(); 2615 uint32_t x; 2616 2617 if (!o0.as<Reg>().is_gp32() || !check_gp_type(o1, op_data.reg_type, &x)) 2618 goto InvalidInstruction; 2619 2620 if (!check_gp_id(o0, o1, kZR)) 2621 goto InvalidPhysId; 2622 2623 opcode.reset(op_data.opcode()); 2624 opcode.add_imm(x, op_data.x_offset); 2625 opcode.add_reg(o0, 16); 2626 opcode.add_reg(o1, 0); 2627 2628 rm_rel = &m; 2629 goto EmitOp_MemBaseNoImm_Rn5; 2630 } 2631 2632 break; 2633 } 2634 2635 case InstDB::kEncodingBaseLdxp: { 2636 const InstDB::EncodingData::BaseLdxp& op_data = InstDB::EncodingData::baseLdxp[encoding_index]; 2637 2638 if (isign4 == ENC_OPS3(Reg, Reg, Mem)) { 2639 const Mem& m = o2.as<Mem>(); 2640 uint32_t x; 2641 2642 if (!check_gp_type(o0, op_data.reg_type, &x) || !check_signature(o0, o1)) 2643 goto InvalidInstruction; 2644 2645 if (!check_gp_id(o0, o1, kZR)) 2646 goto InvalidPhysId; 2647 2648 opcode.reset(op_data.opcode()); 2649 opcode.add_imm(x, op_data.x_offset); 2650 opcode.add_reg(o1, 10); 2651 opcode.add_reg(o0, 0); 2652 2653 rm_rel = &m; 2654 goto EmitOp_MemBaseNoImm_Rn5; 2655 } 2656 2657 break; 2658 } 2659 2660 case InstDB::kEncodingBaseStxp: { 2661 const InstDB::EncodingData::BaseStxp& op_data = InstDB::EncodingData::baseStxp[encoding_index]; 2662 2663 if (isign4 == ENC_OPS4(Reg, Reg, Reg, Mem)) { 2664 const Mem& m = o3.as<Mem>(); 2665 uint32_t x; 2666 2667 if (!o0.as<Reg>().is_gp32() || !check_gp_type(o1, op_data.reg_type, &x) || !check_signature(o1, o2)) 2668 goto InvalidInstruction; 2669 2670 if (!check_gp_id(o0, o1, o2, kZR)) 2671 goto InvalidPhysId; 2672 2673 opcode.reset(op_data.opcode()); 2674 opcode.add_imm(x, op_data.x_offset); 2675 opcode.add_reg(o0, 16); 2676 opcode.add_reg(o2, 10); 2677 opcode.add_reg(o1, 0); 2678 2679 rm_rel = &m; 2680 goto EmitOp_MemBaseNoImm_Rn5; 2681 } 2682 2683 break; 2684 } 2685 2686 case InstDB::kEncodingBaseRM_NoImm: { 2687 const InstDB::EncodingData::BaseRM_NoImm& op_data = InstDB::EncodingData::baseRM_NoImm[encoding_index]; 2688 2689 if (isign4 == ENC_OPS2(Reg, Mem)) { 2690 const Mem& m = o1.as<Mem>(); 2691 rm_rel = &m; 2692 2693 uint32_t x; 2694 if (!check_gp_type(o0, op_data.reg_type, &x)) 2695 goto InvalidInstruction; 2696 2697 if (!check_gp_id(o0, op_data.reg_hi_id)) 2698 goto InvalidPhysId; 2699 2700 opcode.reset(op_data.opcode()); 2701 opcode.add_imm(x, op_data.x_offset); 2702 opcode.add_reg(o0, 0); 2703 goto EmitOp_MemBaseNoImm_Rn5; 2704 } 2705 2706 break; 2707 } 2708 2709 case InstDB::kEncodingBaseRM_SImm9: { 2710 Case_BaseLdurStur: 2711 const InstDB::EncodingData::BaseRM_SImm9& op_data = InstDB::EncodingData::baseRM_SImm9[encoding_index]; 2712 2713 if (isign4 == ENC_OPS2(Reg, Mem)) { 2714 const Mem& m = o1.as<Mem>(); 2715 rm_rel = &m; 2716 2717 uint32_t x; 2718 if (!check_gp_type(o0, op_data.reg_type, &x)) 2719 goto InvalidInstruction; 2720 2721 if (!check_gp_id(o0, op_data.reg_hi_id)) 2722 goto InvalidPhysId; 2723 2724 if (m.has_base_reg() && !m.has_index()) { 2725 if (m.is_offset_64bit()) 2726 goto InvalidDisplacement; 2727 2728 int32_t offset32 = m.offset_lo32() >> op_data.imm_shift; 2729 if (Support::shl(offset32, op_data.imm_shift) != m.offset_lo32()) 2730 goto InvalidDisplacement; 2731 2732 if (!Support::is_int_n<9>(offset32)) 2733 goto InvalidDisplacement; 2734 2735 if (m.is_fixed_offset()) { 2736 opcode.reset(op_data.offset_op()); 2737 } 2738 else { 2739 if (!op_data.pre_post_op()) 2740 goto InvalidInstruction; 2741 2742 opcode.reset(op_data.pre_post_op()); 2743 opcode.xor_imm(m.is_pre_index(), 11); 2744 } 2745 2746 opcode.xor_imm(x, op_data.x_offset); 2747 opcode.add_imm(offset32 & 0x1FF, 12); 2748 opcode.add_reg(o0, 0); 2749 goto EmitOp_MemBase_Rn5; 2750 } 2751 2752 goto InvalidAddress; 2753 } 2754 2755 break; 2756 } 2757 2758 case InstDB::kEncodingBaseRM_SImm10: { 2759 const InstDB::EncodingData::BaseRM_SImm10& op_data = InstDB::EncodingData::baseRM_SImm10[encoding_index]; 2760 2761 if (isign4 == ENC_OPS2(Reg, Mem)) { 2762 const Mem& m = o1.as<Mem>(); 2763 rm_rel = &m; 2764 2765 uint32_t x; 2766 if (!check_gp_type(o0, op_data.reg_type, &x)) 2767 goto InvalidInstruction; 2768 2769 if (!check_gp_id(o0, op_data.reg_hi_id)) 2770 goto InvalidPhysId; 2771 2772 if (m.has_base_reg() && !m.has_index()) { 2773 if (m.is_offset_64bit()) 2774 goto InvalidDisplacement; 2775 2776 int32_t offset32 = m.offset_lo32() >> op_data.imm_shift; 2777 if (Support::shl(offset32, op_data.imm_shift) != m.offset_lo32()) 2778 goto InvalidDisplacement; 2779 2780 if (!Support::is_int_n<10>(offset32)) 2781 goto InvalidDisplacement; 2782 2783 if (m.is_post_index()) 2784 goto InvalidAddress; 2785 2786 // Offset has 10 bits, sign is stored in the 10th bit. 2787 offset32 &= 0x3FF; 2788 2789 opcode.reset(op_data.opcode()); 2790 opcode.xor_imm(m.is_pre_index(), 11); 2791 opcode.xor_imm(x, op_data.x_offset); 2792 opcode.add_imm(offset32 >> 9, 22); 2793 opcode.add_imm(offset32, 12); 2794 opcode.add_reg(o0, 0); 2795 goto EmitOp_MemBase_Rn5; 2796 } 2797 2798 goto InvalidAddress; 2799 } 2800 2801 break; 2802 } 2803 2804 case InstDB::kEncodingBaseAtomicOp: { 2805 const InstDB::EncodingData::BaseAtomicOp& op_data = InstDB::EncodingData::baseAtomicOp[encoding_index]; 2806 2807 if (isign4 == ENC_OPS3(Reg, Reg, Mem)) { 2808 const Mem& m = o2.as<Mem>(); 2809 uint32_t x; 2810 2811 if (!check_gp_type(o0, op_data.reg_type, &x) || !check_signature(o0, o1)) 2812 goto InvalidInstruction; 2813 2814 if (!check_gp_id(o0, o1, kZR)) 2815 goto InvalidInstruction; 2816 2817 opcode.reset(op_data.opcode()); 2818 opcode.add_imm(x, op_data.x_offset); 2819 opcode.add_reg(o0, 16); 2820 opcode.add_reg(o1, 0); 2821 2822 rm_rel = &m; 2823 goto EmitOp_MemBaseNoImm_Rn5; 2824 } 2825 2826 break; 2827 } 2828 2829 case InstDB::kEncodingBaseAtomicSt: { 2830 const InstDB::EncodingData::BaseAtomicSt& op_data = InstDB::EncodingData::baseAtomicSt[encoding_index]; 2831 2832 if (isign4 == ENC_OPS2(Reg, Mem)) { 2833 const Mem& m = o1.as<Mem>(); 2834 uint32_t x; 2835 2836 if (!check_gp_type(o0, op_data.reg_type, &x)) 2837 goto InvalidInstruction; 2838 2839 if (!check_gp_id(o0, kZR)) 2840 goto InvalidPhysId; 2841 2842 opcode.reset(op_data.opcode()); 2843 opcode.add_imm(x, op_data.x_offset); 2844 opcode.add_reg(o0, 16); 2845 opcode.add_reg(Gp::kIdZr, 0); 2846 2847 rm_rel = &m; 2848 goto EmitOp_MemBaseNoImm_Rn5; 2849 } 2850 2851 break; 2852 } 2853 2854 case InstDB::kEncodingBaseAtomicCasp: { 2855 const InstDB::EncodingData::BaseAtomicCasp& op_data = InstDB::EncodingData::baseAtomicCasp[encoding_index]; 2856 const Operand_& o4 = op_ext[EmitterUtils::kOp4]; 2857 2858 if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg) && o4.is_mem()) { 2859 const Mem& m = o4.as<Mem>(); 2860 uint32_t x; 2861 2862 if (!check_gp_type(o0, op_data.reg_type, &x)) 2863 goto InvalidInstruction; 2864 2865 if (!check_signature(o0, o1, o2, o3)) 2866 goto InvalidInstruction; 2867 2868 if (!check_even(o0, o2) || !check_gp_id(o0, o2, kZR)) 2869 goto InvalidPhysId; 2870 2871 if (!check_consecutive(o0, o1) || !check_consecutive(o2, o3)) 2872 goto InvalidPhysId; 2873 2874 opcode.reset(op_data.opcode()); 2875 opcode.add_imm(x, op_data.x_offset); 2876 opcode.add_reg(o0, 16); 2877 opcode.add_reg(o2, 0); 2878 2879 rm_rel = &m; 2880 goto EmitOp_MemBaseNoImm_Rn5; 2881 } 2882 2883 break; 2884 } 2885 2886 // ------------------------------------------------------------------------ 2887 // [FSimd - Instructions] 2888 // ------------------------------------------------------------------------ 2889 2890 case InstDB::kEncodingFSimdSV: { 2891 const InstDB::EncodingData::FSimdSV& op_data = InstDB::EncodingData::fSimdSV[encoding_index]; 2892 2893 if (isign4 == ENC_OPS2(Reg, Reg)) { 2894 uint32_t q = diff(o1.as<Reg>().reg_type(), RegType::kVec64); 2895 if (q > 1) 2896 goto InvalidInstruction; 2897 2898 if (o0.as<Vec>().has_element_type()) 2899 goto InvalidInstruction; 2900 2901 // This operation is only defined for: 2902 // hD, vS.{4|8}h (16-bit) 2903 // sD, vS.4s (32-bit) 2904 uint32_t sz = diff(o0.as<Reg>().reg_type(), RegType::kVec16); 2905 uint32_t element_sz = diff(o1.as<Vec>().element_type(), VecElementType::kH); 2906 2907 // Size greater than 1 means 64-bit elements, not supported. 2908 if ((sz | element_sz) > 1 || sz != element_sz) 2909 goto InvalidInstruction; 2910 2911 // Size 1 (32-bit float) requires at least 4 elements. 2912 if (sz && !q) 2913 goto InvalidInstruction; 2914 2915 // Bit flipping according to sz. 2916 static const uint32_t sz_bits_table[] = { B(29), 0 }; 2917 2918 opcode.reset(op_data.opcode << 10); 2919 opcode ^= sz_bits_table[sz]; 2920 opcode.add_imm(q, 30); 2921 goto EmitOp_Rd0_Rn5; 2922 } 2923 2924 break; 2925 } 2926 2927 case InstDB::kEncodingFSimdVV: { 2928 const InstDB::EncodingData::FSimdVV& op_data = InstDB::EncodingData::fSimdVV[encoding_index]; 2929 2930 if (isign4 == ENC_OPS2(Reg, Reg)) { 2931 if (!match_signature(o0, o1, inst_flags)) 2932 goto InvalidInstruction; 2933 2934 if (!pick_fp_opcode(o0.as<Vec>(), op_data.scalar_op(), op_data.scalar_hf(), op_data.vector_op(), op_data.vector_hf(), &opcode)) 2935 goto InvalidInstruction; 2936 2937 goto EmitOp_Rd0_Rn5; 2938 } 2939 2940 break; 2941 } 2942 2943 case InstDB::kEncodingFSimdVVV: { 2944 const InstDB::EncodingData::FSimdVVV& op_data = InstDB::EncodingData::fSimdVVV[encoding_index]; 2945 2946 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 2947 if (!match_signature(o0, o1, o2, inst_flags)) 2948 goto InvalidInstruction; 2949 2950 if (!pick_fp_opcode(o0.as<Vec>(), op_data.scalar_op(), op_data.scalar_hf(), op_data.vector_op(), op_data.vector_hf(), &opcode)) 2951 goto InvalidInstruction; 2952 2953 goto EmitOp_Rd0_Rn5_Rm16; 2954 } 2955 2956 break; 2957 } 2958 2959 case InstDB::kEncodingFSimdVVVe: { 2960 const InstDB::EncodingData::FSimdVVVe& op_data = InstDB::EncodingData::fSimdVVVe[encoding_index]; 2961 2962 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 2963 if (!o2.as<Vec>().has_element_index()) { 2964 if (!match_signature(o0, o1, o2, inst_flags)) 2965 goto InvalidInstruction; 2966 2967 if (!pick_fp_opcode(o0.as<Vec>(), op_data.scalar_op(), op_data.scalar_hf(), op_data.vector_op(), op_data.vector_hf(), &opcode)) 2968 goto InvalidInstruction; 2969 2970 goto EmitOp_Rd0_Rn5_Rm16; 2971 } 2972 else { 2973 if (!match_signature(o0, o1, inst_flags)) 2974 goto InvalidInstruction; 2975 2976 uint32_t q = o1.as<Reg>().is_vec128(); 2977 uint32_t sz; 2978 2979 if (!pick_fp_opcode(o0.as<Vec>(), op_data.element_scalar_op(), InstDB::kHF_D, op_data.element_vector_op(), InstDB::kHF_D, &opcode, &sz)) 2980 goto InvalidInstruction; 2981 2982 if (sz == 0 && o2.as<Reg>().id() > 15) 2983 goto InvalidPhysId; 2984 2985 uint32_t element_index = o2.as<Vec>().element_index(); 2986 if (element_index > (7u >> sz)) 2987 goto InvalidElementIndex; 2988 2989 uint32_t hlm = element_index << sz; 2990 opcode.add_imm(q, 30); 2991 opcode.add_imm(hlm & 3u, 20); 2992 opcode.add_imm(hlm >> 2, 11); 2993 goto EmitOp_Rd0_Rn5_Rm16; 2994 } 2995 } 2996 2997 break; 2998 } 2999 3000 case InstDB::kEncodingFSimdVVVV: { 3001 const InstDB::EncodingData::FSimdVVVV& op_data = InstDB::EncodingData::fSimdVVVV[encoding_index]; 3002 3003 if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) { 3004 if (!match_signature(o0, o1, o2, o3, inst_flags)) 3005 goto InvalidInstruction; 3006 3007 if (!pick_fp_opcode(o0.as<Vec>(), op_data.scalar_op(), op_data.scalar_hf(), op_data.vector_op(), op_data.vector_hf(), &opcode)) 3008 goto InvalidInstruction; 3009 3010 goto EmitOp_Rd0_Rn5_Rm16_Ra10; 3011 } 3012 3013 break; 3014 } 3015 3016 case InstDB::kEncodingSimdFcadd: { 3017 const InstDB::EncodingData::SimdFcadd& op_data = InstDB::EncodingData::simdFcadd[encoding_index]; 3018 3019 if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) { 3020 if (!check_signature(o0, o1, o2) || o0.as<Vec>().has_element_index()) 3021 goto InvalidInstruction; 3022 3023 uint32_t q = diff(o0.as<Reg>().reg_type(), RegType::kVec64); 3024 if (q > 1) 3025 goto InvalidInstruction; 3026 3027 uint32_t sz = diff(o0.as<Vec>().element_type(), VecElementType::kB); 3028 if (sz == 0 || sz > 3) 3029 goto InvalidInstruction; 3030 3031 // 0 <- 90deg. 3032 // 1 <- 270deg. 3033 uint32_t rot = 0; 3034 if (o3.as<Imm>().value() == 270) 3035 rot = 1; 3036 else if (o3.as<Imm>().value() != 90) 3037 goto InvalidImmediate; 3038 3039 opcode.reset(op_data.opcode()); 3040 opcode.add_imm(q, 30); 3041 opcode.add_imm(sz, 22); 3042 opcode.add_imm(rot, 12); 3043 goto EmitOp_Rd0_Rn5_Rm16; 3044 } 3045 3046 break; 3047 } 3048 3049 case InstDB::kEncodingSimdFccmpFccmpe: { 3050 const InstDB::EncodingData::SimdFccmpFccmpe& op_data = InstDB::EncodingData::simdFccmpFccmpe[encoding_index]; 3051 3052 if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) { 3053 uint32_t sz = diff(o0.as<Reg>().reg_type(), RegType::kVec16); 3054 if (sz > 2) 3055 goto InvalidInstruction; 3056 3057 if (!check_signature(o0, o1) || o0.as<Vec>().has_element_type()) 3058 goto InvalidInstruction; 3059 3060 uint64_t nzcv = o2.as<Imm>().value_as<uint64_t>(); 3061 uint64_t cond = o3.as<Imm>().value_as<uint64_t>(); 3062 3063 if ((nzcv | cond) > 0xFu) 3064 goto InvalidImmediate; 3065 3066 uint32_t type = (sz - 1) & 0x3u; 3067 3068 opcode.reset(op_data.opcode()); 3069 opcode.add_imm(type, 22); 3070 opcode.add_imm(cond_code_to_opcode_field(uint32_t(cond)), 12); 3071 opcode.add_imm(nzcv, 0); 3072 3073 goto EmitOp_Rn5_Rm16; 3074 } 3075 3076 break; 3077 } 3078 3079 case InstDB::kEncodingSimdFcm: { 3080 const InstDB::EncodingData::SimdFcm& op_data = InstDB::EncodingData::simdFcm[encoding_index]; 3081 3082 if (isign4 == ENC_OPS3(Reg, Reg, Reg) && op_data.has_register_op()) { 3083 if (!match_signature(o0, o1, o2, inst_flags)) 3084 goto InvalidInstruction; 3085 3086 if (!pick_fp_opcode(o0.as<Vec>(), op_data.register_scalar_op(), op_data.register_scalar_hf(), op_data.register_vector_op(), op_data.register_vector_hf(), &opcode)) 3087 goto InvalidInstruction; 3088 3089 goto EmitOp_Rd0_Rn5_Rm16; 3090 } 3091 3092 if (isign4 == ENC_OPS3(Reg, Reg, Imm) && op_data.has_zero_op()) { 3093 if (!check_signature(o0, o1)) 3094 goto InvalidInstruction; 3095 3096 if (o2.as<Imm>().value() != 0 || o2.as<Imm>().predicate() != 0) 3097 goto InvalidImmediate; 3098 3099 if (!pick_fp_opcode(o0.as<Vec>(), op_data.zero_scalar_op(), InstDB::kHF_B, op_data.zero_vector_op(), InstDB::kHF_B, &opcode)) 3100 goto InvalidInstruction; 3101 3102 goto EmitOp_Rd0_Rn5; 3103 } 3104 3105 break; 3106 } 3107 3108 case InstDB::kEncodingSimdFcmla: { 3109 const InstDB::EncodingData::SimdFcmla& op_data = InstDB::EncodingData::simdFcmla[encoding_index]; 3110 3111 if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) { 3112 if (!check_signature(o0, o1)) 3113 goto InvalidInstruction; 3114 3115 uint32_t q = diff(o0.as<Reg>().reg_type(), RegType::kVec64); 3116 if (q > 1) 3117 goto InvalidInstruction; 3118 3119 uint32_t sz = diff(o0.as<Vec>().element_type(), VecElementType::kB); 3120 if (sz == 0 || sz > 3) 3121 goto InvalidInstruction; 3122 3123 uint32_t rot = 0; 3124 switch (o3.as<Imm>().value()) { 3125 case 0 : rot = 0; break; 3126 case 90 : rot = 1; break; 3127 case 180: rot = 2; break; 3128 case 270: rot = 3; break; 3129 default: 3130 goto InvalidImmediate; 3131 } 3132 3133 if (!o2.as<Vec>().has_element_index()) { 3134 if (!check_signature(o1, o2)) 3135 goto InvalidInstruction; 3136 3137 opcode.reset(op_data.regular_op()); 3138 opcode.add_imm(q, 30); 3139 opcode.add_imm(sz, 22); 3140 opcode.add_imm(rot, 11); 3141 goto EmitOp_Rd0_Rn5_Rm16; 3142 } 3143 else { 3144 if (o0.as<Vec>().element_type() != o2.as<Vec>().element_type()) 3145 goto InvalidInstruction; 3146 3147 // Only allowed vectors are: 4H, 8H, and 4S. 3148 if (!(sz == 1 || (q == 1 && sz == 2))) 3149 goto InvalidInstruction; 3150 3151 // Element index ranges: 3152 // 4H - ElementIndex[0..1] (index 2..3 is UNDEFINED). 3153 // 8H - ElementIndex[0..3]. 3154 // 4S - ElementIndex[0..1]. 3155 uint32_t element_index = o2.as<Vec>().element_index(); 3156 uint32_t hl_field_shift = sz == 1 ? 0u : 1u; 3157 uint32_t max_element_index = q == 1 && sz == 1 ? 3u : 1u; 3158 3159 if (element_index > max_element_index) 3160 goto InvalidElementIndex; 3161 3162 uint32_t hl = element_index << hl_field_shift; 3163 3164 opcode.reset(op_data.element_op()); 3165 opcode.add_imm(q, 30); 3166 opcode.add_imm(sz, 22); 3167 opcode.add_imm(hl & 1u, 21); // L field. 3168 opcode.add_imm(hl >> 1, 11); // H field. 3169 opcode.add_imm(rot, 13); 3170 goto EmitOp_Rd0_Rn5_Rm16; 3171 } 3172 } 3173 3174 break; 3175 } 3176 3177 case InstDB::kEncodingSimdFcmpFcmpe: { 3178 const InstDB::EncodingData::SimdFcmpFcmpe& op_data = InstDB::EncodingData::simdFcmpFcmpe[encoding_index]; 3179 3180 uint32_t sz = diff(o0.as<Reg>().reg_type(), RegType::kVec16); 3181 uint32_t type = (sz - 1) & 0x3u; 3182 3183 if (sz > 2) 3184 goto InvalidInstruction; 3185 3186 if (o0.as<Vec>().has_element_type()) 3187 goto InvalidInstruction; 3188 3189 opcode.reset(op_data.opcode()); 3190 opcode.add_imm(type, 22); 3191 3192 if (isign4 == ENC_OPS2(Reg, Reg)) { 3193 if (!check_signature(o0, o1)) 3194 goto InvalidInstruction; 3195 3196 goto EmitOp_Rn5_Rm16; 3197 } 3198 3199 if (isign4 == ENC_OPS2(Reg, Imm)) { 3200 if (o1.as<Imm>().value() != 0 || o1.as<Imm>().predicate() != 0) 3201 goto InvalidInstruction; 3202 3203 opcode |= B(3); 3204 goto EmitOp_Rn5; 3205 } 3206 3207 break; 3208 } 3209 3210 case InstDB::kEncodingSimdFcsel: { 3211 if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) { 3212 if (!check_signature(o0, o1, o2)) 3213 goto InvalidInstruction; 3214 3215 uint32_t sz = diff(o0.as<Reg>().reg_type(), RegType::kVec16); 3216 uint32_t type = (sz - 1) & 0x3u; 3217 3218 if (sz > 2 || o0.as<Vec>().has_element_type()) 3219 goto InvalidInstruction; 3220 3221 uint64_t cond = o3.as<Imm>().value_as<uint64_t>(); 3222 if (cond > 0xFu) 3223 goto InvalidImmediate; 3224 3225 opcode.reset(0b00011110001000000000110000000000); 3226 opcode.add_imm(type, 22); 3227 opcode.add_imm(cond_code_to_opcode_field(uint32_t(cond)), 12); 3228 goto EmitOp_Rd0_Rn5_Rm16; 3229 } 3230 3231 break; 3232 } 3233 3234 case InstDB::kEncodingSimdFcvt: { 3235 if (isign4 == ENC_OPS2(Reg, Reg)) { 3236 uint32_t dst_sz = diff(o0.as<Reg>().reg_type(), RegType::kVec16); 3237 uint32_t src_sz = diff(o1.as<Reg>().reg_type(), RegType::kVec16); 3238 3239 if ((dst_sz | src_sz) > 3) 3240 goto InvalidInstruction; 3241 3242 if (o0.as<Vec>().has_element_type() || o1.as<Vec>().has_element_type()) 3243 goto InvalidInstruction; 3244 3245 // Table that provides 'type' and 'opc' according to the dst/src combination. 3246 static const uint8_t table[] = { 3247 0xFFu, // H <- H (Invalid). 3248 0x03u, // H <- S (type=00 opc=11). 3249 0x13u, // H <- D (type=01 opc=11). 3250 0xFFu, // H <- Q (Invalid). 3251 0x30u, // S <- H (type=11 opc=00). 3252 0xFFu, // S <- S (Invalid). 3253 0x10u, // S <- D (type=01 opc=00). 3254 0xFFu, // S <- Q (Invalid). 3255 0x31u, // D <- H (type=11 opc=01). 3256 0x01u, // D <- S (type=00 opc=01). 3257 0xFFu, // D <- D (Invalid). 3258 0xFFu, // D <- Q (Invalid). 3259 0xFFu, // Q <- H (Invalid). 3260 0xFFu, // Q <- S (Invalid). 3261 0xFFu, // Q <- D (Invalid). 3262 0xFFu // Q <- Q (Invalid). 3263 }; 3264 3265 uint32_t type_opc = table[(dst_sz << 2) | src_sz]; 3266 opcode.reset(0b0001111000100010010000 << 10); 3267 opcode.add_imm(type_opc >> 4, 22); 3268 opcode.add_imm(type_opc & 15, 15); 3269 goto EmitOp_Rd0_Rn5; 3270 } 3271 3272 break; 3273 } 3274 3275 case InstDB::kEncodingSimdFcvtLN: { 3276 const InstDB::EncodingData::SimdFcvtLN& op_data = InstDB::EncodingData::simdFcvtLN[encoding_index]; 3277 3278 if (isign4 == ENC_OPS2(Reg, Reg)) { 3279 // Scalar form - only FCVTXN. 3280 if (o0.as<Vec>().is_vec32() && o1.as<Vec>().is_vec64()) { 3281 if (!op_data.has_scalar()) 3282 goto InvalidInstruction; 3283 3284 if (o0.as<Vec>().has_element_type() || o1.as<Vec>().has_element_type()) 3285 goto InvalidInstruction; 3286 3287 opcode.reset(op_data.scalar_op()); 3288 opcode |= B(22); // sz bit must be 1, the only supported combination of FCVTXN. 3289 goto EmitOp_Rd0_Rn5; 3290 } 3291 3292 opcode.reset(op_data.vector_op()); 3293 3294 const Vec& rl = (inst_flags & InstDB::kInstFlagLong) ? o0.as<Vec>() : o1.as<Vec>(); 3295 const Vec& rn = (inst_flags & InstDB::kInstFlagLong) ? o1.as<Vec>() : o0.as<Vec>(); 3296 3297 uint32_t q = diff(rn.reg_type(), RegType::kVec64); 3298 if (uint32_t(opcode.has_q()) != q) 3299 goto InvalidInstruction; 3300 3301 if (rl.is_vec_s4() && rn.element_type() == VecElementType::kH && !op_data.is_cvtxn()) { 3302 goto EmitOp_Rd0_Rn5; 3303 } 3304 3305 if (rl.is_vec_d2() && rn.element_type() == VecElementType::kS) { 3306 opcode |= B(22); 3307 goto EmitOp_Rd0_Rn5; 3308 } 3309 } 3310 3311 break; 3312 } 3313 3314 case InstDB::kEncodingSimdFcvtSV: { 3315 const InstDB::EncodingData::SimdFcvtSV& op_data = InstDB::EncodingData::simdFcvtSV[encoding_index]; 3316 3317 // So we can support both IntToFloat and FloatToInt conversions. 3318 const Operand_& op_gp = op_data.is_float_to_int() ? o0 : o1; 3319 const Operand_& op_vec = op_data.is_float_to_int() ? o1 : o0; 3320 3321 if (isign4 == ENC_OPS2(Reg, Reg)) { 3322 if (op_gp.as<Reg>().is_gp() && op_vec.as<Reg>().is_vec()) { 3323 uint32_t x = op_gp.as<Reg>().is_gp64(); 3324 uint32_t type = diff(op_vec.as<Reg>().reg_type(), RegType::kVec16); 3325 3326 if (type > 2u) 3327 goto InvalidInstruction; 3328 3329 type = (type - 1u) & 0x3; 3330 opcode.reset(op_data.general_op()); 3331 opcode.add_imm(type, 22); 3332 opcode.add_imm(x, 31); 3333 goto EmitOp_Rd0_Rn5; 3334 } 3335 3336 if (o0.as<Reg>().is_vec() && o1.as<Reg>().is_vec()) { 3337 if (!check_signature(o0, o1)) 3338 goto InvalidInstruction; 3339 3340 if (!pick_fp_opcode(o0.as<Vec>(), op_data.scalar_int_op(), InstDB::kHF_B, op_data.vector_int_op(), InstDB::kHF_B, &opcode)) 3341 goto InvalidInstruction; 3342 3343 goto EmitOp_Rd0_Rn5; 3344 } 3345 } 3346 3347 if (isign4 == ENC_OPS3(Reg, Reg, Imm) && op_data.is_fixed_point()) { 3348 if (o2.as<Imm>().value_as<uint64_t>() >= 64) 3349 goto InvalidInstruction; 3350 3351 uint32_t scale = o2.as<Imm>().value_as<uint32_t>(); 3352 if (scale == 0) 3353 goto InvalidInstruction; 3354 3355 if (op_gp.as<Reg>().is_gp() && op_vec.as<Reg>().is_vec()) { 3356 uint32_t x = op_gp.as<Reg>().is_gp64(); 3357 uint32_t type = diff(op_vec.as<Reg>().reg_type(), RegType::kVec16); 3358 3359 uint32_t scale_limit = 32u << x; 3360 if (scale > scale_limit) 3361 goto InvalidInstruction; 3362 3363 type = (type - 1u) & 0x3; 3364 opcode.reset(op_data.general_op() ^ B(21)); 3365 opcode.add_imm(type, 22); 3366 opcode.add_imm(x, 31); 3367 opcode.add_imm(64u - scale, 10); 3368 goto EmitOp_Rd0_Rn5; 3369 } 3370 3371 if (o0.as<Reg>().is_vec() && o1.as<Reg>().is_vec()) { 3372 if (!check_signature(o0, o1)) 3373 goto InvalidInstruction; 3374 3375 uint32_t sz; 3376 if (!pick_fp_opcode(o0.as<Vec>(), op_data.scalar_fp_op(), InstDB::kHF_0, op_data.vector_fp_op(), InstDB::kHF_0, &opcode, &sz)) 3377 goto InvalidInstruction; 3378 3379 uint32_t scale_limit = 16u << sz; 3380 if (scale > scale_limit) 3381 goto InvalidInstruction; 3382 3383 uint32_t imm = Support::neg(scale) & Support::lsb_mask<uint32_t>(sz + 4 + 1); 3384 opcode.add_imm(imm, 16); 3385 goto EmitOp_Rd0_Rn5; 3386 } 3387 } 3388 3389 break; 3390 } 3391 3392 case InstDB::kEncodingSimdFmlal: { 3393 const InstDB::EncodingData::SimdFmlal& op_data = InstDB::EncodingData::simdFmlal[encoding_index]; 3394 3395 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 3396 uint32_t q = diff(o0.as<Reg>().reg_type(), RegType::kVec64); 3397 uint32_t q_is_optional = op_data.optional_q(); 3398 3399 if (q_is_optional) { 3400 // This instruction works with either 64-bit or 128-bit registers, 3401 // encoded by Q bit. 3402 if (q > 1) 3403 goto InvalidInstruction; 3404 } 3405 else { 3406 // This instruction requires 128-bit vector registers. 3407 if (q != 1) 3408 goto InvalidInstruction; 3409 3410 // The instruction is ehtier B (bottom) or T (top), which is part of 3411 // the opcode, which uses Q bit, so we have to clear it explicitly. 3412 q = 0; 3413 } 3414 3415 if (uint32_t(o0.as<Reg>().reg_type()) != uint32_t(o1.as<Reg>().reg_type()) + q_is_optional || 3416 uint32_t(o0.as<Vec>().element_type()) != op_data.ta || 3417 uint32_t(o1.as<Vec>().element_type()) != op_data.tb) 3418 goto InvalidInstruction; 3419 3420 if (!o2.as<Vec>().has_element_index()) { 3421 if (!check_signature(o1, o2)) 3422 goto InvalidInstruction; 3423 3424 opcode.reset(op_data.vector_op()); 3425 opcode.add_imm(q, 30); 3426 goto EmitOp_Rd0_Rn5_Rm16; 3427 } 3428 else { 3429 if (uint32_t(o2.as<Vec>().element_type()) != op_data.tElement) 3430 goto InvalidInstruction; 3431 3432 if (o2.as<Reg>().id() > 15) 3433 goto InvalidPhysId; 3434 3435 uint32_t element_index = o2.as<Vec>().element_index(); 3436 if (element_index > 7u) 3437 goto InvalidElementIndex; 3438 3439 opcode.reset(op_data.element_op()); 3440 opcode.add_imm(q, 30); 3441 opcode.add_imm(element_index & 3u, 20); 3442 opcode.add_imm(element_index >> 2, 11); 3443 goto EmitOp_Rd0_Rn5_Rm16; 3444 } 3445 } 3446 3447 break; 3448 } 3449 3450 case InstDB::kEncodingSimdFmov: { 3451 if (isign4 == ENC_OPS2(Reg, Reg)) { 3452 // FMOV Gp <-> Vec opcode: 3453 opcode.reset(0b00011110001001100000000000000000); 3454 3455 if (o0.as<Reg>().is_gp() && o1.as<Reg>().is_vec()) { 3456 // FMOV Wd, Hn (sf=0 type=11 rmode=00 op=110) 3457 // FMOV Xd, Hn (sf=1 type=11 rmode=00 op=110) 3458 // FMOV Wd, Sn (sf=0 type=00 rmode=00 op=110) 3459 // FMOV Xd, Dn (sf=1 type=11 rmode=00 op=110) 3460 // FMOV Xd, Vn.d[1] (sf=1 type=10 rmode=01 op=110) 3461 uint32_t x = o0.as<Reg>().is_gp64(); 3462 uint32_t sz = diff(o1.as<Reg>().reg_type(), RegType::kVec16); 3463 3464 uint32_t type = (sz - 1) & 0x3u; 3465 uint32_t r_mode_op = 0b00110; 3466 3467 if (o1.as<Vec>().has_element_index()) { 3468 // Special case. 3469 if (!x || !o1.as<Vec>().is_vec_d2() || o1.as<Vec>().element_index() != 1) 3470 goto InvalidInstruction; 3471 type = 0b10; 3472 r_mode_op = 0b01110; 3473 } 3474 else { 3475 // Must be scalar. 3476 if (sz > 2) 3477 goto InvalidInstruction; 3478 3479 if (o1.as<Vec>().has_element_type()) 3480 goto InvalidInstruction; 3481 3482 if (o1.as<Vec>().is_vec32() && x) 3483 goto InvalidInstruction; 3484 3485 if (o1.as<Vec>().is_vec64() && !x) 3486 goto InvalidInstruction; 3487 } 3488 3489 opcode.add_imm(x, 31); 3490 opcode.add_imm(type, 22); 3491 opcode.add_imm(r_mode_op, 16); 3492 goto EmitOp_Rd0_Rn5; 3493 } 3494 3495 if (o0.as<Reg>().is_vec() && o1.as<Reg>().is_gp()) { 3496 // FMOV Hd, Wn (sf=0 type=11 rmode=00 op=111) 3497 // FMOV Hd, Xn (sf=1 type=11 rmode=00 op=111) 3498 // FMOV Sd, Wn (sf=0 type=00 rmode=00 op=111) 3499 // FMOV Dd, Xn (sf=1 type=11 rmode=00 op=111) 3500 // FMOV Vd.d[1], Xn (sf=1 type=10 rmode=01 op=111) 3501 uint32_t x = o1.as<Reg>().is_gp64(); 3502 uint32_t sz = diff(o0.as<Reg>().reg_type(), RegType::kVec16); 3503 3504 uint32_t type = (sz - 1) & 0x3u; 3505 uint32_t r_mode_op = 0b00111; 3506 3507 if (o0.as<Vec>().has_element_index()) { 3508 // Special case. 3509 if (!x || !o0.as<Vec>().is_vec_d2() || o0.as<Vec>().element_index() != 1) 3510 goto InvalidInstruction; 3511 type = 0b10; 3512 r_mode_op = 0b01111; 3513 } 3514 else { 3515 // Must be scalar. 3516 if (sz > 2) 3517 goto InvalidInstruction; 3518 3519 if (o0.as<Vec>().has_element_type()) 3520 goto InvalidInstruction; 3521 3522 if (o0.as<Vec>().is_vec32() && x) 3523 goto InvalidInstruction; 3524 3525 if (o0.as<Vec>().is_vec64() && !x) 3526 goto InvalidInstruction; 3527 } 3528 3529 opcode.add_imm(x, 31); 3530 opcode.add_imm(type, 22); 3531 opcode.add_imm(r_mode_op, 16); 3532 goto EmitOp_Rd0_Rn5; 3533 } 3534 3535 if (check_signature(o0, o1)) { 3536 uint32_t sz = diff(o0.as<Reg>().reg_type(), RegType::kVec16); 3537 if (sz > 2) 3538 goto InvalidInstruction; 3539 3540 if (o0.as<Vec>().has_element_type()) 3541 goto InvalidInstruction; 3542 3543 uint32_t type = (sz - 1) & 0x3; 3544 opcode.reset(0b00011110001000000100000000000000); 3545 opcode.add_imm(type, 22); 3546 goto EmitOp_Rd0_Rn5; 3547 } 3548 } 3549 3550 if (isign4 == ENC_OPS2(Reg, Imm)) { 3551 if (o0.as<Reg>().is_vec()) { 3552 double fp_value; 3553 if (o1.as<Imm>().is_double()) 3554 fp_value = o1.as<Imm>().value_as<double>(); 3555 else if (o1.as<Imm>().is_int32()) 3556 fp_value = o1.as<Imm>().value_as<int32_t>(); 3557 else 3558 goto InvalidImmediate; 3559 3560 if (!Utils::is_fp64_imm8(fp_value)) 3561 goto InvalidImmediate; 3562 3563 uint32_t imm8 = Utils::encode_fp64_to_imm8(fp_value); 3564 if (!o0.as<Vec>().has_element_type()) { 3565 // FMOV (scalar, immediate). 3566 uint32_t sz = diff(o0.as<Reg>().reg_type(), RegType::kVec16); 3567 uint32_t type = (sz - 1u) & 0x3u; 3568 3569 if (sz > 2) 3570 goto InvalidInstruction; 3571 3572 opcode.reset(0b00011110001000000001000000000000); 3573 opcode.add_imm(type, 22); 3574 opcode.add_imm(imm8, 13); 3575 goto EmitOp_Rd0; 3576 } 3577 else { 3578 uint32_t q = diff(o0.as<Vec>().reg_type(), RegType::kVec64); 3579 uint32_t sz = diff(o0.as<Vec>().element_type(), VecElementType::kH); 3580 3581 if (q > 1 || sz > 2) 3582 goto InvalidInstruction; 3583 3584 static const uint32_t sz_bits_table[3] = { B(11), B(0), B(29) }; 3585 opcode.reset(0b00001111000000001111010000000000); 3586 opcode ^= sz_bits_table[sz]; 3587 opcode.add_imm(q, 30); 3588 opcode.add_imm(imm8 >> 5, 16); 3589 opcode.add_imm(imm8 & 31, 5); 3590 goto EmitOp_Rd0; 3591 } 3592 } 3593 } 3594 3595 break; 3596 } 3597 3598 case InstDB::kEncodingFSimdPair: { 3599 const InstDB::EncodingData::FSimdPair& op_data = InstDB::EncodingData::fSimdPair[encoding_index]; 3600 3601 if (isign4 == ENC_OPS2(Reg, Reg)) { 3602 // This operation is only defined for: 3603 // hD, vS.2h (16-bit) 3604 // sD, vS.2s (32-bit) 3605 // dD, vS.2d (64-bit) 3606 uint32_t sz = diff(o0.as<Reg>().reg_type(), RegType::kVec16); 3607 if (sz > 2) 3608 goto InvalidInstruction; 3609 3610 static const uint32_t szSignatures[3] = { 3611 RegTraits<RegType::kVec32>::kSignature | (Vec::kSignatureElementH), 3612 RegTraits<RegType::kVec64>::kSignature | (Vec::kSignatureElementS), 3613 RegTraits<RegType::kVec128>::kSignature | (Vec::kSignatureElementD) 3614 }; 3615 3616 if (o1.signature() != szSignatures[sz]) 3617 goto InvalidInstruction; 3618 3619 static const uint32_t sz_bits_table[] = { B(29), 0, B(22) }; 3620 opcode.reset(op_data.scalar_op()); 3621 opcode ^= sz_bits_table[sz]; 3622 goto EmitOp_Rd0_Rn5; 3623 } 3624 3625 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 3626 if (!check_signature(o0, o1, o2)) 3627 goto InvalidInstruction; 3628 3629 uint32_t q = diff(o0.as<Reg>().reg_type(), RegType::kVec64); 3630 if (q > 1) 3631 goto InvalidInstruction; 3632 3633 uint32_t sz = diff(o0.as<Vec>().element_type(), VecElementType::kH); 3634 if (sz > 2) 3635 goto InvalidInstruction; 3636 3637 static const uint32_t sz_bits_table[3] = { B(22) | B(21) | B(15) | B(14), 0, B(22) }; 3638 opcode.reset(op_data.vector_op()); 3639 opcode ^= sz_bits_table[sz]; 3640 opcode.add_imm(q, 30); 3641 goto EmitOp_Rd0_Rn5_Rm16; 3642 } 3643 3644 break; 3645 } 3646 3647 // ------------------------------------------------------------------------ 3648 // [ISimd - Instructions] 3649 // ------------------------------------------------------------------------ 3650 3651 case InstDB::kEncodingISimdSV: { 3652 const InstDB::EncodingData::ISimdSV& op_data = InstDB::EncodingData::iSimdSV[encoding_index]; 3653 3654 if (isign4 == ENC_OPS2(Reg, Reg)) { 3655 // The first destination operand is scalar, which matches element-type of source vectors. 3656 uint32_t L = (inst_flags & InstDB::kInstFlagLong) != 0; 3657 if (diff(o0.as<Vec>().reg_type(), RegType::kVec8) != diff(o1.as<Vec>().element_type(), VecElementType::kB) + L) 3658 goto InvalidInstruction; 3659 3660 SizeOp size_op = element_type_to_size_op(op_data.vec_op_type, o1.as<Reg>().reg_type(), o1.as<Vec>().element_type()); 3661 if (!size_op.is_valid()) 3662 goto InvalidInstruction; 3663 3664 opcode.reset(op_data.opcode()); 3665 opcode.add_imm(size_op.q(), 30); 3666 opcode.add_imm(size_op.size(), 22); 3667 goto EmitOp_Rd0_Rn5; 3668 } 3669 3670 break; 3671 } 3672 3673 case InstDB::kEncodingISimdVV: { 3674 const InstDB::EncodingData::ISimdVV& op_data = InstDB::EncodingData::iSimdVV[encoding_index]; 3675 3676 if (isign4 == ENC_OPS2(Reg, Reg)) { 3677 const Operand_& sop = significant_simd_op(o0, o1, inst_flags); 3678 if (!match_signature(o0, o1, inst_flags)) 3679 goto InvalidInstruction; 3680 3681 SizeOp size_op = element_type_to_size_op(op_data.vec_op_type, sop.as<Reg>().reg_type(), sop.as<Vec>().element_type()); 3682 if (!size_op.is_valid()) 3683 goto InvalidInstruction; 3684 3685 opcode.reset(op_data.opcode()); 3686 opcode.add_imm(size_op.qs(), 30); 3687 opcode.add_imm(size_op.scalar(), 28); 3688 opcode.add_imm(size_op.size(), 22); 3689 goto EmitOp_Rd0_Rn5; 3690 } 3691 3692 break; 3693 } 3694 3695 case InstDB::kEncodingISimdVVx: { 3696 const InstDB::EncodingData::ISimdVVx& op_data = InstDB::EncodingData::iSimdVVx[encoding_index]; 3697 3698 if (isign4 == ENC_OPS2(Reg, Reg)) { 3699 if (o0.signature() != op_data.op0_signature || 3700 o1.signature() != op_data.op1_signature) 3701 goto InvalidInstruction; 3702 3703 opcode.reset(op_data.opcode()); 3704 goto EmitOp_Rd0_Rn5; 3705 } 3706 3707 break; 3708 } 3709 3710 case InstDB::kEncodingISimdVVV: { 3711 const InstDB::EncodingData::ISimdVVV& op_data = InstDB::EncodingData::iSimdVVV[encoding_index]; 3712 3713 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 3714 const Operand_& sop = significant_simd_op(o0, o1, inst_flags); 3715 if (!match_signature(o0, o1, o2, inst_flags)) 3716 goto InvalidInstruction; 3717 3718 SizeOp size_op = element_type_to_size_op(op_data.vec_op_type, sop.as<Reg>().reg_type(), sop.as<Vec>().element_type()); 3719 if (!size_op.is_valid()) 3720 goto InvalidInstruction; 3721 3722 opcode.reset(op_data.opcode()); 3723 opcode.add_imm(size_op.qs(), 30); 3724 opcode.add_imm(size_op.scalar(), 28); 3725 opcode.add_imm(size_op.size(), 22); 3726 goto EmitOp_Rd0_Rn5_Rm16; 3727 } 3728 3729 break; 3730 } 3731 3732 case InstDB::kEncodingISimdVVVx: { 3733 const InstDB::EncodingData::ISimdVVVx& op_data = InstDB::EncodingData::iSimdVVVx[encoding_index]; 3734 3735 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 3736 if (o0.signature() != op_data.op0_signature || 3737 o1.signature() != op_data.op1_signature || 3738 o2.signature() != op_data.op2_signature) 3739 goto InvalidInstruction; 3740 3741 opcode.reset(op_data.opcode()); 3742 goto EmitOp_Rd0_Rn5_Rm16; 3743 } 3744 3745 break; 3746 } 3747 3748 case InstDB::kEncodingISimdWWV: { 3749 // Special case for wide add/sub [s|b][add|sub][w]{2}. 3750 const InstDB::EncodingData::ISimdWWV& op_data = InstDB::EncodingData::iSimdWWV[encoding_index]; 3751 3752 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 3753 SizeOp size_op = element_type_to_size_op(op_data.vec_op_type, o2.as<Reg>().reg_type(), o2.as<Vec>().element_type()); 3754 if (!size_op.is_valid()) 3755 goto InvalidInstruction; 3756 3757 if (!check_signature(o0, o1) || !o0.as<Reg>().is_vec128() || uint32_t(o0.as<Vec>().element_type()) != uint32_t(o2.as<Vec>().element_type()) + 1u) 3758 goto InvalidInstruction; 3759 3760 opcode.reset(op_data.opcode()); 3761 opcode.add_imm(size_op.qs(), 30); 3762 opcode.add_imm(size_op.scalar(), 28); 3763 opcode.add_imm(size_op.size(), 22); 3764 goto EmitOp_Rd0_Rn5_Rm16; 3765 } 3766 3767 break; 3768 } 3769 3770 case InstDB::kEncodingISimdVVVe: { 3771 const InstDB::EncodingData::ISimdVVVe& op_data = InstDB::EncodingData::iSimdVVVe[encoding_index]; 3772 3773 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 3774 const Operand_& sop = significant_simd_op(o0, o1, inst_flags); 3775 if (!match_signature(o0, o1, inst_flags)) 3776 goto InvalidInstruction; 3777 3778 if (!o2.as<Vec>().has_element_index()) { 3779 SizeOp size_op = element_type_to_size_op(op_data.regular_vec_type, sop.as<Reg>().reg_type(), sop.as<Vec>().element_type()); 3780 if (!size_op.is_valid()) 3781 goto InvalidInstruction; 3782 3783 if (!check_signature(o1, o2)) 3784 goto InvalidInstruction; 3785 3786 opcode.reset(uint32_t(op_data.regular_op) << 10); 3787 opcode.add_imm(size_op.qs(), 30); 3788 opcode.add_imm(size_op.scalar(), 28); 3789 opcode.add_imm(size_op.size(), 22); 3790 goto EmitOp_Rd0_Rn5_Rm16; 3791 } 3792 else { 3793 SizeOp size_op = element_type_to_size_op(op_data.element_vec_type, sop.as<Reg>().reg_type(), sop.as<Vec>().element_type()); 3794 if (!size_op.is_valid()) 3795 goto InvalidInstruction; 3796 3797 uint32_t element_index = o2.as<Vec>().element_index(); 3798 LMHImm lmh; 3799 3800 if (!encode_lmh(size_op.size(), element_index, Out(lmh))) 3801 goto InvalidElementIndex; 3802 3803 if (o2.as<Reg>().id() > lmh.max_rm_id) 3804 goto InvalidPhysId; 3805 3806 opcode.reset(uint32_t(op_data.element_op) << 10); 3807 opcode.add_imm(size_op.q(), 30); 3808 opcode.add_imm(size_op.size(), 22); 3809 opcode.add_imm(lmh.lm, 20); 3810 opcode.add_imm(lmh.h, 11); 3811 goto EmitOp_Rd0_Rn5_Rm16; 3812 } 3813 } 3814 3815 break; 3816 } 3817 3818 case InstDB::kEncodingISimdVVVI: { 3819 const InstDB::EncodingData::ISimdVVVI& op_data = InstDB::EncodingData::iSimdVVVI[encoding_index]; 3820 3821 if (isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) { 3822 const Operand_& sop = significant_simd_op(o0, o1, inst_flags); 3823 if (!match_signature(o0, o1, o2, inst_flags)) 3824 goto InvalidInstruction; 3825 3826 SizeOp size_op = element_type_to_size_op(op_data.vec_op_type, sop.as<Reg>().reg_type(), sop.as<Vec>().element_type()); 3827 if (!size_op.is_valid()) 3828 goto InvalidInstruction; 3829 3830 uint64_t imm_value = o3.as<Imm>().value_as<uint64_t>(); 3831 uint32_t imm_size = op_data.imm_size; 3832 3833 if (op_data.imm64_has_one_bit_less && !size_op.q()) 3834 imm_size--; 3835 3836 uint32_t immMax = 1u << imm_size; 3837 if (imm_value >= immMax) 3838 goto InvalidImmediate; 3839 3840 opcode.reset(op_data.opcode()); 3841 opcode.add_imm(size_op.qs(), 30); 3842 opcode.add_imm(size_op.scalar(), 28); 3843 opcode.add_imm(size_op.size(), 22); 3844 opcode.add_imm(imm_value, op_data.imm_shift); 3845 goto EmitOp_Rd0_Rn5_Rm16; 3846 } 3847 3848 break; 3849 } 3850 3851 case InstDB::kEncodingISimdVVVV: { 3852 const InstDB::EncodingData::ISimdVVVV& op_data = InstDB::EncodingData::iSimdVVVV[encoding_index]; 3853 3854 if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) { 3855 const Operand_& sop = significant_simd_op(o0, o1, inst_flags); 3856 if (!match_signature(o0, o1, o2, o3, inst_flags)) 3857 goto InvalidInstruction; 3858 3859 SizeOp size_op = element_type_to_size_op(op_data.vec_op_type, sop.as<Reg>().reg_type(), sop.as<Vec>().element_type()); 3860 if (!size_op.is_valid()) 3861 goto InvalidInstruction; 3862 3863 opcode.reset(uint32_t(op_data.opcode) << 10); 3864 opcode.add_imm(size_op.qs(), 30); 3865 opcode.add_imm(size_op.scalar(), 28); 3866 opcode.add_imm(size_op.size(), 22); 3867 goto EmitOp_Rd0_Rn5_Rm16_Ra10; 3868 } 3869 3870 break; 3871 } 3872 3873 case InstDB::kEncodingISimdVVVVx: { 3874 const InstDB::EncodingData::ISimdVVVVx& op_data = InstDB::EncodingData::iSimdVVVVx[encoding_index]; 3875 3876 if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) { 3877 if (o0.signature() != op_data.op0_signature || 3878 o1.signature() != op_data.op1_signature || 3879 o2.signature() != op_data.op2_signature || 3880 o3.signature() != op_data.op3_signature) 3881 goto InvalidInstruction; 3882 3883 opcode.reset(uint32_t(op_data.opcode) << 10); 3884 goto EmitOp_Rd0_Rn5_Rm16_Ra10; 3885 } 3886 3887 break; 3888 } 3889 3890 3891 case InstDB::kEncodingISimdPair: { 3892 const InstDB::EncodingData::ISimdPair& op_data = InstDB::EncodingData::iSimdPair[encoding_index]; 3893 3894 if (isign4 == ENC_OPS2(Reg, Reg) && op_data.opcode2) { 3895 if (o0.as<Vec>().is_vec_d1() && o1.as<Vec>().is_vec_d2()) { 3896 opcode.reset(uint32_t(op_data.opcode2) << 10); 3897 opcode.add_imm(0x3, 22); // size. 3898 goto EmitOp_Rd0_Rn5; 3899 } 3900 } 3901 3902 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 3903 if (!match_signature(o0, o1, o2, inst_flags)) 3904 goto InvalidInstruction; 3905 3906 SizeOp size_op = element_type_to_size_op(op_data.op_type3, o0.as<Reg>().reg_type(), o0.as<Vec>().element_type()); 3907 if (!size_op.is_valid()) 3908 goto InvalidInstruction; 3909 3910 opcode.reset(uint32_t(op_data.opcode3) << 10); 3911 opcode.add_imm(size_op.qs(), 30); 3912 opcode.add_imm(size_op.scalar(), 28); 3913 opcode.add_imm(size_op.size(), 22); 3914 goto EmitOp_Rd0_Rn5_Rm16; 3915 } 3916 3917 break; 3918 } 3919 3920 case InstDB::kEncodingSimdBicOrr: { 3921 const InstDB::EncodingData::SimdBicOrr& op_data = InstDB::EncodingData::simdBicOrr[encoding_index]; 3922 3923 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 3924 if (!match_signature(o0, o1, o2, inst_flags)) 3925 goto InvalidInstruction; 3926 3927 SizeOp size_op = element_type_to_size_op(InstDB::kVO_V_B, o0.as<Reg>().reg_type(), o0.as<Vec>().element_type()); 3928 if (!size_op.is_valid()) 3929 goto InvalidInstruction; 3930 3931 opcode.reset(uint32_t(op_data.register_op) << 10); 3932 opcode.add_imm(size_op.q(), 30); 3933 goto EmitOp_Rd0_Rn5_Rm16; 3934 } 3935 3936 if (isign4 == ENC_OPS2(Reg, Imm) || isign4 == ENC_OPS3(Reg, Imm, Imm)) { 3937 SizeOp size_op = element_type_to_size_op(InstDB::kVO_V_HS, o0.as<Reg>().reg_type(), o0.as<Vec>().element_type()); 3938 if (!size_op.is_valid()) 3939 goto InvalidInstruction; 3940 3941 if (o1.as<Imm>().value_as<uint64_t>() > 0xFFFFFFFFu) 3942 goto InvalidImmediate; 3943 3944 uint32_t imm = o1.as<Imm>().value_as<uint32_t>(); 3945 uint32_t shift = 0; 3946 uint32_t max_shift = (8u << size_op.size()) - 8u; 3947 3948 if (o2.is_imm()) { 3949 if (o2.as<Imm>().predicate() != uint32_t(ShiftOp::kLSL)) 3950 goto InvalidImmediate; 3951 3952 if (imm > 0xFFu || o2.as<Imm>().value_as<uint64_t>() > max_shift) 3953 goto InvalidImmediate; 3954 3955 shift = o2.as<Imm>().value_as<uint32_t>(); 3956 if ((shift & 0x7u) != 0u) 3957 goto InvalidImmediate; 3958 } 3959 else if (imm) { 3960 shift = Support::ctz(imm) & ~0x7u; 3961 imm >>= shift; 3962 3963 if (imm > 0xFFu || shift > max_shift) 3964 goto InvalidImmediate; 3965 } 3966 3967 uint32_t cmode = 0x1u | ((shift / 8u) << 1); 3968 if (size_op.size() == 1) 3969 cmode |= B(3); 3970 3971 // The immediate value is split into ABC and DEFGH parts. 3972 uint32_t abc = (imm >> 5) & 0x7u; 3973 uint32_t defgh = imm & 0x1Fu; 3974 3975 opcode.reset(uint32_t(op_data.immediate_op) << 10); 3976 opcode.add_imm(size_op.q(), 30); 3977 opcode.add_imm(abc, 16); 3978 opcode.add_imm(cmode, 12); 3979 opcode.add_imm(defgh, 5); 3980 goto EmitOp_Rd0; 3981 } 3982 3983 break; 3984 } 3985 3986 case InstDB::kEncodingSimdCmp: { 3987 const InstDB::EncodingData::SimdCmp& op_data = InstDB::EncodingData::simdCmp[encoding_index]; 3988 3989 if (isign4 == ENC_OPS3(Reg, Reg, Reg) && op_data.register_op) { 3990 if (!match_signature(o0, o1, o2, inst_flags)) 3991 goto InvalidInstruction; 3992 3993 SizeOp size_op = element_type_to_size_op(op_data.vec_op_type, o0.as<Reg>().reg_type(), o0.as<Vec>().element_type()); 3994 if (!size_op.is_valid()) 3995 goto InvalidInstruction; 3996 3997 opcode.reset(uint32_t(op_data.register_op) << 10); 3998 opcode.add_imm(size_op.qs(), 30); 3999 opcode.add_imm(size_op.scalar(), 28); 4000 opcode.add_imm(size_op.size(), 22); 4001 goto EmitOp_Rd0_Rn5_Rm16; 4002 } 4003 4004 if (isign4 == ENC_OPS3(Reg, Reg, Imm) && op_data.zero_op) { 4005 if (!match_signature(o0, o1, inst_flags)) 4006 goto InvalidInstruction; 4007 4008 if (o2.as<Imm>().value() != 0) 4009 goto InvalidImmediate; 4010 4011 SizeOp size_op = element_type_to_size_op(op_data.vec_op_type, o0.as<Reg>().reg_type(), o0.as<Vec>().element_type()); 4012 if (!size_op.is_valid()) 4013 goto InvalidInstruction; 4014 4015 opcode.reset(uint32_t(op_data.zero_op) << 10); 4016 opcode.add_imm(size_op.qs(), 30); 4017 opcode.add_imm(size_op.scalar(), 28); 4018 opcode.add_imm(size_op.size(), 22); 4019 goto EmitOp_Rd0_Rn5; 4020 } 4021 4022 break; 4023 } 4024 4025 case InstDB::kEncodingSimdDot: { 4026 const InstDB::EncodingData::SimdDot& op_data = InstDB::EncodingData::simdDot[encoding_index]; 4027 4028 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 4029 uint32_t q = diff(o0.as<Reg>().reg_type(), RegType::kVec64); 4030 uint32_t size = 2; 4031 4032 if (q > 1u) 4033 goto InvalidInstruction; 4034 4035 if (!o2.as<Vec>().has_element_index()) { 4036 if (!op_data.vector_op) 4037 goto InvalidInstruction; 4038 4039 if (o0.as<Reg>().reg_type() != o1.as<Reg>().reg_type() || o1.as<Reg>().reg_type() != o2.as<Reg>().reg_type()) 4040 goto InvalidInstruction; 4041 4042 if (uint32_t(o0.as<Vec>().element_type()) != op_data.ta || 4043 uint32_t(o1.as<Vec>().element_type()) != op_data.tb || 4044 uint32_t(o2.as<Vec>().element_type()) != op_data.tb) 4045 goto InvalidInstruction; 4046 4047 opcode.reset(uint32_t(op_data.vector_op) << 10); 4048 opcode.add_imm(q, 30); 4049 goto EmitOp_Rd0_Rn5_Rm16; 4050 } 4051 else { 4052 if (!op_data.element_op) 4053 goto InvalidInstruction; 4054 4055 if (o0.as<Reg>().reg_type() != o1.as<Reg>().reg_type() || !o2.as<Reg>().is_vec128()) 4056 goto InvalidInstruction; 4057 4058 if (uint32_t(o0.as<Vec>().element_type()) != op_data.ta || 4059 uint32_t(o1.as<Vec>().element_type()) != op_data.tb || 4060 uint32_t(o2.as<Vec>().element_type()) != op_data.tElement) 4061 goto InvalidInstruction; 4062 4063 uint32_t element_index = o2.as<Vec>().element_index(); 4064 LMHImm lmh; 4065 4066 if (!encode_lmh(size, element_index, Out(lmh))) 4067 goto InvalidElementIndex; 4068 4069 if (o2.as<Reg>().id() > lmh.max_rm_id) 4070 goto InvalidPhysId; 4071 4072 opcode.reset(uint32_t(op_data.element_op) << 10); 4073 opcode.add_imm(q, 30); 4074 opcode.add_imm(lmh.lm, 20); 4075 opcode.add_imm(lmh.h, 11); 4076 goto EmitOp_Rd0_Rn5_Rm16; 4077 } 4078 } 4079 4080 break; 4081 } 4082 4083 case InstDB::kEncodingSimdDup: SimdDup: { 4084 if (isign4 == ENC_OPS2(Reg, Reg)) { 4085 // Truth table of valid encodings of `Q:1|ElementType:3` 4086 uint32_t kValidEncodings = B(uint32_t(VecElementType::kB) + 0) | 4087 B(uint32_t(VecElementType::kH) + 0) | 4088 B(uint32_t(VecElementType::kS) + 0) | 4089 B(uint32_t(VecElementType::kB) + 8) | 4090 B(uint32_t(VecElementType::kH) + 8) | 4091 B(uint32_t(VecElementType::kS) + 8) | 4092 B(uint32_t(VecElementType::kD) + 8) ; 4093 4094 uint32_t q = diff(o0.as<Reg>().reg_type(), RegType::kVec64); 4095 4096 if (o1.as<Reg>().is_gp()) { 4097 // DUP - Vec (scalar|vector) <- GP register. 4098 // 4099 // NOTE: This is only scalar for `dup d, x` case, otherwise the value 4100 // would be duplicated across all vector elements (1, 2, 4, 8, or 16). 4101 uint32_t element_type = uint32_t(o0.as<Vec>().element_type()); 4102 if (q > 1 || !Support::bit_test(kValidEncodings, (q << 3) | element_type)) 4103 goto InvalidInstruction; 4104 4105 uint32_t lsb_index = element_type - 1u; 4106 uint32_t imm5 = 1u << lsb_index; 4107 4108 opcode.reset(0b0000111000000000000011 << 10); 4109 opcode.add_imm(q, 30); 4110 opcode.add_imm(imm5, 16); 4111 goto EmitOp_Rd0_Rn5; 4112 } 4113 4114 if (!o1.as<Reg>().is_vec() || !o1.as<Vec>().has_element_index()) 4115 goto InvalidInstruction; 4116 4117 uint32_t dst_index = o1.as<Vec>().element_index(); 4118 if (!o0.as<Vec>().has_element_type()) { 4119 // DUP - Vec (scalar) <- Vec[N]. 4120 uint32_t lsb_index = diff(o0.as<Reg>().reg_type(), RegType::kVec8); 4121 4122 if (lsb_index != diff(o1.as<Vec>().element_type(), VecElementType::kB) || lsb_index > 3) 4123 goto InvalidInstruction; 4124 4125 uint32_t imm5 = ((dst_index << 1) | 1u) << lsb_index; 4126 if (imm5 > 31) 4127 goto InvalidElementIndex; 4128 4129 opcode.reset(0b0101111000000000000001 << 10); 4130 opcode.add_imm(imm5, 16); 4131 goto EmitOp_Rd0_Rn5; 4132 } 4133 else { 4134 // DUP - Vec (all) <- Vec[N]. 4135 uint32_t element_type = uint32_t(o0.as<Vec>().element_type()); 4136 if (q > 1 || !Support::bit_test(kValidEncodings, (q << 3) | element_type)) 4137 goto InvalidInstruction; 4138 4139 uint32_t lsb_index = element_type - 1u; 4140 uint32_t imm5 = ((dst_index << 1) | 1u) << lsb_index; 4141 4142 if (imm5 > 31) 4143 goto InvalidElementIndex; 4144 4145 opcode.reset(0b0000111000000000000001 << 10); 4146 opcode.add_imm(q, 30); 4147 opcode.add_imm(imm5, 16); 4148 goto EmitOp_Rd0_Rn5; 4149 } 4150 } 4151 4152 break; 4153 } 4154 4155 case InstDB::kEncodingSimdIns: SimdIns: { 4156 if (isign4 == ENC_OPS2(Reg, Reg) && o0.as<Reg>().is_vec128()) { 4157 if (!o0.as<Vec>().has_element_index()) 4158 goto InvalidInstruction; 4159 4160 uint32_t element_type = uint32_t(o0.as<Vec>().element_type()); 4161 uint32_t dst_index = o0.as<Vec>().element_index(); 4162 uint32_t lsb_index = element_type - 1u; 4163 4164 uint32_t imm5 = ((dst_index << 1) | 1u) << lsb_index; 4165 if (imm5 > 31) 4166 goto InvalidElementIndex; 4167 4168 if (o1.as<Reg>().is_gp()) { 4169 // INS - Vec[N] <- GP register. 4170 opcode.reset(0b0100111000000000000111 << 10); 4171 opcode.add_imm(imm5, 16); 4172 goto EmitOp_Rd0_Rn5; 4173 } 4174 else if (o1.as<Reg>().is_vec128() && o1.as<Vec>().has_element_index()) { 4175 // INS - Vec[N] <- Vec[M]. 4176 if (o0.as<Vec>().element_type() != o1.as<Vec>().element_type()) 4177 goto InvalidInstruction; 4178 4179 uint32_t src_index = o1.as<Vec>().element_index(); 4180 if (o0.as<Reg>().reg_type() != o1.as<Reg>().reg_type()) 4181 goto InvalidInstruction; 4182 4183 uint32_t imm4 = src_index << lsb_index; 4184 if (imm4 > 15) 4185 goto InvalidElementIndex; 4186 4187 opcode.reset(0b0110111000000000000001 << 10); 4188 opcode.add_imm(imm5, 16); 4189 opcode.add_imm(imm4, 11); 4190 goto EmitOp_Rd0_Rn5; 4191 } 4192 } 4193 4194 break; 4195 } 4196 4197 case InstDB::kEncodingSimdMov: { 4198 if (isign4 == ENC_OPS2(Reg, Reg)) { 4199 if (o0.as<Reg>().is_vec() && o1.as<Reg>().is_vec()) { 4200 // INS v.x[index], v.x[index]. 4201 if (o0.as<Vec>().has_element_index() && o1.as<Vec>().has_element_index()) 4202 goto SimdIns; 4203 4204 // DUP {b|h|s|d}, v.{b|h|s|d}[index]. 4205 if (o1.as<Vec>().has_element_index()) 4206 goto SimdDup; 4207 4208 if (!check_signature(o0, o1)) 4209 goto InvalidInstruction; 4210 4211 // ORR Vd, Vn, Vm 4212 uint32_t q = diff(o0.as<Reg>().reg_type(), RegType::kVec64); 4213 if (q > 1) 4214 goto InvalidInstruction; 4215 4216 opcode.reset(0b0000111010100000000111 << 10); 4217 opcode.add_imm(q, 30); 4218 opcode.add_reg(o1, 16); // Vn == Vm. 4219 goto EmitOp_Rd0_Rn5; 4220 } 4221 4222 if (o0.as<Reg>().is_vec() && o1.as<Reg>().is_gp()) { 4223 // INS v.x[index], Rn. 4224 if (o0.as<Vec>().has_element_index()) 4225 goto SimdIns; 4226 4227 goto InvalidInstruction; 4228 } 4229 4230 if (o0.as<Reg>().is_gp() && o1.as<Reg>().is_vec()) { 4231 // UMOV Rd, V.{s|d}[index]. 4232 encoding_index = 1; 4233 goto SimdUmov; 4234 } 4235 } 4236 4237 break; 4238 } 4239 4240 case InstDB::kEncodingSimdMoviMvni: { 4241 const InstDB::EncodingData::SimdMoviMvni& op_data = InstDB::EncodingData::simdMoviMvni[encoding_index]; 4242 4243 if (isign4 == ENC_OPS2(Reg, Imm) || isign4 == ENC_OPS3(Reg, Imm, Imm)) { 4244 SizeOp size_op = element_type_to_size_op(InstDB::kVO_V_Any, o0.as<Reg>().reg_type(), o0.as<Vec>().element_type()); 4245 if (!size_op.is_valid()) 4246 goto InvalidInstruction; 4247 4248 uint64_t imm64 = o1.as<Imm>().value_as<uint64_t>(); 4249 uint32_t imm8 = 0; 4250 uint32_t cmode = 0; 4251 uint32_t inverted = op_data.inverted; 4252 uint32_t op = 0; 4253 uint32_t shift = 0; 4254 uint32_t shift_op = uint32_t(ShiftOp::kLSL); 4255 4256 if (size_op.size() == 3u) { 4257 // The second immediate should not be present, however, we accept 4258 // an immediate value of zero as some user code may still pass it. 4259 if (o2.is_imm() && o0.as<Imm>().value() != 0) 4260 goto InvalidImmediate; 4261 4262 if (Utils::is_byte_mask_imm(imm64)) { 4263 imm8 = Utils::encode_imm64_byte_mask_to_imm8(imm64); 4264 } 4265 else { 4266 // Change from D to S and from 64-bit imm to 32-bit imm if this 4267 // is not a byte-mask pattern. 4268 if ((imm64 >> 32) == (imm64 & 0xFFFFFFFFu)) { 4269 imm64 &= 0xFFFFFFFFu; 4270 size_op.decrement_size(); 4271 } 4272 else { 4273 goto InvalidImmediate; 4274 } 4275 } 4276 } 4277 4278 if (size_op.size() < 3u) { 4279 if (imm64 > 0xFFFFFFFFu) 4280 goto InvalidImmediate; 4281 imm8 = uint32_t(imm64); 4282 4283 if (size_op.size() == 2) { 4284 if ((imm8 >> 16) == (imm8 & 0xFFFFu)) { 4285 imm8 >>= 16; 4286 size_op.decrement_size(); 4287 } 4288 } 4289 4290 if (size_op.size() == 1) { 4291 if (imm8 > 0xFFFFu) 4292 goto InvalidImmediate; 4293 4294 if ((imm8 >> 8) == (imm8 & 0xFFu)) { 4295 imm8 >>= 8; 4296 size_op.decrement_size(); 4297 } 4298 } 4299 4300 uint32_t max_shift = (8u << size_op.size()) - 8u; 4301 if (o2.is_imm()) { 4302 if (imm8 > 0xFFu || o2.as<Imm>().value_as<uint64_t>() > max_shift) 4303 goto InvalidImmediate; 4304 4305 shift = o2.as<Imm>().value_as<uint32_t>(); 4306 shift_op = o2.as<Imm>().predicate(); 4307 } 4308 else if (imm8) { 4309 shift = Support::ctz(imm8) & ~0x7u; 4310 imm8 >>= shift; 4311 4312 if (imm8 > 0xFFu || shift > max_shift) 4313 goto InvalidImmediate; 4314 } 4315 4316 if ((shift & 0x7u) != 0u) 4317 goto InvalidImmediate; 4318 } 4319 4320 shift /= 8u; 4321 4322 switch (size_op.size()) { 4323 case 0: 4324 if (shift_op != uint32_t(ShiftOp::kLSL)) 4325 goto InvalidImmediate; 4326 4327 if (inverted) { 4328 imm8 = ~imm8 & 0xFFu; 4329 } 4330 4331 cmode = B(3) | B(2) | B(1); 4332 break; 4333 4334 case 1: 4335 if (shift_op != uint32_t(ShiftOp::kLSL)) 4336 goto InvalidImmediate; 4337 4338 cmode = B(3) | (shift << 1); 4339 op = inverted; 4340 break; 4341 4342 case 2: 4343 if (shift_op == uint32_t(ShiftOp::kLSL)) { 4344 cmode = shift << 1; 4345 } 4346 else if (shift_op == uint32_t(ShiftOp::kMSL)) { 4347 if (shift == 0 || shift > 2) 4348 goto InvalidImmediate; 4349 cmode = B(3) | B(2) | (shift - 1u); 4350 } 4351 else { 4352 goto InvalidImmediate; 4353 } 4354 4355 op = inverted; 4356 break; 4357 4358 case 3: 4359 if (inverted) { 4360 imm8 = ~imm8 & 0xFFu; 4361 } 4362 4363 op = 1; 4364 cmode = B(3) | B(2) | B(1); 4365 break; 4366 } 4367 4368 // The immediate value is split into ABC and DEFGH parts. 4369 uint32_t abc = (imm8 >> 5) & 0x7u; 4370 uint32_t defgh = imm8 & 0x1Fu; 4371 4372 opcode.reset(uint32_t(op_data.opcode) << 10); 4373 opcode.add_imm(size_op.q(), 30); 4374 opcode.add_imm(op, 29); 4375 opcode.add_imm(abc, 16); 4376 opcode.add_imm(cmode, 12); 4377 opcode.add_imm(defgh, 5); 4378 goto EmitOp_Rd0; 4379 } 4380 4381 break; 4382 } 4383 4384 case InstDB::kEncodingSimdShift: { 4385 const InstDB::EncodingData::SimdShift& op_data = InstDB::EncodingData::simdShift[encoding_index]; 4386 4387 const Operand_& sop = significant_simd_op(o0, o1, inst_flags); 4388 SizeOp size_op = element_type_to_size_op(op_data.vec_op_type, sop.as<Reg>().reg_type(), sop.as<Vec>().element_type()); 4389 4390 if (!size_op.is_valid()) 4391 goto InvalidInstruction; 4392 4393 if (isign4 == ENC_OPS3(Reg, Reg, Imm) && op_data.immediate_op) { 4394 if (!match_signature(o0, o1, inst_flags)) 4395 goto InvalidInstruction; 4396 4397 if (o2.as<Imm>().value_as<uint64_t>() > 63) 4398 goto InvalidImmediate; 4399 4400 uint32_t lsb_shift = size_op.size() + 3u; 4401 uint32_t lsb_mask = (1u << lsb_shift) - 1u; 4402 uint32_t imm = o2.as<Imm>().value_as<uint32_t>(); 4403 4404 // Some instructions use IMM and some X - IMM, so negate if required. 4405 if (op_data.inverted_imm) { 4406 if (imm == 0 || imm > (1u << lsb_shift)) 4407 goto InvalidImmediate; 4408 imm = Support::neg(imm) & lsb_mask; 4409 } 4410 4411 if (imm > lsb_mask) 4412 goto InvalidImmediate; 4413 imm |= (1u << lsb_shift); 4414 4415 opcode.reset(uint32_t(op_data.immediate_op) << 10); 4416 opcode.add_imm(size_op.qs(), 30); 4417 opcode.add_imm(size_op.scalar(), 28); 4418 opcode.add_imm(imm, 16); 4419 goto EmitOp_Rd0_Rn5; 4420 } 4421 4422 if (isign4 == ENC_OPS3(Reg, Reg, Reg) && op_data.register_op) { 4423 if (!match_signature(o0, o1, o2, inst_flags)) 4424 goto InvalidInstruction; 4425 4426 opcode.reset(uint32_t(op_data.register_op) << 10); 4427 opcode.add_imm(size_op.qs(), 30); 4428 opcode.add_imm(size_op.scalar(), 28); 4429 opcode.add_imm(size_op.size(), 22); 4430 goto EmitOp_Rd0_Rn5_Rm16; 4431 } 4432 4433 break; 4434 } 4435 4436 case InstDB::kEncodingSimdShiftES: { 4437 const InstDB::EncodingData::SimdShiftES& op_data = InstDB::EncodingData::simdShiftES[encoding_index]; 4438 4439 if (isign4 == ENC_OPS3(Reg, Reg, Imm)) { 4440 SizeOp size_op = element_type_to_size_op(op_data.vec_op_type, o1.as<Reg>().reg_type(), o1.as<Vec>().element_type()); 4441 if (!size_op.is_valid()) 4442 goto InvalidInstruction; 4443 4444 if (!match_signature(o0, o1, inst_flags)) 4445 goto InvalidInstruction; 4446 4447 // The immediate value must match the element size. 4448 uint64_t shift = o2.as<Imm>().value_as<uint64_t>(); 4449 uint32_t shift_op = o2.as<Imm>().predicate(); 4450 4451 if (shift != (8u << size_op.size()) || shift_op != uint32_t(ShiftOp::kLSL)) 4452 goto InvalidImmediate; 4453 4454 opcode.reset(uint32_t(op_data.opcode) << 10); 4455 opcode.add_imm(size_op.q(), 30); 4456 opcode.add_imm(size_op.size(), 22); 4457 goto EmitOp_Rd0_Rn5; 4458 } 4459 4460 break; 4461 } 4462 4463 case InstDB::kEncodingSimdSm3tt: { 4464 const InstDB::EncodingData::SimdSm3tt& op_data = InstDB::EncodingData::simdSm3tt[encoding_index]; 4465 4466 if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { 4467 if (o0.as<Vec>().is_vec_s4() && o1.as<Vec>().is_vec_s4() && o2.as<Vec>().is_vec_s4() && o2.as<Vec>().has_element_index()) { 4468 uint32_t imm2 = o2.as<Vec>().element_index(); 4469 if (imm2 > 3) 4470 goto InvalidElementIndex; 4471 4472 opcode.reset(uint32_t(op_data.opcode) << 10); 4473 opcode.add_imm(imm2, 12); 4474 goto EmitOp_Rd0_Rn5_Rm16; 4475 } 4476 } 4477 4478 break; 4479 } 4480 4481 4482 case InstDB::kEncodingSimdSmovUmov: SimdUmov: { 4483 const InstDB::EncodingData::SimdSmovUmov& op_data = InstDB::EncodingData::simdSmovUmov[encoding_index]; 4484 4485 if (isign4 == ENC_OPS2(Reg, Reg) && o0.as<Reg>().is_gp() && o1.as<Reg>().is_vec()) { 4486 SizeOp size_op = element_type_to_size_op(op_data.vec_op_type, o1.as<Reg>().reg_type(), o1.as<Vec>().element_type()); 4487 if (!size_op.is_valid()) 4488 goto InvalidInstruction; 4489 4490 if (!o1.as<Vec>().has_element_index()) 4491 goto InvalidInstruction; 4492 4493 uint32_t x = o0.as<Gp>().is_gp64(); 4494 uint32_t gp_must_be_x = uint32_t(size_op.size() >= 3u - op_data.is_signed); 4495 4496 if (op_data.is_signed) { 4497 if (gp_must_be_x && !x) 4498 goto InvalidInstruction; 4499 } 4500 else { 4501 if (x != gp_must_be_x) 4502 goto InvalidInstruction; 4503 } 4504 4505 uint32_t element_index = o1.as<Vec>().element_index(); 4506 uint32_t max_element_index = 15u >> size_op.size(); 4507 4508 if (element_index > max_element_index) 4509 goto InvalidElementIndex; 4510 4511 uint32_t imm5 = (1u | (element_index << 1)) << size_op.size(); 4512 4513 opcode.reset(uint32_t(op_data.opcode) << 10); 4514 opcode.add_imm(x, 30); 4515 opcode.add_imm(imm5, 16); 4516 goto EmitOp_Rd0_Rn5; 4517 } 4518 4519 break; 4520 } 4521 4522 case InstDB::kEncodingSimdSxtlUxtl: { 4523 const InstDB::EncodingData::SimdSxtlUxtl& op_data = InstDB::EncodingData::simdSxtlUxtl[encoding_index]; 4524 4525 if (isign4 == ENC_OPS2(Reg, Reg)) { 4526 SizeOp size_op = element_type_to_size_op(op_data.vec_op_type, o1.as<Reg>().reg_type(), o1.as<Vec>().element_type()); 4527 if (!size_op.is_valid()) 4528 goto InvalidInstruction; 4529 4530 if (!match_signature(o0, o1, inst_flags)) 4531 goto InvalidInstruction; 4532 4533 opcode.reset(uint32_t(op_data.opcode) << 10); 4534 opcode.add_imm(size_op.q(), 30); 4535 opcode.add_imm(1u, size_op.size() + 19); 4536 goto EmitOp_Rd0_Rn5; 4537 } 4538 4539 break; 4540 } 4541 4542 case InstDB::kEncodingSimdTblTbx: { 4543 const InstDB::EncodingData::SimdTblTbx& op_data = InstDB::EncodingData::simdTblTbx[encoding_index]; 4544 4545 if (isign4 == ENC_OPS3(Reg, Reg, Reg) || isign4 == ENC_OPS4(Reg, Reg, Reg, Reg)) { 4546 // TBL/TBX <Vd>.<Ta>, { <Vn>.16B }, <Vm>.<Ta> 4547 // TBL/TBX <Vd>.<Ta>, { <Vn>.16B, <Vn+1>.16B }, <Vm>.<Ta> 4548 // TBL/TBX <Vd>.<Ta>, { <Vn>.16B, <Vn+1>.16B, <Vn+2>.16B }, <Vm>.<Ta> 4549 // TBL/TBX <Vd>.<Ta>, { <Vn>.16B, <Vn+1>.16B, <Vn+2>.16B, <Vn+3>.16B }, <Vm>.<Ta> 4550 opcode.reset(uint32_t(op_data.opcode) << 10); 4551 4552 const Operand_& o4 = op_ext[EmitterUtils::kOp4]; 4553 const Operand_& o5 = op_ext[EmitterUtils::kOp5]; 4554 4555 uint32_t q = diff(o0.as<Reg>().reg_type(), RegType::kVec64); 4556 if (q > 1 || o0.as<Vec>().has_element_index()) 4557 goto InvalidInstruction; 4558 4559 if (!o1.as<Vec>().is_vec_b16() || o1.as<Vec>().has_element_index()) 4560 goto InvalidInstruction; 4561 4562 uint32_t len = uint32_t(!o3.is_none()) + uint32_t(!o4.is_none()) + uint32_t(!o5.is_none()); 4563 opcode.add_imm(q, 30); 4564 opcode.add_imm(len, 13); 4565 4566 switch (len) { 4567 case 0: 4568 if (!check_signature(o0, o2)) 4569 goto InvalidInstruction; 4570 4571 if (o2.id() > 31) 4572 goto InvalidPhysId; 4573 4574 opcode.add_reg(o2, 16); 4575 goto EmitOp_Rd0_Rn5; 4576 4577 case 1: 4578 if (!check_signature(o0, o3)) 4579 goto InvalidInstruction; 4580 4581 if (o3.id() > 31) 4582 goto InvalidPhysId; 4583 4584 opcode.add_reg(o3, 16); 4585 goto EmitOp_Rd0_Rn5; 4586 4587 case 2: 4588 if (!check_signature(o0, o4)) 4589 goto InvalidInstruction; 4590 4591 if (o4.id() > 31) 4592 goto InvalidPhysId; 4593 4594 opcode.add_reg(o4, 16); 4595 goto EmitOp_Rd0_Rn5; 4596 4597 case 3: 4598 if (!check_signature(o0, o5)) 4599 goto InvalidInstruction; 4600 4601 if (o5.id() > 31) 4602 goto InvalidPhysId; 4603 4604 opcode.add_reg(o5, 16); 4605 goto EmitOp_Rd0_Rn5; 4606 4607 default: 4608 // Should never happen. 4609 goto InvalidInstruction; 4610 } 4611 } 4612 4613 break; 4614 } 4615 4616 // ------------------------------------------------------------------------ 4617 // [Simd - Load / Store] 4618 // ------------------------------------------------------------------------ 4619 4620 case InstDB::kEncodingSimdLdSt: { 4621 const InstDB::EncodingData::SimdLdSt& op_data = InstDB::EncodingData::simdLdSt[encoding_index]; 4622 4623 if (isign4 == ENC_OPS2(Reg, Mem)) { 4624 const Mem& m = o1.as<Mem>(); 4625 rm_rel = &m; 4626 4627 // Width | SZ | XY | XSZ 4628 // -------+----------+-----------+----- 4629 // 8-bit | size==00 | opc == 01 | 000 4630 // 16-bit | size==01 | opc == 01 | 001 4631 // 32-bit | size==10 | opc == 01 | 010 4632 // 64-bit | size==11 | opc == 01 | 011 4633 // 128-bit| size==00 | opc == 11 | 100 4634 uint32_t xsz = diff(o0.as<Reg>().reg_type(), RegType::kVec8); 4635 if (xsz > 4u || o0.as<Vec>().has_element_index()) 4636 goto InvalidRegType; 4637 4638 if (!check_vec_id(o0)) 4639 goto InvalidPhysId; 4640 4641 if (!check_mem_base_index_rel(m)) 4642 goto InvalidAddress; 4643 4644 int64_t offset = m.offset(); 4645 if (m.has_base_reg()) { 4646 // [Base {Offset | Index}] 4647 if (m.has_index()) { 4648 uint32_t opt = shift_op_to_ld_st_opt_map[size_t(m.shift_op())]; 4649 if (opt == 0xFFu) 4650 goto InvalidAddress; 4651 4652 uint32_t shift = m.shift(); 4653 uint32_t s = (shift != 0); 4654 4655 if (s && shift != xsz) 4656 goto InvalidAddressScale; 4657 4658 opcode.reset(uint32_t(op_data.register_op) << 21); 4659 opcode.add_imm(xsz & 3u, 30); 4660 opcode.add_imm(xsz >> 2, 23); 4661 opcode.add_imm(opt, 13); 4662 opcode.add_imm(s, 12); 4663 opcode |= B(11); 4664 opcode.add_reg(o0, 0); 4665 goto EmitOp_MemBaseIndex_Rn5_Rm16; 4666 } 4667 4668 // Makes it easier to work with the offset especially on 32-bit arch. 4669 if (!Support::is_int_n<32>(offset)) 4670 goto InvalidDisplacement; 4671 int32_t offset32 = int32_t(offset); 4672 4673 if (m.is_pre_or_post()) { 4674 if (!Support::is_int_n<9>(offset32)) 4675 goto InvalidDisplacement; 4676 4677 opcode.reset(uint32_t(op_data.pre_post_op) << 21); 4678 opcode.add_imm(xsz & 3u, 30); 4679 opcode.add_imm(xsz >> 2, 23); 4680 opcode.add_imm(offset32 & 0x1FF, 12); 4681 opcode.add_imm(m.is_pre_index(), 11); 4682 opcode |= B(10); 4683 opcode.add_reg(o0, 0); 4684 goto EmitOp_MemBase_Rn5; 4685 } 4686 else { 4687 uint32_t imm12 = uint32_t(offset32) >> xsz; 4688 4689 // If this instruction is not encodable with scaled unsigned offset, try unscaled signed offset. 4690 if (!Support::is_uint_n<12>(imm12) || (imm12 << xsz) != uint32_t(offset32)) { 4691 inst_id = op_data.u_alt_inst_id; 4692 inst_info = &InstDB::_inst_info_table[inst_id]; 4693 encoding_index = inst_info->_encoding_data_index; 4694 goto Case_SimdLdurStur; 4695 } 4696 4697 opcode.reset(uint32_t(op_data.u_offset_op) << 22); 4698 opcode.add_imm(xsz & 3u, 30); 4699 opcode.add_imm(xsz >> 2, 23); 4700 opcode.add_imm(imm12, 10); 4701 opcode.add_reg(o0, 0); 4702 goto EmitOp_MemBase_Rn5; 4703 } 4704 } 4705 else { 4706 if (!op_data.literal_op) 4707 goto InvalidAddress; 4708 4709 if (xsz < 2u) 4710 goto InvalidRegType; 4711 4712 uint32_t opc = xsz - 2u; 4713 opcode.reset(uint32_t(op_data.literal_op) << 24); 4714 opcode.add_imm(opc, 30); 4715 opcode.add_reg(o0, 0); 4716 offset_format.reset_to_imm_value(OffsetType::kSignedOffset, 4, 5, 19, 2); 4717 goto EmitOp_Rel; 4718 } 4719 } 4720 4721 break; 4722 } 4723 4724 case InstDB::kEncodingSimdLdpStp: { 4725 const InstDB::EncodingData::SimdLdpStp& op_data = InstDB::EncodingData::simdLdpStp[encoding_index]; 4726 4727 if (isign4 == ENC_OPS3(Reg, Reg, Mem)) { 4728 const Mem& m = o2.as<Mem>(); 4729 rm_rel = &m; 4730 4731 uint32_t opc = diff(o0.as<Reg>().reg_type(), RegType::kVec32); 4732 if (opc > 2u || o0.as<Vec>().has_element_type_or_index()) 4733 goto InvalidInstruction; 4734 4735 if (!check_signature(o0, o1)) 4736 goto InvalidInstruction; 4737 4738 if (!check_vec_id(o0, o1)) 4739 goto InvalidPhysId; 4740 4741 if (m.base_type() != RegType::kGp64 || m.has_index()) 4742 goto InvalidAddress; 4743 4744 if (m.is_offset_64bit()) 4745 goto InvalidDisplacement; 4746 4747 uint32_t offset_shift = 2u + opc; 4748 int32_t offset32 = m.offset_lo32() >> offset_shift; 4749 4750 // Make sure we didn't lose bits by applying the mandatory offset shift. 4751 if (Support::shl(offset32, offset_shift) != m.offset_lo32()) 4752 goto InvalidDisplacement; 4753 4754 // Offset is encoded as a 7-bit immediate. 4755 if (!Support::is_int_n<7>(offset32)) 4756 goto InvalidDisplacement; 4757 4758 if (m.is_pre_or_post() && offset32 != 0) { 4759 if (!op_data.pre_post_op) 4760 goto InvalidAddress; 4761 4762 opcode.reset(uint32_t(op_data.pre_post_op) << 22); 4763 opcode.add_imm(m.is_pre_index(), 24); 4764 } 4765 else { 4766 opcode.reset(uint32_t(op_data.offset_op) << 22); 4767 } 4768 4769 opcode.add_imm(opc, 30); 4770 opcode.add_imm(offset32 & 0x7F, 15); 4771 opcode.add_reg(o1, 10); 4772 opcode.add_reg(o0, 0); 4773 goto EmitOp_MemBase_Rn5; 4774 } 4775 4776 break; 4777 } 4778 4779 case InstDB::kEncodingSimdLdurStur: { 4780 Case_SimdLdurStur: 4781 const InstDB::EncodingData::SimdLdurStur& op_data = InstDB::EncodingData::simdLdurStur[encoding_index]; 4782 4783 if (isign4 == ENC_OPS2(Reg, Mem)) { 4784 const Mem& m = o1.as<Mem>(); 4785 rm_rel = &m; 4786 4787 uint32_t sz = diff(o0.as<Reg>().reg_type(), RegType::kVec8); 4788 if (sz > 4 || o0.as<Vec>().has_element_type_or_index()) 4789 goto InvalidInstruction; 4790 4791 if (!check_vec_id(o0)) 4792 goto InvalidPhysId; 4793 4794 if (!check_mem_base_index_rel(m)) 4795 goto InvalidAddress; 4796 4797 if (m.has_base_reg() && !m.has_index() && !m.is_pre_or_post()) { 4798 if (m.is_offset_64bit()) 4799 goto InvalidDisplacement; 4800 4801 int32_t offset32 = m.offset_lo32(); 4802 if (!Support::is_int_n<9>(offset32)) 4803 goto InvalidDisplacement; 4804 4805 opcode.reset(uint32_t(op_data.opcode) << 10); 4806 opcode.add_imm(sz & 3u, 30); 4807 opcode.add_imm(sz >> 2, 23); 4808 opcode.add_imm(offset32 & 0x1FF, 12); 4809 opcode.add_reg(o0, 0); 4810 goto EmitOp_MemBase_Rn5; 4811 } 4812 4813 goto InvalidAddress; 4814 } 4815 4816 break; 4817 } 4818 4819 case InstDB::kEncodingSimdLdNStN: { 4820 const InstDB::EncodingData::SimdLdNStN& op_data = InstDB::EncodingData::simdLdNStN[encoding_index]; 4821 const Operand_& o4 = op_ext[EmitterUtils::kOp4]; 4822 4823 uint32_t n = 1; 4824 4825 if (isign4 == ENC_OPS2(Reg, Mem)) { 4826 if (op_data.n != 1) 4827 goto InvalidInstruction; 4828 4829 rm_rel = &o1; 4830 } 4831 else if (isign4 == ENC_OPS3(Reg, Reg, Mem)) { 4832 if (op_data.n != 1 && op_data.n != 2) 4833 goto InvalidInstruction; 4834 4835 if (!check_signature(o0, o1) || !check_consecutive(o0, o1)) 4836 goto InvalidInstruction; 4837 4838 n = 2; 4839 rm_rel = &o2; 4840 } 4841 else if (isign4 == ENC_OPS4(Reg, Reg, Reg, Mem) && o4.is_none()) { 4842 if (op_data.n != 1 && op_data.n != 3) 4843 goto InvalidInstruction; 4844 4845 if (!check_signature(o0, o1, o2) || !check_consecutive(o0, o1, o2)) 4846 goto InvalidInstruction; 4847 4848 n = 3; 4849 rm_rel = &o3; 4850 } 4851 else if (isign4 == ENC_OPS4(Reg, Reg, Reg, Reg) && o4.is_mem()) { 4852 if (op_data.n != 1 && op_data.n != 4) 4853 goto InvalidInstruction; 4854 4855 if (!check_signature(o0, o1, o2, o3) || !check_consecutive(o0, o1, o2, o3)) 4856 goto InvalidInstruction; 4857 4858 n = 4; 4859 rm_rel = &o4; 4860 } 4861 else { 4862 goto InvalidInstruction; 4863 } 4864 4865 // We will use `v` and `m` from now as those are relevant for encoding. 4866 const Vec& v = o0.as<Vec>(); 4867 const Mem& m = rm_rel->as<Mem>(); 4868 4869 uint32_t q = 0; 4870 uint32_t rm = 0; 4871 uint32_t rn = m.base_id(); 4872 uint32_t sz = diff(v.element_type(), VecElementType::kB); 4873 uint32_t opc_s_size = sz; 4874 uint32_t offset_possibility = 0; 4875 4876 if (sz > 3) 4877 goto InvalidInstruction; 4878 4879 if (m.base_type() != RegType::kGp64) 4880 goto InvalidAddress; 4881 4882 // Rn cannot be ZR, but can be SP. 4883 if (rn > 30 && rn != Gp::kIdSp) 4884 goto InvalidAddress; 4885 4886 rn &= 31; 4887 4888 if (op_data.replicate) { 4889 if (n != op_data.n) 4890 goto InvalidInstruction; 4891 4892 // Replicates to the whole register, element index cannot be used. 4893 if (v.has_element_index()) 4894 goto InvalidInstruction; 4895 4896 q = diff(v.reg_type(), RegType::kVec64); 4897 if (q > 1) 4898 goto InvalidInstruction; 4899 4900 opcode.reset(uint32_t(op_data.single_op) << 10); 4901 offset_possibility = (1u << sz) * n; 4902 } 4903 else if (v.has_element_index()) { 4904 if (n != op_data.n) 4905 goto InvalidInstruction; 4906 4907 // LDx/STx (single structure). 4908 static const uint8_t opc_s_size_by_sz_table[] = { 0x0u << 3, 0x2u << 3, 0x4u << 3, (0x4u << 3) | 1u }; 4909 4910 opcode.reset(uint32_t(op_data.single_op) << 10); 4911 opc_s_size = opc_s_size_by_sz_table[sz]; 4912 offset_possibility = (1u << sz) * op_data.n; 4913 4914 uint32_t element_index = v.element_index(); 4915 uint32_t max_element_index = 15 >> sz; 4916 4917 if (element_index > max_element_index) 4918 goto InvalidElementIndex; 4919 4920 element_index <<= sz; 4921 q = element_index >> 3; 4922 opc_s_size |= element_index & 0x7u; 4923 } 4924 else { 4925 // LDx/STx (multiple structures). 4926 static const uint8_t opc_s_size_by_n_table[] = { 0u, 0x7u << 2, 0xAu << 2, 0x6u << 2, 0x2u << 2 }; 4927 4928 q = diff(v.reg_type(), RegType::kVec64); 4929 if (q > 1) 4930 goto InvalidInstruction; 4931 4932 if (op_data.n == 1) 4933 opc_s_size |= opc_s_size_by_n_table[n]; 4934 4935 opcode.reset(uint32_t(op_data.multiple_op) << 10); 4936 offset_possibility = (8u << q) * n; 4937 } 4938 4939 if (m.has_index()) { 4940 if (m.has_offset() || !m.is_post_index()) 4941 goto InvalidAddress; 4942 4943 rm = m.index_id(); 4944 if (rm > 30) 4945 goto InvalidAddress; 4946 4947 // Bit 23 - PostIndex. 4948 opcode |= B(23); 4949 } 4950 else { 4951 if (m.has_offset()) { 4952 if (m.offset() != int32_t(offset_possibility) || !m.is_post_index()) 4953 goto InvalidAddress; 4954 rm = 31; 4955 4956 // Bit 23 - PostIndex. 4957 opcode |= B(23); 4958 } 4959 } 4960 4961 opcode.add_imm(q, 30); 4962 opcode.add_imm(rm, 16); 4963 opcode.add_imm(opc_s_size, 10); 4964 opcode.add_imm(rn, 5); 4965 goto EmitOp_Rd0; 4966 } 4967 4968 default: 4969 break; 4970 } 4971 4972 goto InvalidInstruction; 4973 4974 // -------------------------------------------------------------------------- 4975 // [EmitGp - Single] 4976 // -------------------------------------------------------------------------- 4977 4978 EmitOp_Rd0: 4979 if (!check_valid_regs(o0)) 4980 goto InvalidPhysId; 4981 4982 opcode.add_reg(o0, 0); 4983 goto EmitOp; 4984 4985 EmitOp_Rn5: 4986 if (!check_valid_regs(o0)) 4987 goto InvalidPhysId; 4988 4989 opcode.add_reg(o0, 5); 4990 goto EmitOp; 4991 4992 EmitOp_Rn5_Rm16: 4993 if (!check_valid_regs(o0, o1)) 4994 goto InvalidPhysId; 4995 4996 opcode.add_reg(o0, 5); 4997 opcode.add_reg(o1, 16); 4998 goto EmitOp; 4999 5000 EmitOp_Rd0_Rn5: 5001 if (!check_valid_regs(o0, o1)) 5002 goto InvalidPhysId; 5003 5004 opcode.add_reg(o0, 0); 5005 opcode.add_reg(o1, 5); 5006 goto EmitOp; 5007 5008 EmitOp_Rd0_Rn5_Rm16_Ra10: 5009 if (!check_valid_regs(o0, o1, o2, o3)) 5010 goto InvalidPhysId; 5011 5012 opcode.add_reg(o0, 0); 5013 opcode.add_reg(o1, 5); 5014 opcode.add_reg(o2, 16); 5015 opcode.add_reg(o3, 10); 5016 goto EmitOp; 5017 5018 EmitOp_Rd0_Rn5_Rm16: 5019 if (!check_valid_regs(o0, o1, o3)) 5020 goto InvalidPhysId; 5021 5022 opcode.add_reg(o0, 0); 5023 opcode.add_reg(o1, 5); 5024 opcode.add_reg(o2, 16); 5025 goto EmitOp; 5026 5027 // -------------------------------------------------------------------------- 5028 // [EmitGp - Multiple] 5029 // -------------------------------------------------------------------------- 5030 5031 EmitOp_Multiple: 5032 { 5033 ASMJIT_ASSERT(multiple_op_count > 0); 5034 err = writer.ensure_space(this, multiple_op_count * 4u); 5035 if (ASMJIT_UNLIKELY(err != Error::kOk)) { 5036 goto Failed; 5037 } 5038 5039 for (uint32_t i = 0; i < multiple_op_count; i++) { 5040 writer.emit32u_le(multiple_op_data[i]); 5041 } 5042 5043 goto EmitDone; 5044 } 5045 5046 // -------------------------------------------------------------------------- 5047 // [EmitGp - Memory] 5048 // -------------------------------------------------------------------------- 5049 5050 EmitOp_MemBase_Rn5: 5051 if (!check_mem_base(rm_rel->as<Mem>())) { 5052 goto InvalidAddress; 5053 } 5054 5055 opcode.add_reg(rm_rel->as<Mem>().base_id(), 5); 5056 goto EmitOp; 5057 5058 EmitOp_MemBaseNoImm_Rn5: 5059 if (!check_mem_base(rm_rel->as<Mem>()) || rm_rel->as<Mem>().has_index()) { 5060 goto InvalidAddress; 5061 } 5062 5063 if (rm_rel->as<Mem>().has_offset()) { 5064 goto InvalidDisplacement; 5065 } 5066 5067 opcode.add_reg(rm_rel->as<Mem>().base_id(), 5); 5068 goto EmitOp; 5069 5070 EmitOp_MemBaseIndex_Rn5_Rm16: 5071 if (!rm_rel->as<Mem>().has_base_reg()) { 5072 goto InvalidAddress; 5073 } 5074 5075 if (rm_rel->as<Mem>().index_id() > 30 && rm_rel->as<Mem>().index_id() != Gp::kIdZr) { 5076 goto InvalidPhysId; 5077 } 5078 5079 opcode.add_reg(rm_rel->as<Mem>().index_id(), 16); 5080 opcode.add_reg(rm_rel->as<Mem>().base_id(), 5); 5081 goto EmitOp; 5082 5083 // -------------------------------------------------------------------------- 5084 // [EmitOp - PC Relative] 5085 // -------------------------------------------------------------------------- 5086 5087 EmitOp_Rel: 5088 { 5089 if (rm_rel->is_label() || rm_rel->is_mem()) { 5090 uint32_t label_id; 5091 int64_t label_offset = 0; 5092 5093 if (rm_rel->is_label()) { 5094 label_id = rm_rel->as<Label>().id(); 5095 } 5096 else { 5097 label_id = rm_rel->as<Mem>().base_id(); 5098 label_offset = rm_rel->as<Mem>().offset(); 5099 } 5100 5101 if (ASMJIT_UNLIKELY(!_code->is_label_valid(label_id))) { 5102 goto InvalidLabel; 5103 } 5104 5105 LabelEntry& le = _code->label_entry_of(label_id); 5106 5107 if (offset_format.type() == OffsetType::kAArch64_ADRP) { 5108 // TODO: [ARM] Always create relocation entry. 5109 } 5110 5111 if (le.is_bound_to(_section)) { 5112 // Label bound to the current section. 5113 offset_value = le.offset() - uint64_t(offset()) + uint64_t(label_offset); 5114 goto EmitOp_DispImm; 5115 } 5116 else { 5117 // Create a fixup referencing an non-bound label. 5118 size_t code_offset = writer.offset_from(_buffer_data); 5119 Fixup* fixup = _code->new_fixup(le, _section->section_id(), code_offset, intptr_t(label_offset), offset_format); 5120 5121 if (ASMJIT_UNLIKELY(!fixup)) { 5122 goto OutOfMemory; 5123 } 5124 5125 goto EmitOp; 5126 } 5127 } 5128 } 5129 5130 if (rm_rel->is_imm()) { 5131 uint64_t base_address = _code->base_address(); 5132 uint64_t target_offset = rm_rel->as<Imm>().value_as<uint64_t>(); 5133 5134 size_t code_offset = writer.offset_from(_buffer_data); 5135 5136 if (base_address == Globals::kNoBaseAddress || _section->section_id() != 0) { 5137 // Create a new RelocEntry as we cannot calculate the offset right now. 5138 RelocEntry* re; 5139 err = _code->new_reloc_entry(Out(re), RelocType::kAbsToRel); 5140 if (err != Error::kOk) { 5141 goto Failed; 5142 } 5143 5144 re->_source_section_id = _section->section_id(); 5145 re->_source_offset = code_offset; 5146 re->_format = offset_format; 5147 re->_payload = rm_rel->as<Imm>().value_as<uint64_t>() + 4u; 5148 goto EmitOp; 5149 } 5150 else { 5151 uint64_t pc = base_address + code_offset; 5152 5153 if (offset_format.type() == OffsetType::kAArch64_ADRP) { 5154 pc &= ~uint64_t(4096 - 1); 5155 } 5156 5157 offset_value = target_offset - pc; 5158 goto EmitOp_DispImm; 5159 } 5160 } 5161 5162 goto InvalidInstruction; 5163 5164 EmitOp_DispImm: 5165 { 5166 if ((offset_value & Support::lsb_mask<uint32_t>(offset_format.imm_discard_lsb())) != 0) { 5167 goto InvalidDisplacement; 5168 } 5169 5170 int64_t disp_imm64 = int64_t(offset_value) >> offset_format.imm_discard_lsb(); 5171 if (!Support::is_encodable_offset_64(disp_imm64, offset_format.imm_bit_count())) { 5172 goto InvalidDisplacement; 5173 } 5174 5175 uint32_t disp_imm32 = uint32_t(disp_imm64 & Support::lsb_mask<uint32_t>(offset_format.imm_bit_count())); 5176 switch (offset_format.type()) { 5177 case OffsetType::kSignedOffset: { 5178 opcode.add_imm(disp_imm32, offset_format.imm_bit_shift()); 5179 goto EmitOp; 5180 } 5181 5182 case OffsetType::kAArch64_ADR: 5183 case OffsetType::kAArch64_ADRP: { 5184 uint32_t imm_lo = disp_imm32 & 0x3u; 5185 uint32_t imm_hi = disp_imm32 >> 2; 5186 opcode.add_imm(imm_lo, 29); 5187 opcode.add_imm(imm_hi, 5); 5188 goto EmitOp; 5189 } 5190 5191 default: 5192 goto InvalidDisplacement; 5193 } 5194 } 5195 5196 // -------------------------------------------------------------------------- 5197 // [EmitOp - Opcode] 5198 // -------------------------------------------------------------------------- 5199 5200 EmitOp: 5201 writer.emit32u_le(opcode.get()); 5202 goto EmitDone; 5203 5204 // -------------------------------------------------------------------------- 5205 // [Done] 5206 // -------------------------------------------------------------------------- 5207 5208 EmitDone: 5209 if (Support::test(options, InstOptions::kReserved)) { 5210 #ifndef ASMJIT_NO_LOGGING 5211 if (_logger) { 5212 EmitterUtils::log_instruction_emitted(this, BaseInst::compose_arm_inst_id(inst_id, inst_cc), options, o0, o1, o2, op_ext, 0, 0, writer.cursor()); 5213 } 5214 #endif 5215 } 5216 5217 reset_state(); 5218 5219 writer.done(this); 5220 return Error::kOk; 5221 5222 // -------------------------------------------------------------------------- 5223 // [Error Handler] 5224 // -------------------------------------------------------------------------- 5225 5226 #define ERROR_HANDLER(ERR) ERR: err = make_error(Error::k##ERR); goto Failed; 5227 ERROR_HANDLER(OutOfMemory) 5228 ERROR_HANDLER(InvalidAddress) 5229 ERROR_HANDLER(InvalidAddressScale) 5230 ERROR_HANDLER(InvalidDisplacement) 5231 ERROR_HANDLER(InvalidElementIndex) 5232 ERROR_HANDLER(InvalidLabel) 5233 ERROR_HANDLER(InvalidImmediate) 5234 ERROR_HANDLER(InvalidInstruction) 5235 ERROR_HANDLER(InvalidPhysId) 5236 ERROR_HANDLER(InvalidRegType) 5237 #undef ERROR_HANDLER 5238 5239 Failed: 5240 #ifndef ASMJIT_NO_LOGGING 5241 return EmitterUtils::log_instruction_failed(this, err, inst_id, options, o0, o1, o2, op_ext); 5242 #else 5243 reset_state(); 5244 return report_error(err); 5245 #endif 5246 } 5247 5248 #undef ENC_OPS1 5249 #undef ENC_OPS2 5250 #undef ENC_OPS3 5251 #undef ENC_OPS4 5252 5253 // a64::Assembler - Align 5254 // ====================== 5255 5256 Error Assembler::align(AlignMode align_mode, uint32_t alignment) { 5257 constexpr uint32_t kNopA64 = 0xD503201Fu; // [11010101|00000011|00100000|00011111]. 5258 5259 if (ASMJIT_UNLIKELY(!_code)) { 5260 return report_error(make_error(Error::kNotInitialized)); 5261 } 5262 5263 if (ASMJIT_UNLIKELY(uint32_t(align_mode) > uint32_t(AlignMode::kMaxValue))) { 5264 return report_error(make_error(Error::kInvalidArgument)); 5265 } 5266 5267 if (alignment <= 1) { 5268 return Error::kOk; 5269 } 5270 5271 if (ASMJIT_UNLIKELY(!Support::is_power_of_2_up_to(alignment, Globals::kMaxAlignment))) { 5272 return report_error(make_error(Error::kInvalidArgument)); 5273 } 5274 5275 uint32_t i = uint32_t(Support::align_up_diff<size_t>(offset(), alignment)); 5276 if (i == 0) { 5277 return Error::kOk; 5278 } 5279 5280 CodeWriter writer(this); 5281 ASMJIT_PROPAGATE(writer.ensure_space(this, i)); 5282 5283 switch (align_mode) { 5284 case AlignMode::kCode: { 5285 uint32_t pattern = kNopA64; 5286 5287 if (ASMJIT_UNLIKELY(offset() & 0x3u)) { 5288 return make_error(Error::kInvalidState); 5289 } 5290 5291 while (i >= 4) { 5292 writer.emit32u_le(pattern); 5293 i -= 4; 5294 } 5295 5296 ASMJIT_ASSERT(i == 0); 5297 break; 5298 } 5299 5300 case AlignMode::kData: 5301 case AlignMode::kZero: 5302 writer.emit_zeros(i); 5303 break; 5304 } 5305 5306 writer.done(this); 5307 5308 #ifndef ASMJIT_NO_LOGGING 5309 if (_logger) { 5310 StringTmp<128> sb; 5311 sb.append_chars(' ', _logger->indentation(FormatIndentationGroup::kCode)); 5312 sb.append_format("align %u\n", alignment); 5313 _logger->log(sb); 5314 } 5315 #endif 5316 5317 return Error::kOk; 5318 } 5319 5320 // a64::Assembler - Events 5321 // ======================= 5322 5323 Error Assembler::on_attach(CodeHolder& code) noexcept { 5324 ASMJIT_PROPAGATE(Base::on_attach(code)); 5325 5326 _instruction_alignment = uint8_t(4); 5327 update_emitter_funcs(this); 5328 5329 return Error::kOk; 5330 } 5331 5332 Error Assembler::on_detach(CodeHolder& code) noexcept { 5333 return Base::on_detach(code); 5334 } 5335 5336 ASMJIT_END_SUB_NAMESPACE 5337 5338 #endif // !ASMJIT_NO_AARCH64