fetchpatternpart.cpp (80694B)
1 // This file is part of Blend2D project <https://blend2d.com> 2 // 3 // See blend2d.h or LICENSE.md for license and copyright information 4 // SPDX-License-Identifier: Zlib 5 6 #include "../../api-build_p.h" 7 #if !defined(BL_BUILD_NO_JIT) 8 9 #include "../../pipeline/jit/compoppart_p.h" 10 #include "../../pipeline/jit/fetchpatternpart_p.h" 11 #include "../../pipeline/jit/fetchutilsbilinear_p.h" 12 #include "../../pipeline/jit/fetchutilspixelaccess_p.h" 13 #include "../../pipeline/jit/pipecompiler_p.h" 14 #include "../../support/intops_p.h" 15 16 namespace bl::Pipeline::JIT { 17 18 #define REL_PATTERN(FIELD) BL_OFFSET_OF(FetchData::Pattern, FIELD) 19 20 // bl::Pipeline::JIT::FetchPadRoRContext 21 // ===================================== 22 23 class FetchPadRoRContext { 24 public: 25 PipeCompiler* pc {}; 26 FetchSimplePatternPart* _fetch_part {}; 27 28 //! Whether the pattern fetcher has fractional X. 29 bool _has_frac_x {}; 30 //! Whether the pattern fetcher has fractional Y. 31 bool _has_frac_y {}; 32 33 //! Horizontal extend mode. 34 ExtendMode _extend_x {}; 35 36 //! Describes the current pixel index. 37 uint32_t _fetch_index {}; 38 39 //! Describes the current index related to advancing. 40 //! 41 //! \note Fetch index and advance index could be different. In Fx and FxFy case, the fetcher needs to advance before 42 //! a next index is calculated, because the current index was already either pre-fetched or fetched by previous loop 43 //! iteration, which implies that we never want to fetch the current index again. The reason we use two counters is 44 //! actually simplicity - `next_index()` uses `_fetch_index` and `_advance_pad_x()` uses `_advance_index`. 45 uint32_t _advance_index {}; 46 47 //! Index extractor used to extract indexes from a vector so we can use them in regular [base + index] address. 48 FetchUtils::IndexExtractor _index_extractor; 49 50 Gp _x; 51 Gp _w; 52 Gp _idx; 53 Gp _predicate_count; 54 55 BL_NOINLINE explicit FetchPadRoRContext(FetchSimplePatternPart* fetch_part, PixelPredicate& predicate) noexcept 56 : pc(fetch_part->pc), 57 _fetch_part(fetch_part), 58 _has_frac_x(fetch_part->has_frac_x()), 59 _has_frac_y(fetch_part->has_frac_y()), 60 _extend_x(fetch_part->extend_x()), 61 _fetch_index(0), 62 _advance_index(0), 63 _index_extractor(fetch_part->pc) { 64 65 if (!predicate.is_empty()) { 66 _predicate_count = predicate.count(); 67 } 68 69 FetchSimplePatternPart::SimpleRegs* f = &_fetch_part->f; 70 71 if (_extend_x == ExtendMode::kPad) { 72 _x = f->x; 73 _w = f->w; 74 _idx = f->x_padded; 75 } 76 else { 77 _idx = pc->new_gpz("@idx"); 78 } 79 } 80 81 BL_INLINE_NODEBUG bool has_predicate() const noexcept { return _predicate_count.is_valid(); } 82 83 BL_NOINLINE void begin() noexcept { 84 if (_extend_x == ExtendMode::kPad) { 85 // Nothing to setup here as each index is calculated by advancing `x` and then padding to `x_padded`. 86 } 87 else { 88 FetchSimplePatternPart::SimpleRegs* f = &_fetch_part->f; 89 Vec v_idx = pc->new_vec128("@v_idx"); 90 Vec v_src = f->x_vec_4; 91 92 if (has_predicate()) { 93 Gp gp_off = pc->gpz(_predicate_count); 94 Vec v_off = pc->new_vec128("@v_off"); 95 96 Mem m = pc->tmp_stack(PipeCompiler::StackId::kCustom, 16); 97 v_src = pc->new_vec128("@v_src"); 98 99 pc->v_storea128(m, f->x_set_4); 100 101 #if defined(BL_JIT_ARCH_X86) 102 m.set_index(gp_off, 2); 103 #else 104 Gp gp_base = pc->new_gpz("@base"); 105 pc->cc->load_address_of(gp_base, m); 106 m = mem_ptr(gp_base, gp_off, 2); 107 #endif 108 pc->v_broadcast_u32(v_off, m); 109 110 pc->shift_or_rotate_left(v_src, f->x_vec_4, 4); 111 pc->v_add_i32(f->x_vec_4, f->x_vec_4, v_off); 112 _fixup_reflected_x(); 113 pc->v_alignr_u128(v_src, f->x_vec_4, v_src, 4); 114 } 115 116 pc->v_srai_i32(v_idx, v_src, 31); 117 pc->v_xor_i32(v_idx, v_idx, v_src); 118 119 if (!has_predicate()) { 120 pc->v_add_i32(f->x_vec_4, f->x_vec_4, f->x_inc_4); 121 } 122 123 _index_extractor.begin(FetchUtils::IndexExtractor::kTypeUInt32, v_idx); 124 } 125 } 126 127 BL_NOINLINE void end() noexcept { 128 if (_extend_x == ExtendMode::kPad) { 129 if (!_has_frac_x) { 130 _advance_pad_x(); 131 } 132 } 133 else { 134 if (!has_predicate()) { 135 _fixup_reflected_x(); 136 } 137 } 138 } 139 140 BL_NOINLINE Gp next_index() noexcept { 141 if (_extend_x == ExtendMode::kPad) { 142 if (_has_frac_x || _fetch_index != 0) { 143 _advance_pad_x(); 144 } 145 146 _fetch_index++; 147 return _idx; 148 } 149 else { 150 _index_extractor.extract(_idx.r32(), _fetch_index); 151 152 _fetch_index++; 153 return _idx; 154 } 155 } 156 157 BL_NOINLINE void _advance_pad_x() noexcept { 158 if (has_predicate() && _advance_index >= 2u) { 159 // Make the last fetch point to the last predicated value, which would be correct if the pattern gets advanced. 160 if (_advance_index == 2) { 161 pc->add_ext(_x, _x, _predicate_count.clone_as(_x), 1, -2); 162 pc->cmov(_idx.r32(), _x.r32(), ucmp_le(_x, _w)); 163 } 164 } 165 else { 166 pc->inc(_x); 167 pc->cmov(_idx.r32(), _x.r32(), ucmp_le(_x, _w)); 168 } 169 170 _advance_index++; 171 } 172 173 BL_NOINLINE void _fixup_reflected_x() noexcept { 174 FetchSimplePatternPart::SimpleRegs* f = &_fetch_part->f; 175 Vec v_tmp = pc->new_vec128("v_tmp"); 176 177 pc->v_cmp_gt_i32(v_tmp, f->x_vec_4, f->x_max_4); 178 pc->v_and_i32(v_tmp, v_tmp, f->x_nrm_4); 179 pc->v_sub_i32(f->x_vec_4, f->x_vec_4, v_tmp); 180 } 181 }; 182 183 // bl::Pipeline::JIT::FetchPatternPart - Construction & Destruction 184 // ================================================================ 185 186 FetchPatternPart::FetchPatternPart(PipeCompiler* pc, FetchType fetch_type, FormatExt format) noexcept 187 : FetchPart(pc, fetch_type, format) {} 188 189 // bl::Pipeline::JIT::FetchSimplePatternPart - Construction & Destruction 190 // ====================================================================== 191 192 FetchSimplePatternPart::FetchSimplePatternPart(PipeCompiler* pc, FetchType fetch_type, FormatExt format) noexcept 193 : FetchPatternPart(pc, fetch_type, format) { 194 195 static constexpr ExtendMode fExtendTable[] = { ExtendMode::kPad, ExtendMode::kRoR }; 196 197 _part_flags |= PipePartFlags::kAdvanceXNeedsDiff; 198 _idx_shift = 0; 199 _max_pixels = 4; 200 201 // Setup registers, extend mode, and the maximum number of pixels that can be fetched at once. 202 switch (fetch_type) { 203 case FetchType::kPatternAlignedBlit: 204 _part_flags |= PipePartFlags::kAdvanceXIsSimple; 205 _max_vec_width_supported = kMaxPlatformWidth; 206 _max_pixels = kUnlimitedMaxPixels; 207 208 if (pc->has_masked_access_of(bpp())) 209 _part_flags |= PipePartFlags::kMaskedAccess; 210 break; 211 212 case FetchType::kPatternAlignedPad: 213 // TODO: [JIT] OPTIMIZATION: We have removed fetch2x4, so `_max_pixels` cannot be raised to 8. 214 // _max_pixels = 8; 215 _extend_x = ExtendMode::kPad; 216 break; 217 218 case FetchType::kPatternAlignedRepeat: 219 _extend_x = ExtendMode::kRepeat; 220 #if defined(BL_JIT_ARCH_X86) 221 _max_vec_width_supported = VecWidth::k256; 222 #endif // BL_JIT_ARCH_X86 223 break; 224 225 case FetchType::kPatternAlignedRoR: 226 _extend_x = ExtendMode::kRoR; 227 break; 228 229 case FetchType::kPatternFxPad: 230 case FetchType::kPatternFxRoR: 231 _extend_x = fExtendTable[uint32_t(fetch_type) - uint32_t(FetchType::kPatternFxPad)]; 232 break; 233 234 case FetchType::kPatternFyPad: 235 case FetchType::kPatternFyRoR: 236 _extend_x = fExtendTable[uint32_t(fetch_type) - uint32_t(FetchType::kPatternFyPad)]; 237 break; 238 239 case FetchType::kPatternFxFyPad: 240 case FetchType::kPatternFxFyRoR: 241 _extend_x = fExtendTable[uint32_t(fetch_type) - uint32_t(FetchType::kPatternFxFyPad)]; 242 add_part_flags(PipePartFlags::kExpensive); 243 break; 244 245 default: 246 BL_NOT_REACHED(); 247 } 248 249 if (extend_x() == ExtendMode::kPad || extend_x() == ExtendMode::kRoR) { 250 if (IntOps::is_power_of_2(_bpp)) 251 _idx_shift = uint8_t(IntOps::ctz(_bpp)); 252 } 253 254 OpUtils::reset_var_struct(&f, sizeof(f)); 255 } 256 257 // bl::Pipeline::JIT::FetchSimplePatternPart - Init & Fini 258 // ======================================================= 259 260 void FetchSimplePatternPart::_init_part(const PipeFunction& fn, Gp& x, Gp& y) noexcept { 261 if (is_aligned_blit()) { 262 // This is a special-case designed only for rectangular blits that never 263 // go out of image bounds (this implies that no extend mode is applied). 264 BL_ASSERT(is_rect_fill()); 265 266 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 267 f->stride = pc->new_gpz("f.stride"); // Mem. 268 f->srcp1 = pc->new_gpz("f.srcp1"); // Reg. 269 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 270 271 pc->load(f->stride, mem_ptr(fn.fetch_data(), REL_PATTERN(src.stride))); 272 pc->sub(f->srcp1.r32(), y.r32(), mem_ptr(fn.fetch_data(), REL_PATTERN(simple.ty))); 273 pc->mul(f->srcp1, f->srcp1, f->stride); 274 275 pc->add(f->srcp1, f->srcp1, mem_ptr(fn.fetch_data(), REL_PATTERN(src.pixel_data))); 276 pc->prefetch(mem_ptr(f->srcp1)); 277 278 Gp cut = pc->new_gpz("@stride_cut"); 279 pc->mul(cut.r32(), mem_ptr(fn.fetch_data(), REL_PATTERN(src.size.w)), int(bpp())); 280 pc->sub(f->stride, f->stride, cut); 281 } 282 else { 283 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 284 f->srcp0 = pc->new_gpz("f.srcp0"); // Reg. 285 f->srcp1 = pc->new_gpz("f.srcp1"); // Reg (Fy|FxFy). 286 f->w = pc->new_gp32("f.w"); // Mem. 287 f->h = pc->new_gp32("f.h"); // Mem. 288 f->y = pc->new_gp32("f.y"); // Reg. 289 290 f->stride = pc->new_gpz("f.stride"); // Init only. 291 f->ry = pc->new_gp32("f.ry"); // Init only. 292 f->v_extend_data = cc->new_stack(sizeof(FetchData::Pattern::VertExtendData), 16, "f.v_extend_data"); 293 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 294 295 // Apply alpha offset to source pointers. 296 if (_alpha_fetch && extend_x() != ExtendMode::kRepeat) { 297 _fetch_info.apply_alpha_offset(); 298 } 299 300 pc->add(f->y, y, mem_ptr(fn.fetch_data(), REL_PATTERN(simple.ty))); 301 302 // The idea is that both Fx and Fy are compatible with FxFy so we increment Y if this is Fx only fetch. 303 if (is_pattern_fx()) { 304 pc->inc(f->y); 305 } 306 307 pc->load_u32(f->h, mem_ptr(fn.fetch_data(), REL_PATTERN(src.size.h))); 308 pc->load_u32(f->ry, mem_ptr(fn.fetch_data(), REL_PATTERN(simple.ry))); 309 pc->load(f->stride, mem_ptr(fn.fetch_data(), REL_PATTERN(src.stride))); 310 311 // Vertical Extend 312 // --------------- 313 // 314 // Vertical extend modes are not hardcoded in the generated pipeline to decrease the number of possible pipeline 315 // combinations. This means that the compiled pipeline supports all vertical extend modes. The amount of code that 316 // handles vertical extend modes has been minimized so runtime overhead during `advance_y()` should be negligible. 317 318 { 319 // Vertical Extend - Prepare 320 // ------------------------- 321 322 Label L_VertRoR = pc->new_label(); 323 Label L_VertSwap = pc->new_label(); 324 Label L_VertDone = pc->new_label(); 325 326 Gp y_mod = pc->new_gpz("f.y_mod").r32(); 327 Gp hMinus1 = pc->new_gpz("f.hMinus1").r32(); 328 Gp yModReg = y_mod.clone_as(f->stride); 329 330 VecArray vStrideStopVec; 331 Vec vRewindDataVec = pc->new_vec128("f.vRewindData"); 332 333 if (pc->is_32bit()) { 334 pc->new_vec_array(vStrideStopVec, 1, VecWidth::k128, "f.vStrideStopVec"); 335 336 constexpr int kRewindDataOffset = 16; 337 pc->v_loadu64(vRewindDataVec, mem_ptr(fn.fetch_data(), REL_PATTERN(simple.v_extend_data) + kRewindDataOffset)); 338 pc->v_storeu64(f->v_extend_data.clone_adjusted(kRewindDataOffset), vRewindDataVec); 339 } 340 else { 341 constexpr int kRewindDataOffset = 32; 342 343 #if defined(BL_JIT_ARCH_X86) 344 if (pc->has_avx2()) 345 { 346 pc->new_vec_array(vStrideStopVec, 1, VecWidth::k256, "f.vStrideStopVec"); 347 } 348 else 349 #endif // BL_JIT_ARCH_X86 350 { 351 pc->new_vec_array(vStrideStopVec, 2, VecWidth::k128, "f.vStrideStopVec"); 352 } 353 354 pc->v_loadu128(vRewindDataVec, mem_ptr(fn.fetch_data(), REL_PATTERN(simple.v_extend_data) + kRewindDataOffset)); 355 pc->v_storea128(f->v_extend_data.clone_adjusted(kRewindDataOffset), vRewindDataVec); 356 } 357 358 pc->v_loadavec(vStrideStopVec, mem_ptr(fn.fetch_data(), REL_PATTERN(simple.v_extend_data)), Alignment(8)); 359 360 // Don't do anything if we are within bounds as this is the case v_extend_data was prepared for. 361 pc->mov(y_mod, f->y); 362 pc->j(L_VertDone, ucmp_lt(f->y, f->h)); 363 364 // Decide between PAD and RoR. 365 pc->j(L_VertRoR, test_nz(f->ry)); 366 367 // Handle PAD - we know that we are outside of bounds, so y_mod would become either 0 or h-1. 368 pc->sar(y_mod, y_mod, 31); 369 pc->sub(hMinus1, f->h, 1); 370 371 pc->bic(y_mod, hMinus1, y_mod); 372 pc->j(L_VertSwap); 373 374 // Handle RoR - we have to repeat to `ry`, which is double the height in reflect case. 375 pc->bind(L_VertRoR); 376 pc->umod(f->y, f->y, f->ry); 377 pc->mov(y_mod, f->y); 378 379 // If we are within bounds already it means this is either repeat or reflection, which is in repeat part. 380 pc->j(L_VertDone, ucmp_lt(f->y, f->h)); 381 382 // We are reflecting at the moment, `y_mod` has to be updated. 383 pc->sub(y_mod, y_mod, f->ry); 384 pc->sub(f->y, f->y, f->h); 385 pc->not_(y_mod, y_mod); 386 387 // Vertical Extend - Done 388 // ---------------------- 389 390 pc->bind(L_VertSwap); 391 swap_stride_stop_data(vStrideStopVec); 392 393 pc->bind(L_VertDone); 394 pc->mul(yModReg, yModReg, f->stride); 395 pc->v_storeavec(f->v_extend_data, vStrideStopVec, Alignment(16)); 396 pc->add(f->srcp1, y_mod.clone_as(f->srcp1), mem_ptr(fn.fetch_data(), REL_PATTERN(src.pixel_data))); 397 398 if (_fetch_info.applied_offset()) { 399 pc->add(f->srcp1, f->srcp1, _fetch_info.applied_offset()); 400 } 401 } 402 403 // Horizontal Extend 404 // ----------------- 405 // 406 // Horizontal extend modes are hardcoded for performance reasons. Every extend mode 407 // requires different strategy to make horizontal advancing as fast as possible. 408 409 if (extend_x() == ExtendMode::kPad) { 410 // Horizontal Pad 411 // -------------- 412 // 413 // There is not much to invent to clamp horizontally. The `f->x` is a raw coordinate that is clamped each 414 // time it's used as an index. To make it fast we use two variables `x` and `x_padded`, which always contains 415 // `x` clamped to `[x, w]` range. The advantage of this approach is that every time we increment `1` to `x` we 416 // need only 2 instructions to calculate new `x_padded` value as it was already padded to the previous index, 417 // and it could only get greater by `1` or stay where it was in a case we already reached the width `w`. 418 419 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 420 f->x = pc->new_gp32("f.x"); // Reg. 421 f->x_padded = pc->new_gpz("f.x_padded"); // Reg. 422 f->x_origin = pc->new_gp32("f.x_origin"); // Mem. 423 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 424 425 pc->load_u32(f->w, mem_ptr(fn.fetch_data(), REL_PATTERN(src.size.w))); 426 pc->load_u32(f->x_origin, mem_ptr(fn.fetch_data(), REL_PATTERN(simple.tx))); 427 428 // Fy pattern falls to Fx/Fy/FxFy category, which means that it's compatible with FxFy, we must increment the 429 // X origin in that case as we know that weights for the first pixel are all zeros (compatibility with FxFy). 430 if (is_pattern_fy()) { 431 pc->inc(f->x_origin); 432 } 433 434 if (is_rect_fill()) { 435 pc->add(f->x_origin, f->x_origin, x); 436 } 437 438 pc->dec(f->w); 439 } 440 441 if (extend_x() == ExtendMode::kRepeat) { 442 // Horizontal Repeat - AA-Only, Large Fills 443 // ---------------------------------------- 444 // 445 // This extend mode is only used to blit patterns that are tiled and that exceed some predefined width-limit 446 // (like 16|32|etc). It's specialized for larger patterns because it contains a condition in fetchN() that 447 // jumps if `f->x` is at the end or near of the patterns end. That's why the pattern width should be large 448 // enough that this branch is not mispredicted often. For smaller patterns RoR more is more suitable as there 449 // is no branch required and the repeat|reflect is handled by SIMD instructions. 450 // 451 // This implementation generally uses two tricks to make the tiling faster: 452 // 453 // 1. It changes row indexing from [0..width) to [-width..0). The reason for such change is that when ADD 454 // instruction is executed it updates processor FLAGS register, if SIGN flag is zero it means that repeat 455 // is needed. This saves us one condition. 456 // 457 // 2. It multiplies X coordinates (all of them) by pattern's BPP (bytes per pixel). The reason is to completely 458 // eliminate `index * scale` in memory addressing (and in case of weird BPP to eliminate IMUL). 459 460 // NOTE: These all must be `intptr_t` because of memory indexing and the 461 // use of the sign (when f->x is used as an index it's always negative). 462 463 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 464 f->x = pc->new_gpz("f.x"); // Reg. 465 f->x_origin = pc->new_gpz("f.x_origin"); // Mem. 466 f->x_restart = pc->new_gpz("f.x_restart"); // Mem. 467 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 468 469 pc->load_u32(f->w, mem_ptr(fn.fetch_data(), REL_PATTERN(src.size.w))); 470 pc->load_u32(f->x_origin.r32(), mem_ptr(fn.fetch_data(), REL_PATTERN(simple.tx))); 471 472 if (is_pattern_fy()) 473 pc->inc(f->x_origin.r32()); 474 475 if (is_rect_fill()) { 476 pc->add(f->x_origin.r32(), f->x_origin.r32(), x); 477 pc->umod(f->x_origin.r32(), f->x_origin.r32(), f->w); 478 } 479 480 pc->mul(f->w , f->w , int(bpp())); 481 pc->mul(f->x_origin, f->x_origin, int(bpp())); 482 483 pc->sub(f->x_origin, f->x_origin, f->w.clone_as(f->x_origin)); 484 pc->add(f->srcp1, f->srcp1, f->w.clone_as(f->srcp1)); 485 pc->neg(f->x_restart, f->w.clone_as(f->x_restart)); 486 } 487 488 if (extend_x() == ExtendMode::kRoR) { 489 // Horizontal RoR [Repeat or Reflect] 490 // ---------------------------------- 491 // 492 // This mode handles both Repeat and Reflect cases. It uses the following formula to either REPEAT or REFLECT 493 // X coordinate: 494 // 495 // int index = (x >> 31) ^ x; 496 // 497 // The beauty of this method is that if X is negative it reflects, if it's positive it's kept as is. Then the 498 // implementation handles both modes the following way: 499 // 500 // 1. REPEAT - X is always bound to interval [0...Width), so when the index is calculated it never reflects. 501 // When `f->x` reaches the pattern width it's simply corrected as `f->x -= f->rx`, where `f->rx` is equal 502 // to `pattern.size.w`. 503 // 504 // 2. REFLECT - X is always bound to interval [-Width...Width) so it can reflect. When `f->x` reaches the 505 // pattern width it's simply corrected as `f->x -= f->rx`, where `f->rx` is equal to `pattern.size.w * 2` 506 // so it goes negative. 507 508 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 509 f->x = pc->new_gp32("f.x"); // Reg. 510 f->x_origin = pc->new_gp32("f.x_origin"); // Mem. 511 f->x_restart = pc->new_gp32("f.x_restart"); // Mem. 512 f->rx = pc->new_gp32("f.rx"); // Mem. 513 514 if (max_pixels() >= 4) { 515 f->x_vec_4 = pc->new_vec128("f.x_vec_4"); // Reg (fetchN). 516 f->x_set_4 = pc->new_vec128("f.x_set_4"); // Mem (fetchN). 517 f->x_inc_4 = pc->new_vec128("f.x_inc_4"); // Mem (fetchN). 518 f->x_nrm_4 = pc->new_vec128("f.x_nrm_4"); // Mem (fetchN). 519 f->x_max_4 = pc->new_vec128("f.x_max_4"); // Mem (fetchN). 520 } 521 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 522 523 pc->load_u32(f->w , mem_ptr(fn.fetch_data(), REL_PATTERN(src.size.w))); 524 pc->load_u32(f->rx, mem_ptr(fn.fetch_data(), REL_PATTERN(simple.rx))); 525 526 if (max_pixels() >= 4) { 527 pc->v_cvt_u8_to_u32(f->x_set_4, mem_ptr(fn.fetch_data(), REL_PATTERN(simple.ix))); 528 pc->v_swizzle_u32x4(f->x_inc_4, f->x_set_4, swizzle(3, 3, 3, 3)); 529 530 if (!has_frac_x()) { 531 pc->v_sllb_u128(f->x_set_4, f->x_set_4, 4); 532 } 533 } 534 535 pc->sub(f->x_restart, f->w, f->rx); 536 pc->dec(f->w); 537 538 if (max_pixels() >= 4) { 539 pc->v_broadcast_u32(f->x_max_4, f->w); 540 pc->v_broadcast_u32(f->x_nrm_4, f->rx); 541 } 542 543 pc->load_u32(f->x_origin, mem_ptr(fn.fetch_data(), REL_PATTERN(simple.tx))); 544 545 if (is_pattern_fy()) { 546 pc->inc(f->x_origin); 547 } 548 549 if (is_rect_fill()) { 550 Gp norm = pc->new_gp32("@norm"); 551 552 pc->add(f->x_origin, f->x_origin, x); 553 pc->umod(f->x_origin, f->x_origin, f->rx); 554 555 pc->select(norm, Imm(0), f->rx, ucmp_le(f->x_origin, f->w)); 556 pc->sub(f->x_origin, f->x_origin, norm); 557 } 558 } 559 560 // Fractional - Fx|Fy|FxFy 561 // ----------------------- 562 563 if (is_pattern_unaligned()) { 564 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 565 f->pix_l = pc->new_vec128("f.pix_l"); // Reg (Fx|FxFy). 566 567 f->wa = pc->new_vec128("f.wa"); // Reg/Mem (RGBA mode). 568 f->wb = pc->new_vec128("f.wb"); // Reg/Mem (RGBA mode). 569 f->wc = pc->new_vec128("f.wc"); // Reg/Mem (RGBA mode). 570 f->wd = pc->new_vec128("f.wd"); // Reg/Mem (RGBA mode). 571 572 f->wc_wd = pc->new_vec128("f.wc_wd"); // Reg/Mem (RGBA mode). 573 f->wa_wb = pc->new_vec128("f.wa_wb"); // Reg/Mem (RGBA mode). 574 575 f->wd_wb = pc->new_vec128("f.wd_wb"); // Reg/Mem (Alpha mode). 576 f->wa_wc = pc->new_vec128("f.wa_wc"); // Reg/Mem (Alpha mode). 577 f->wb_wd = pc->new_vec128("f.wb_wd"); // Reg/Mem (Alpha mode). 578 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 579 580 Vec weights = pc->new_vec128("weights"); 581 Mem wPtr = mem_ptr(fn.fetch_data(), REL_PATTERN(simple.wa)); 582 583 // [00 Wd 00 Wc 00 Wb 00 Wa] 584 pc->v_loadu128(weights, wPtr); 585 // [Wd Wc Wb Wa Wd Wc Wb Wa] 586 pc->v_packs_i32_i16(weights, weights, weights); 587 588 if (is_alpha_fetch()) { 589 if (is_pattern_fy()) { 590 pc->v_swizzle_lo_u16x4(f->wd_wb, weights, swizzle(3, 1, 3, 1)); 591 if (max_pixels() >= 4) { 592 pc->v_swizzle_u32x4(f->wd_wb, f->wd_wb, swizzle(1, 0, 1, 0)); 593 } 594 } 595 else if (is_pattern_fx()) { 596 pc->v_swizzle_u32x4(f->wc_wd, weights, swizzle(3, 3, 3, 3)); 597 } 598 else { 599 pc->v_swizzle_lo_u16x4(f->wa_wc, weights, swizzle(2, 0, 2, 0)); 600 pc->v_swizzle_lo_u16x4(f->wb_wd, weights, swizzle(3, 1, 3, 1)); 601 if (max_pixels() >= 4) { 602 pc->v_swizzle_u32x4(f->wa_wc, f->wa_wc, swizzle(1, 0, 1, 0)); 603 pc->v_swizzle_u32x4(f->wb_wd, f->wb_wd, swizzle(1, 0, 1, 0)); 604 } 605 } 606 } 607 else { 608 // [Wd Wd Wc Wc Wb Wb Wa Wa] 609 pc->v_interleave_lo_u16(weights, weights, weights); 610 611 if (is_pattern_fy()) { 612 pc->v_swizzle_u32x4(f->wb, weights, swizzle(1, 1, 1, 1)); 613 pc->v_swizzle_u32x4(f->wd, weights, swizzle(3, 3, 3, 3)); 614 } 615 else if (is_pattern_fx()) { 616 pc->v_swizzle_u32x4(f->wc_wd, weights, swizzle(3, 3, 2, 2)); 617 if (max_pixels() >= 4) { 618 pc->v_swizzle_u32x4(f->wc, weights, swizzle(2, 2, 2, 2)); 619 pc->v_swizzle_u32x4(f->wd, weights, swizzle(3, 3, 3, 3)); 620 } 621 } 622 else { 623 pc->v_swizzle_u32x4(f->wa_wc, weights, swizzle(0, 0, 2, 2)); 624 pc->v_swizzle_u32x4(f->wb_wd, weights, swizzle(1, 1, 3, 3)); 625 626 if (max_pixels() >= 4) { 627 pc->v_swizzle_u32x4(f->wa, weights, swizzle(0, 0, 0, 0)); 628 pc->v_swizzle_u32x4(f->wb, weights, swizzle(1, 1, 1, 1)); 629 pc->v_swizzle_u32x4(f->wc, weights, swizzle(2, 2, 2, 2)); 630 pc->v_swizzle_u32x4(f->wd, weights, swizzle(3, 3, 3, 3)); 631 } 632 } 633 } 634 } 635 636 // If the pattern has a fractional Y then advance in vertical direction. 637 // This ensures that both `srcp0` and `srcp1` are initialized, otherwise 638 // `srcp0` would contain undefined content. 639 if (has_frac_y()) { 640 advance_y(); 641 } 642 } 643 } 644 645 void FetchSimplePatternPart::_fini_part() noexcept {} 646 647 // bl::Pipeline::JIT::FetchSimplePatternPart - Utilities 648 // ===================================================== 649 650 void FetchSimplePatternPart::swap_stride_stop_data(VecArray& v) noexcept { 651 if (pc->is_32bit()) 652 pc->v_swap_u32(v, v); 653 else 654 pc->v_swap_u64(v, v); 655 } 656 657 // bl::Pipeline::JIT::FetchSimplePatternPart - Advance 658 // =================================================== 659 660 void FetchSimplePatternPart::advance_y() noexcept { 661 if (is_aligned_blit()) { 662 // Blit AA 663 // ------- 664 665 // That's the beauty of AABlit - no checks needed, no extend modes used. 666 pc->add(f->srcp1, f->srcp1, f->stride); 667 } 668 else { 669 // Vertical Extend Mode Handling 670 // ----------------------------- 671 672 int kStrideArrayOffset = 0; 673 int kYStopArrayOffset = int(pc->register_size()) * 2; 674 int kYRewindOffset = int(pc->register_size()) * 4; 675 int kPixelPtrRewindOffset = int(pc->register_size()) * 5; 676 677 Label L_Done = pc->new_label(); 678 Label L_YStop = pc->new_label(); 679 680 pc->inc(f->y); 681 682 // If this pattern fetch uses two source pointers (one for current scanline 683 // and one for previous one) copy current to the previous so it can be used 684 // (only fetchers that use Fy). 685 if (has_frac_y()) { 686 pc->mov(f->srcp0, f->srcp1); 687 } 688 689 pc->j(L_YStop, cmp_eq(f->y, f->v_extend_data.clone_adjusted(kYStopArrayOffset))); 690 pc->add(f->srcp1, f->srcp1, f->v_extend_data.clone_adjusted(kStrideArrayOffset)); 691 pc->bind(L_Done); 692 693 PipeInjectAtTheEnd injected(pc); 694 pc->bind(L_YStop); 695 696 // Swap stride and y_stop pairs. 697 if (pc->is_64bit()) { 698 #if defined(BL_JIT_ARCH_X86) 699 if (pc->has_avx2()) { 700 Vec v = pc->new_vec256("f.v_tmp"); 701 pc->v_swap_u64(v, f->v_extend_data); 702 pc->v_storeu256(f->v_extend_data, v); 703 } 704 else 705 #endif // BL_JIT_ARCH_X86 706 { 707 Vec v = pc->new_vec128("f.v_tmp"); 708 Mem stride_array = f->v_extend_data.clone_adjusted(kStrideArrayOffset); 709 Mem yStopArray = f->v_extend_data.clone_adjusted(kYStopArrayOffset); 710 pc->v_swap_u64(v, stride_array); 711 pc->v_storea128(stride_array, v); 712 pc->v_swap_u64(v, yStopArray); 713 pc->v_storea128(yStopArray, v); 714 } 715 } 716 else { 717 Vec v0 = pc->new_vec128("f.v_tmp"); 718 pc->v_swap_u32(v0, f->v_extend_data); 719 pc->v_storea128(f->v_extend_data, v0); 720 } 721 722 // Rewind y and pixel-ptr. 723 pc->sub(f->y, f->y, f->v_extend_data.clone_adjusted(kYRewindOffset)); 724 pc->sub(f->srcp1, f->srcp1, f->v_extend_data.clone_adjusted(kPixelPtrRewindOffset)); 725 pc->j(L_Done); 726 } 727 } 728 729 void FetchSimplePatternPart::start_at_x(const Gp& x) noexcept { 730 if (is_aligned_blit()) { 731 // Blit AA 732 // ------- 733 734 // TODO: [JIT] OPTIMIZATION: Relax this constraint. 735 // Rectangular blits only. 736 BL_ASSERT(is_rect_fill()); 737 } 738 else if (extend_x() == ExtendMode::kPad) { 739 // Horizontal Pad 740 // -------------- 741 742 if (!is_rect_fill()) 743 pc->add(f->x, f->x_origin, x); // f->x = f->x_origin + x; 744 else 745 pc->mov(f->x, f->x_origin); // f->x = f->x_origin; 746 pc->sbound(f->x_padded.r32(), f->x, f->w); // f->x_padded = signed_bound(f->x, f->w) 747 } 748 else if (extend_x() == ExtendMode::kRepeat) { 749 // Horizontal Repeat - AA-Only, Large Fills 750 // ---------------------------------------- 751 752 pc->mov(f->x, f->x_origin); // f->x = f->x_origin; 753 if (!is_rect_fill()) { // if (!RectFill) { 754 pc->add_scaled(f->x, x.clone_as(f->x), int(bpp())); // f->x += x * pattern.bpp; 755 repeat_or_reflect_x(); // f->x = repeat_large(f->x); 756 } // } 757 } 758 else if (extend_x() == ExtendMode::kRoR) { 759 // Horizontal RoR [Repeat or Reflect] 760 // ---------------------------------- 761 762 pc->mov(f->x, f->x_origin); // f->x = f->x_origin; 763 if (!is_rect_fill()) { // if (!RectFill) { 764 pc->add(f->x, f->x, x); // f->x += x; 765 repeat_or_reflect_x(); // f->x = repeat_or_reflect(f->x); 766 } // } 767 } 768 else { 769 BL_NOT_REACHED(); 770 } 771 772 prefetch_acc_x(); 773 774 if (pixel_granularity() > 1) 775 enter_n(); 776 } 777 778 void FetchSimplePatternPart::advance_x(const Gp& x, const Gp& diff) noexcept { 779 bl_unused(x); 780 Gp fx32 = f->x.r32(); 781 782 if (pixel_granularity() > 1) { 783 leave_n(); 784 } 785 786 if (is_aligned_blit()) { 787 // Blit AA 788 // ------- 789 790 pc->add_scaled(f->srcp1, diff.clone_as(f->srcp1), int(bpp())); 791 } 792 else if (extend_x() == ExtendMode::kPad) { 793 // Horizontal Pad 794 // -------------- 795 796 pc->add(fx32, fx32, diff); // f->x += diff; 797 pc->sbound(f->x_padded.r32(), f->x, f->w); // f->x_padded = signed_bound(f->x, f->w) 798 } 799 else if (extend_x() == ExtendMode::kRepeat) { 800 // Horizontal Repeat - AA-Only, Large Fills 801 // ---------------------------------------- 802 803 pc->add_scaled(f->x, diff.clone_as(f->x), int(bpp())); // f->x += diff * pattern.bpp; 804 repeat_or_reflect_x(); // f->x = repeat_large(f->x); 805 } 806 else if (extend_x() == ExtendMode::kRoR) { 807 // Horizontal RoR [Repeat or Reflect] 808 // ---------------------------------- 809 810 pc->add(fx32, fx32, diff); // f->x += diff; 811 repeat_or_reflect_x(); // f->x = repeat_or_reflect(f->x); 812 } 813 814 prefetch_acc_x(); 815 816 if (pixel_granularity() > 1) { 817 enter_n(); 818 } 819 } 820 821 void FetchSimplePatternPart::advance_x_by_one() noexcept { 822 if (is_aligned_blit()) { 823 // Blit AA 824 // ------- 825 826 pc->add(f->srcp1, f->srcp1, int(bpp())); 827 } 828 else if (extend_x() == ExtendMode::kPad) { 829 // Horizontal Pad 830 // -------------- 831 832 pc->inc(f->x); 833 pc->cmov(f->x_padded.r32(), f->x, ucmp_le(f->x, f->w)); 834 } 835 else if (extend_x() == ExtendMode::kRepeat) { 836 // Horizontal Repeat - AA-Only, Large Fills 837 // ---------------------------------------- 838 839 pc->cmov(f->x, f->x_restart, add_z(f->x, int(bpp()))); 840 } 841 else if (extend_x() == ExtendMode::kRoR) { 842 // Horizontal RoR [Repeat or Reflect] 843 // ---------------------------------- 844 845 pc->inc(f->x); 846 pc->cmov(f->x, f->x_restart, scmp_gt(f->x, f->w)); 847 } 848 } 849 850 void FetchSimplePatternPart::repeat_or_reflect_x() noexcept { 851 if (is_aligned_blit()) { 852 // Blit AA 853 // ------- 854 855 // Nothing... 856 } 857 else if (extend_x() == ExtendMode::kRepeat) { 858 // Horizontal Repeat - AA-Only, Large Fills 859 // ---------------------------------------- 860 861 Label L_HorzSkip = pc->new_label(); 862 863 pc->j(L_HorzSkip, scmp_lt(f->x, 0)); // if (f->x >= 0 && 864 pc->j(L_HorzSkip, add_s(f->x, f->x_restart)); // f->x -= f->w >= 0) { 865 // `f->x` too large to be corrected by `f->w`, so do it the slow way: 866 pc->umod(f->x.r32(), f->x.r32(), f->w.r32()); // f->x %= f->w; 867 pc->add(f->x, f->x, f->x_restart); // f->x -= f->w; 868 pc->bind(L_HorzSkip); // } 869 } 870 else if (extend_x() == ExtendMode::kRoR) { 871 // Horizontal RoR [Repeat or Reflect] 872 // ---------------------------------- 873 874 Label L_HorzSkip = pc->new_label(); 875 Gp norm = pc->new_gp32("@norm"); 876 877 pc->j(L_HorzSkip, scmp_lt(f->x, f->rx)); // if (f->x >= f->rx) { 878 pc->umod(f->x, f->x, f->rx); // f->x %= f->rx; 879 pc->bind(L_HorzSkip); // } 880 pc->select(norm, Imm(0), f->rx, scmp_le(f->x, f->w)); // norm = (f->x < f->w) ? 0 : f->rx; 881 pc->sub(f->x, f->x, norm); // f->x -= norm; 882 } 883 } 884 885 void FetchSimplePatternPart::prefetch_acc_x() noexcept { 886 if (!has_frac_x()) 887 return; 888 889 Gp idx; 890 891 // Horizontal Pad 892 // -------------- 893 894 if (extend_x() == ExtendMode::kPad) { 895 idx = f->x_padded; 896 } 897 898 // Horizontal Repeat - AA-Only, Large Fills 899 // ---------------------------------------- 900 901 if (extend_x() == ExtendMode::kRepeat) { 902 idx = f->x; 903 } 904 905 // Horizontal RoR [Repeat or Reflect] 906 // ---------------------------------- 907 908 if (extend_x() == ExtendMode::kRoR) { 909 idx = pc->new_gpz("@idx"); 910 pc->reflect(idx.r32(), f->x); 911 } 912 913 if (is_alpha_fetch()) { 914 if (is_pattern_fx()) { 915 pc->v_load8(f->pix_l, mem_ptr(f->srcp1, idx, _idx_shift)); 916 } 917 else { 918 pc->v_load8(f->pix_l, mem_ptr(f->srcp0, idx, _idx_shift)); 919 pc->x_insert_word_or_byte(f->pix_l, mem_ptr(f->srcp1, idx, _idx_shift), 1); 920 } 921 } 922 else { 923 if (is_pattern_fx()) { 924 pc->v_broadcast_u32(f->pix_l, mem_ptr(f->srcp1, idx, _idx_shift)); 925 } 926 else { 927 pc->v_loadu32(f->pix_l, mem_ptr(f->srcp1, idx, _idx_shift)); 928 FetchUtils::fetch_second_32bit_element(pc, f->pix_l, mem_ptr(f->srcp0, idx, _idx_shift)); 929 } 930 } 931 } 932 933 // bl::Pipeline::JIT::FetchSimplePatternPart - Fetch 934 // ================================================= 935 936 void FetchSimplePatternPart::enter_n() noexcept { 937 if (is_aligned_blit()) { 938 // Blit AA 939 // ------- 940 941 // Nothing... 942 } 943 else if (extend_x() == ExtendMode::kPad) { 944 // Horizontal Pad 945 // -------------- 946 947 // Nothing... 948 } 949 else if (extend_x() == ExtendMode::kRoR) { 950 // Horizontal RoR [Repeat or Reflect] 951 // ---------------------------------- 952 953 Vec xFix4 = pc->new_vec128("@xFix4"); 954 pc->v_broadcast_u32(f->x_vec_4, f->x.r32()); 955 pc->v_add_i32(f->x_vec_4, f->x_vec_4, f->x_set_4); 956 957 pc->v_cmp_gt_i32(xFix4, f->x_vec_4, f->x_max_4); 958 pc->v_and_i32(xFix4, xFix4, f->x_nrm_4); 959 pc->v_sub_i32(f->x_vec_4, f->x_vec_4, xFix4); 960 } 961 } 962 963 void FetchSimplePatternPart::leave_n() noexcept { 964 if (is_aligned_blit()) { 965 // Blit AA 966 // ------- 967 968 // Nothing... 969 } 970 else if (extend_x() == ExtendMode::kPad) { 971 // Horizontal Pad 972 // -------------- 973 974 // Nothing... 975 } 976 else if (extend_x() == ExtendMode::kRoR) { 977 // Horizontal RoR [Repeat or Reflect] 978 // ---------------------------------- 979 980 pc->s_mov_u32(f->x.r32(), f->x_vec_4); 981 982 if (has_frac_x()) { 983 pc->dec(f->x); 984 pc->cmov(f->x, f->w, scmp_lt(f->x, f->x_restart)); 985 } 986 } 987 } 988 989 void FetchSimplePatternPart::prefetch_n() noexcept {} 990 void FetchSimplePatternPart::postfetch_n() noexcept {} 991 992 void FetchSimplePatternPart::fetch(Pixel& p, PixelCount n, PixelFlags flags, PixelPredicate& predicate) noexcept { 993 p.set_count(n); 994 995 if (!bl_test_flag(flags, PixelFlags::kPA_PI_UA_UI | PixelFlags::kPC_UC)) { 996 if (p.isRGBA32()) 997 flags |= PixelFlags::kPC; 998 else 999 flags |= PixelFlags::kPA; 1000 } 1001 1002 if (is_aligned_blit()) { 1003 FetchUtils::fetch_pixels(pc, p, n, flags, fetch_info(), f->srcp1, Alignment(1), AdvanceMode::kAdvance, predicate); 1004 return; 1005 } 1006 1007 if (!predicate.is_empty()) { 1008 flags |= PixelFlags::kLastPartial; 1009 } 1010 1011 GatherMode gather_mode = predicate.gather_mode(); 1012 1013 switch (uint32_t(n)) { 1014 case 1: { 1015 BL_ASSERT(predicate.is_empty()); 1016 1017 Gp idx; 1018 1019 // Pattern AA or Fx/Fy 1020 // ------------------- 1021 1022 if (has_frac_x()) { 1023 advance_x_by_one(); 1024 } 1025 1026 if (extend_x() == ExtendMode::kPad) { 1027 idx = f->x_padded; 1028 } 1029 else if (extend_x() == ExtendMode::kRepeat) { 1030 idx = f->x; 1031 } 1032 else if (extend_x() == ExtendMode::kRoR) { 1033 idx = pc->new_gpz("@idx"); 1034 pc->reflect(idx.r32(), f->x); 1035 } 1036 1037 if (is_pattern_aligned()) { 1038 FetchUtils::fetch_pixel(pc, p, flags, fetch_info(), mem_ptr(f->srcp1, idx, _idx_shift)); 1039 advance_x_by_one(); 1040 } 1041 else if (is_pattern_fy()) { 1042 if (is_alpha_fetch()) { 1043 Vec pixA = pc->new_vec128("@pixA"); 1044 1045 FetchUtils::x_fetch_unpacked_a8_2x(pc, pixA, fetch_info(), mem_ptr(f->srcp1, idx, _idx_shift), mem_ptr(f->srcp0, idx, _idx_shift)); 1046 pc->v_mhadd_i16_to_i32(pixA, pixA, f->wd_wb); 1047 pc->v_srli_u16(pixA, pixA, 8); 1048 1049 advance_x_by_one(); 1050 1051 FetchUtils::x_assign_unpacked_alpha_values(pc, p, flags, pixA); 1052 FetchUtils::satisfy_pixels(pc, p, flags); 1053 } 1054 else if (p.isRGBA32()) { 1055 Vec pix0 = pc->new_vec128("@pix0"); 1056 Vec pix1 = pc->new_vec128("@pix1"); 1057 1058 pc->v_loadu32(pix0, mem_ptr(f->srcp0, idx, _idx_shift)); 1059 pc->v_loadu32(pix1, mem_ptr(f->srcp1, idx, _idx_shift)); 1060 1061 pc->v_cvt_u8_lo_to_u16(pix0, pix0); 1062 pc->v_cvt_u8_lo_to_u16(pix1, pix1); 1063 1064 pc->v_mul_u16(pix0, pix0, f->wb); 1065 pc->v_mul_u16(pix1, pix1, f->wd); 1066 1067 advance_x_by_one(); 1068 1069 pc->v_add_u16(pix0, pix0, pix1); 1070 pc->v_srli_u16(pix0, pix0, 8); 1071 1072 p.uc.init(pix0); 1073 FetchUtils::satisfy_pixels(pc, p, flags); 1074 } 1075 } 1076 else if (is_pattern_fx()) { 1077 if (is_alpha_fetch()) { 1078 Vec pix_l = f->pix_l; 1079 Vec pixA = pc->new_vec128("@pixA"); 1080 1081 pc->x_insert_word_or_byte(pix_l, mem_ptr(f->srcp1, idx, _idx_shift), 1); 1082 pc->v_mhadd_i16_to_i32(pixA, pix_l, f->wc_wd); 1083 pc->v_srli_u32(pix_l, pix_l, 16); 1084 pc->v_srli_u16(pixA, pixA, 8); 1085 1086 FetchUtils::x_assign_unpacked_alpha_values(pc, p, flags, pixA); 1087 FetchUtils::satisfy_pixels(pc, p, flags); 1088 } 1089 else if (p.isRGBA32()) { 1090 Vec pix_l = f->pix_l; 1091 Vec pix0 = pc->new_vec128("@pix0"); 1092 Vec pix1 = pc->new_vec128("@pix1"); 1093 1094 pc->v_insert_u32(pix_l, mem_ptr(f->srcp1, idx, _idx_shift), 1); 1095 pc->v_cvt_u8_lo_to_u16(pix0, pix_l); 1096 pc->v_mul_u16(pix0, pix0, f->wc_wd); 1097 pc->v_swizzle_u32x4(pix_l, pix_l, swizzle(1, 1, 1, 1)); 1098 pc->v_swap_u64(pix1, pix0); 1099 1100 pc->v_add_u16(pix0, pix0, pix1); 1101 pc->v_srli_u16(pix0, pix0, 8); 1102 1103 p.uc.init(pix0); 1104 FetchUtils::satisfy_pixels(pc, p, flags); 1105 } 1106 } 1107 else if (is_pattern_fx_fy()) { 1108 if (is_alpha_fetch()) { 1109 Vec pix_l = f->pix_l; 1110 Vec pixA = pc->new_vec128("@pixA"); 1111 Vec pixB = pc->new_vec128("@pixB"); 1112 1113 pc->v_load_u8_u16_2x(pixB, mem_ptr(f->srcp0, idx, _idx_shift), mem_ptr(f->srcp1, idx, _idx_shift)); 1114 pc->v_mhadd_i16_to_i32(pixA, pix_l, f->wa_wc); 1115 pc->v_mov(pix_l, pixB); 1116 pc->v_mhadd_i16_to_i32(pixB, pixB, f->wb_wd); 1117 pc->v_add_i32(pixA, pixA, pixB); 1118 pc->v_srli_u16(pixA, pixA, 8); 1119 1120 FetchUtils::x_assign_unpacked_alpha_values(pc, p, flags, pixA); 1121 FetchUtils::satisfy_pixels(pc, p, flags); 1122 } 1123 else if (p.isRGBA32()) { 1124 Vec pix_l = f->pix_l; 1125 Vec pix0 = pc->new_vec128("@pix0"); 1126 Vec pix1 = pc->new_vec128("@pix1"); 1127 1128 pc->v_cvt_u8_lo_to_u16(pix0, pix_l); 1129 pc->v_loadu32(pix_l, mem_ptr(f->srcp1, idx, _idx_shift)); 1130 FetchUtils::fetch_second_32bit_element(pc, pix_l, mem_ptr(f->srcp0, idx, _idx_shift)); 1131 pc->v_cvt_u8_lo_to_u16(pix1, pix_l); 1132 1133 pc->v_mul_u16(pix0, pix0, f->wa_wc); 1134 pc->v_mul_u16(pix1, pix1, f->wb_wd); 1135 pc->v_add_u16(pix0, pix0, pix1); 1136 pc->v_swap_u64(pix1, pix0); 1137 pc->v_add_u16(pix0, pix0, pix1); 1138 pc->v_srli_u16(pix0, pix0, 8); 1139 1140 p.uc.init(pix0); 1141 FetchUtils::satisfy_pixels(pc, p, flags); 1142 } 1143 } 1144 break; 1145 } 1146 1147 case 4: { 1148 PixelType intermediate_type = is_alpha_fetch() ? PixelType::kA8 : PixelType::kRGBA32; 1149 PixelFlags intermediate_flags = is_alpha_fetch() ? PixelFlags::kUA : PixelFlags::kUC; 1150 1151 // Horizontal Pad | RoR 1152 // -------------------- 1153 1154 if (extend_x() == ExtendMode::kPad || extend_x() == ExtendMode::kRoR) { 1155 FetchPadRoRContext pCtx(this, predicate); 1156 pCtx.begin(); 1157 1158 // Horizontal Pad | RoR - Aligned 1159 // ------------------------------ 1160 1161 if (is_pattern_aligned()) { 1162 FetchUtils::FetchContext fCtx(pc, &p, PixelCount(4), flags, fetch_info(), gather_mode); 1163 1164 fCtx.fetch_pixel(mem_ptr(f->srcp1, pCtx.next_index(), _idx_shift)); 1165 fCtx.fetch_pixel(mem_ptr(f->srcp1, pCtx.next_index(), _idx_shift)); 1166 fCtx.fetch_pixel(mem_ptr(f->srcp1, pCtx.next_index(), _idx_shift)); 1167 1168 if (predicate.is_empty()) { 1169 fCtx.fetch_pixel(mem_ptr(f->srcp1, pCtx.next_index(), _idx_shift)); 1170 } 1171 1172 pCtx.end(); 1173 fCtx.end(); 1174 1175 FetchUtils::satisfy_pixels(pc, p, flags); 1176 } 1177 1178 // Horizontal Pad | RoR - Fy 1179 // ------------------------- 1180 1181 if (is_pattern_fy()) { 1182 Gp idx; 1183 1184 if (is_alpha_fetch()) { 1185 Pixel fPix("fPix", intermediate_type); 1186 FetchUtils::FetchContext fCtx(pc, &fPix, PixelCount(8), intermediate_flags, fetch_info(), GatherMode::kFetchAll); 1187 1188 idx = pCtx.next_index(); 1189 fCtx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1190 fCtx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1191 1192 idx = pCtx.next_index(); 1193 fCtx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1194 fCtx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1195 1196 idx = pCtx.next_index(); 1197 fCtx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1198 fCtx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1199 1200 idx = pCtx.next_index(); 1201 fCtx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1202 fCtx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1203 1204 fCtx.end(); 1205 pCtx.end(); 1206 1207 Vec& pix0 = fPix.ua[0]; 1208 1209 pc->v_mhadd_i16_to_i32(pix0, pix0, f->wd_wb); 1210 pc->v_srli_u16(pix0, pix0, 8); 1211 1212 pc->v_packs_i32_i16(pix0, pix0, pix0); 1213 FetchUtils::x_assign_unpacked_alpha_values(pc, p, flags, pix0); 1214 FetchUtils::satisfy_pixels(pc, p, flags); 1215 } 1216 else if (p.isRGBA32()) { 1217 Pixel pix0("pix0", intermediate_type); 1218 Pixel pix1("pix1", intermediate_type); 1219 1220 FetchUtils::FetchContext a_ctx(pc, &pix0, PixelCount(4), intermediate_flags, fetch_info(), gather_mode); 1221 FetchUtils::FetchContext b_ctx(pc, &pix1, PixelCount(4), intermediate_flags, fetch_info(), gather_mode); 1222 1223 idx = pCtx.next_index(); 1224 a_ctx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1225 b_ctx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1226 1227 idx = pCtx.next_index(); 1228 a_ctx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1229 b_ctx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1230 1231 idx = pCtx.next_index(); 1232 a_ctx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1233 b_ctx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1234 1235 if (predicate.is_empty()) { 1236 idx = pCtx.next_index(); 1237 a_ctx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1238 b_ctx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1239 } 1240 1241 a_ctx.end(); 1242 b_ctx.end(); 1243 pCtx.end(); 1244 1245 pc->v_mul_u16(pix0.uc, pix0.uc, f->wb); 1246 pc->v_mul_u16(pix1.uc, pix1.uc, f->wd); 1247 1248 pc->v_add_u16(pix0.uc, pix0.uc, pix1.uc); 1249 pc->v_srli_u16(pix0.uc, pix0.uc, 8); 1250 1251 p.uc.init(pix0.uc[0], pix0.uc[1]); 1252 FetchUtils::satisfy_pixels(pc, p, flags); 1253 } 1254 } 1255 1256 // Horizontal Pad | RoR - Fx 1257 // ------------------------- 1258 1259 if (is_pattern_fx()) { 1260 if (is_alpha_fetch()) { 1261 Pixel fPix("fPix", intermediate_type); 1262 FetchUtils::FetchContext fCtx(pc, &fPix, PixelCount(4), intermediate_flags, fetch_info(), GatherMode::kFetchAll); 1263 1264 Vec& pixA = fPix.ua[0]; 1265 Vec& pix_l = f->pix_l; 1266 1267 fCtx.fetch_pixel(mem_ptr(f->srcp1, pCtx.next_index(), _idx_shift)); 1268 fCtx.fetch_pixel(mem_ptr(f->srcp1, pCtx.next_index(), _idx_shift)); 1269 fCtx.fetch_pixel(mem_ptr(f->srcp1, pCtx.next_index(), _idx_shift)); 1270 fCtx.fetch_pixel(mem_ptr(f->srcp1, pCtx.next_index(), _idx_shift)); 1271 1272 fCtx.end(); 1273 pCtx.end(); 1274 1275 pc->v_interleave_lo_u16(pixA, pixA, pixA); 1276 pc->v_sllb_u128(pixA, pixA, 2); 1277 1278 pc->v_or_i32(pix_l, pix_l, pixA); 1279 pc->v_mhadd_i16_to_i32(pixA, pix_l, f->wc_wd); 1280 1281 pc->v_srlb_u128(pix_l, pix_l, 14); 1282 pc->v_srli_u32(pixA, pixA, 8); 1283 pc->v_packs_i32_i16(pixA, pixA, pixA); 1284 1285 FetchUtils::x_assign_unpacked_alpha_values(pc, p, flags, pixA); 1286 FetchUtils::satisfy_pixels(pc, p, flags); 1287 } 1288 else if (p.isRGBA32()) { 1289 Pixel fPix("fPix", intermediate_type); 1290 FetchUtils::FetchContext fCtx(pc, &fPix, PixelCount(4), PixelFlags::kPC, fetch_info(), GatherMode::kFetchAll); 1291 1292 fCtx.fetch_pixel(mem_ptr(f->srcp1, pCtx.next_index(), _idx_shift)); 1293 fCtx.fetch_pixel(mem_ptr(f->srcp1, pCtx.next_index(), _idx_shift)); 1294 fCtx.fetch_pixel(mem_ptr(f->srcp1, pCtx.next_index(), _idx_shift)); 1295 fCtx.fetch_pixel(mem_ptr(f->srcp1, pCtx.next_index(), _idx_shift)); 1296 1297 fCtx.end(); 1298 pCtx.end(); 1299 1300 Vec pix_l = f->pix_l; 1301 Vec pix0 = pc->new_vec128("@pix0"); 1302 Vec pix1 = pc->new_vec128("@pix1"); 1303 Vec pix2 = fPix.pc[0]; 1304 Vec pix3 = pc->new_vec128("@pix3"); 1305 1306 pc->v_alignr_u128(pix0, pix2, pix_l, 12); 1307 pc->v_swizzle_u32x4(pix_l, pix2, swizzle(3, 3, 3, 3)); 1308 1309 pc->v_cvt_u8_hi_to_u16(pix1, pix0); 1310 pc->v_mul_u16(pix1, pix1, f->wc); 1311 1312 pc->v_cvt_u8_lo_to_u16(pix0, pix0); 1313 pc->v_mul_u16(pix0, pix0, f->wc); 1314 1315 pc->v_cvt_u8_hi_to_u16(pix3, pix2); 1316 pc->v_madd_u16(pix1, pix3, f->wd, pix1); 1317 1318 pc->v_cvt_u8_lo_to_u16(pix2, pix2); 1319 pc->v_madd_u16(pix0, pix2, f->wd, pix0); 1320 1321 pc->v_srli_u16(pix1, pix1, 8); 1322 pc->v_srli_u16(pix0, pix0, 8); 1323 1324 p.uc.init(pix0, pix1); 1325 FetchUtils::satisfy_pixels(pc, p, flags); 1326 } 1327 } 1328 1329 // Horizontal Pad | RoR - FxFy 1330 // --------------------------- 1331 1332 if (is_pattern_fx_fy()) { 1333 Gp idx; 1334 1335 if (is_alpha_fetch()) { 1336 Pixel fPix("fPix", intermediate_type); 1337 FetchUtils::FetchContext fCtx(pc, &fPix, PixelCount(8), intermediate_flags, fetch_info(), GatherMode::kFetchAll); 1338 1339 idx = pCtx.next_index(); 1340 fCtx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1341 fCtx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1342 1343 idx = pCtx.next_index(); 1344 fCtx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1345 fCtx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1346 1347 idx = pCtx.next_index(); 1348 fCtx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1349 fCtx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1350 1351 idx = pCtx.next_index(); 1352 fCtx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1353 fCtx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1354 1355 fCtx.end(); 1356 pCtx.end(); 1357 1358 Vec pix_l = f->pix_l; 1359 Vec pixA = fPix.ua[0]; 1360 Vec pixB = pc->new_vec128("pixB"); 1361 1362 pc->v_sllb_u128(pixB, pixA, 4); 1363 pc->v_or_i32(pixB, pixB, pix_l); 1364 pc->v_srlb_u128(pix_l, pixA, 12); 1365 1366 pc->v_mhadd_i16_to_i32(pixA, pixA, f->wb_wd); 1367 pc->v_mhadd_i16_to_i32(pixB, pixB, f->wa_wc); 1368 1369 pc->v_add_i32(pixA, pixA, pixB); 1370 pc->v_srli_u32(pixA, pixA, 8); 1371 pc->v_packs_i32_i16(pixA, pixA, pixA); 1372 1373 FetchUtils::x_assign_unpacked_alpha_values(pc, p, flags, pixA); 1374 FetchUtils::satisfy_pixels(pc, p, flags); 1375 } 1376 else if (p.isRGBA32()) { 1377 Pixel a_pix("a_pix", intermediate_type); 1378 Pixel b_pix("b_pix", intermediate_type); 1379 1380 FetchUtils::FetchContext a_ctx(pc, &a_pix, PixelCount(4), PixelFlags::kPC, fetch_info(), GatherMode::kFetchAll); 1381 FetchUtils::FetchContext b_ctx(pc, &b_pix, PixelCount(4), PixelFlags::kPC, fetch_info(), GatherMode::kFetchAll); 1382 1383 idx = pCtx.next_index(); 1384 a_ctx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1385 b_ctx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1386 1387 idx = pCtx.next_index(); 1388 a_ctx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1389 b_ctx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1390 1391 idx = pCtx.next_index(); 1392 a_ctx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1393 b_ctx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1394 1395 idx = pCtx.next_index(); 1396 a_ctx.fetch_pixel(mem_ptr(f->srcp0, idx, _idx_shift)); 1397 b_ctx.fetch_pixel(mem_ptr(f->srcp1, idx, _idx_shift)); 1398 1399 a_ctx.end(); 1400 b_ctx.end(); 1401 pCtx.end(); 1402 1403 Vec pix_l = f->pix_l; 1404 Vec pix0 = pc->new_vec128("@pix0"); 1405 Vec pix1 = pc->new_vec128("@pix1"); 1406 Vec pix2 = pc->new_vec128("@pix2"); 1407 Vec pix3 = pc->new_vec128("@pix3"); 1408 1409 Vec pixP = a_pix.pc[0]; 1410 Vec pixQ = b_pix.pc[0]; 1411 1412 pc->v_cvt_u8_lo_to_u16(pix0, pixP); 1413 pc->v_mul_u16(pix0, pix0, f->wb); 1414 pc->v_cvt_u8_hi_to_u16(pix1, pixP); 1415 pc->v_mul_u16(pix1, pix1, f->wb); 1416 1417 pc->v_cvt_u8_lo_to_u16(pix2, pixQ); 1418 pc->v_mul_u16(pix2, pix2, f->wd); 1419 pc->v_cvt_u8_hi_to_u16(pix3, pixQ); 1420 pc->v_mul_u16(pix3, pix3, f->wd); 1421 1422 pc->v_add_u16(pix0, pix0, pix2); 1423 pc->v_swizzle_u32x4(pix2, f->pix_l, swizzle(1, 0, 1, 0)); 1424 pc->v_add_u16(pix1, pix1, pix3); 1425 1426 pc->v_interleave_shuffle_u32x4(pix_l, pixQ, pixP, swizzle(3, 3, 3, 3)); 1427 pc->v_alignr_u128(pixP, pixP, pix2, 12); 1428 pc->v_swizzle_u32x4(pix2, pix2, swizzle(2, 2, 2, 2)); 1429 1430 pc->shift_or_rotate_right(pix_l, pix_l, 4); 1431 pc->v_alignr_u128(pixQ, pixQ, pix2, 12); 1432 1433 pc->v_cvt_u8_lo_to_u16(pix2, pixP); 1434 pc->v_mul_u16(pix2, pix2, f->wa); 1435 pc->v_cvt_u8_lo_to_u16(pix3, pixQ); 1436 pc->v_mul_u16(pix3, pix3, f->wc); 1437 1438 pc->v_cvt_u8_hi_to_u16(pixP, pixP); 1439 pc->v_mul_u16(pixP, pixP, f->wa); 1440 pc->v_add_u16(pix2, pix2, pix3); 1441 1442 pc->v_cvt_u8_hi_to_u16(pixQ, pixQ); 1443 pc->v_mul_u16(pixQ, pixQ, f->wc); 1444 pc->v_add_u16(pixP, pixP, pixQ); 1445 1446 pc->v_add_u16(pix0, pix0, pix2); 1447 pc->v_add_u16(pix1, pix1, pixP); 1448 1449 pc->v_srli_u16(pix0, pix0, 8); 1450 pc->v_srli_u16(pix1, pix1, 8); 1451 1452 p.uc.init(pix0, pix1); 1453 FetchUtils::satisfy_pixels(pc, p, flags); 1454 } 1455 } 1456 } 1457 1458 // Horizontal Repeat - AA-Only (Large Fills) 1459 // ----------------------------------------- 1460 1461 if (extend_x() == ExtendMode::kRepeat) { 1462 // Only generated for AA patterns. 1463 BL_ASSERT(is_pattern_aligned()); 1464 1465 PixelFlags overridden_flags = flags; 1466 if (pc->use_256bit_simd() && p.isRGBA32()) { 1467 overridden_flags = PixelFlags::kPC; 1468 } 1469 1470 FetchUtils::FetchContext fCtx(pc, &p, PixelCount(4), overridden_flags, fetch_info(), gather_mode); 1471 Gp x = f->x; 1472 1473 if (predicate.is_empty()) { 1474 Label L_Done = pc->new_label(); 1475 Label L_Repeat = pc->new_label(); 1476 1477 int offset = int(4 * bpp()); 1478 1479 #if defined(BL_JIT_ARCH_X86) 1480 // This forms a pointer that takes advantage of X86 addressing [base + index + offset]. 1481 // What we want to do in the fast case is to just read [base + x - offset], because we have 1482 // just incremented the offset, so we want to read the pointer `srcp1 + x` pointer before x 1483 // was incremented. 1484 Mem mem = mem_ptr(f->srcp1, x, 0, -offset); 1485 #else 1486 // AArch64 addressing is more restricted than x86 one, so we can form either a [base + index] 1487 // or [base + offset] address. 1488 Gp src_base = pc->new_similar_reg(f->srcp1, "src_base"); 1489 pc->sub(src_base, f->srcp1, offset); 1490 Mem mem = mem_ptr(src_base, x); 1491 #endif 1492 pc->j(L_Repeat, add_c(x, offset)); 1493 1494 // TODO: [JIT] This should use FetchUtils::fetch_pixels() instead - it's identical. 1495 // 1496 // The problem here is only that we want the same registers where the pixels are fetched, where 1497 // pixels are allocated by FetchUtils::FetchContext. However, if we tweak fetch_pixels() and add 1498 // a parameter to reuse the existing vector registers (or simply fetch to existing ones, if 1499 // provided) then this code could be removed. 1500 if (p.isRGBA32()) { 1501 if (bl_test_flag(overridden_flags, PixelFlags::kPC)) { 1502 const Vec& reg = p.pc[0]; 1503 1504 switch (format()) { 1505 case FormatExt::kPRGB32: 1506 case FormatExt::kXRGB32: { 1507 pc->v_loadu128(reg, mem); 1508 break; 1509 } 1510 1511 case FormatExt::kA8: { 1512 pc->v_loadu32(reg, mem); 1513 pc->v_interleave_lo_u8(reg, reg, reg); 1514 pc->v_interleave_lo_u16(reg, reg, reg); 1515 break; 1516 } 1517 1518 default: 1519 BL_NOT_REACHED(); 1520 } 1521 } 1522 else { 1523 const Vec& uc0 = p.uc[0]; 1524 const Vec& uc1 = p.uc[1]; 1525 1526 switch (format()) { 1527 case FormatExt::kPRGB32: 1528 case FormatExt::kXRGB32: { 1529 pc->v_cvt_u8_lo_to_u16(uc0, mem); 1530 pc->v_cvt_u8_lo_to_u16(uc1, mem.clone_adjusted(8)); 1531 break; 1532 } 1533 1534 case FormatExt::kA8: { 1535 pc->v_loadu32(uc0, mem); 1536 pc->v_interleave_lo_u8(uc0, uc0, uc0); 1537 pc->v_cvt_u8_lo_to_u16(uc0, uc0); 1538 pc->v_swizzle_u32x4(uc1, uc0, swizzle(3, 3, 2, 2)); 1539 pc->v_swizzle_u32x4(uc0, uc0, swizzle(1, 1, 0, 0)); 1540 break; 1541 } 1542 1543 default: 1544 BL_NOT_REACHED(); 1545 } 1546 } 1547 } 1548 else { 1549 if (bl_test_flag(overridden_flags, PixelFlags::kPA)) { 1550 const Vec& reg = p.pa[0]; 1551 switch (format()) { 1552 case FormatExt::kPRGB32: 1553 case FormatExt::kXRGB32: { 1554 pc->v_loadu128(reg, mem); 1555 1556 #if defined(BL_JIT_ARCH_X86) 1557 if (!pc->has_ssse3()) { 1558 pc->v_srli_u32(reg, reg, 24); 1559 pc->v_packs_i32_i16(reg, reg, reg); 1560 pc->v_packs_i16_u8(reg, reg, reg); 1561 } 1562 else 1563 #endif // BL_JIT_ARCH_X86 1564 { 1565 pc->v_swizzlev_u8(reg, reg, pc->simd_const(&ct.swizu8_3xxx2xxx1xxx0xxx_to_zzzzzzzzzzzz3210, Bcst::kNA, reg)); 1566 } 1567 break; 1568 } 1569 1570 case FormatExt::kA8: { 1571 pc->v_loadu32(reg, mem); 1572 break; 1573 } 1574 1575 default: 1576 BL_NOT_REACHED(); 1577 } 1578 } 1579 else { 1580 const Vec& reg = p.ua[0]; 1581 switch (format()) { 1582 case FormatExt::kPRGB32: 1583 case FormatExt::kXRGB32: { 1584 pc->v_loadu128(reg, mem); 1585 pc->v_srli_u32(reg, reg, 24); 1586 pc->v_packs_i32_i16(reg, reg, reg); 1587 break; 1588 } 1589 1590 case FormatExt::kA8: { 1591 pc->v_loadu32(reg, mem); 1592 pc->v_cvt_u8_lo_to_u16(reg, reg); 1593 break; 1594 } 1595 1596 default: 1597 BL_NOT_REACHED(); 1598 } 1599 } 1600 } 1601 1602 pc->bind(L_Done); 1603 1604 { 1605 PipeInjectAtTheEnd injected(pc); 1606 pc->bind(L_Repeat); 1607 1608 fCtx.fetch_pixel(mem); 1609 1610 #if defined(BL_JIT_ARCH_X86) 1611 mem.add_offset_lo32(offset); 1612 #else 1613 mem = mem_ptr(f->srcp1, x); 1614 #endif 1615 1616 pc->cmov(x, f->x_restart, sub_z(x, offset - int(bpp()))); 1617 fCtx.fetch_pixel(mem); 1618 1619 pc->cmov(x, f->x_restart, add_z(x, bpp())); 1620 fCtx.fetch_pixel(mem); 1621 1622 pc->cmov(x, f->x_restart, add_z(x, bpp())); 1623 fCtx.fetch_pixel(mem); 1624 1625 pc->cmov(x, f->x_restart, add_z(x, bpp())); 1626 fCtx.end(); 1627 1628 pc->j(L_Done); 1629 } 1630 } 1631 else { 1632 uint32_t kMsk = ((bpp() ) << 16) | // `predicate.count == 2` => always fetch 1, then 1 next. 1633 ((bpp() * 0x11u) << 24) ; // `predicate.count == 3` => always fetch 1, then 2 next. 1634 1635 Gp t0 = pc->new_gpz("@t0"); 1636 Gp t1 = pc->new_gpz("@t1"); 1637 1638 pc->mov(t0.r32(), kMsk); 1639 pc->shl(t1.r32(), predicate.count().r32(), 3); 1640 pc->shr(t0.r32(), t0.r32(), t1.r32()); 1641 1642 Mem mem = mem_ptr(f->srcp1, x); 1643 fCtx.fetch_pixel(mem); 1644 pc->mov(t1.r32(), 0x0F); 1645 pc->cmov(x, f->x_restart, add_z(x, bpp())); 1646 pc->and_(t1.r32(), t1.r32(), t0.r32()); 1647 1648 fCtx.fetch_pixel(mem); 1649 pc->shr(t0.r32(), t0.r32(), 4); 1650 pc->cmov(x, f->x_restart, add_z(x, t1)); 1651 pc->and_(t1.r32(), t1.r32(), t0.r32()); 1652 1653 fCtx.fetch_pixel(mem); 1654 pc->cmov(x, f->x_restart, add_z(x, t1)); 1655 1656 fCtx.end(); 1657 } 1658 1659 FetchUtils::satisfy_pixels(pc, p, flags); 1660 } 1661 break; 1662 } 1663 1664 default: 1665 BL_NOT_REACHED(); 1666 } 1667 } 1668 1669 // bl::Pipeline::JIT::FetchAffinePatternPart - Construction & Destruction 1670 // ====================================================================== 1671 1672 FetchAffinePatternPart::FetchAffinePatternPart(PipeCompiler* pc, FetchType fetch_type, FormatExt format) noexcept 1673 : FetchPatternPart(pc, fetch_type, format) { 1674 1675 _part_flags |= PipePartFlags::kAdvanceXNeedsDiff; 1676 _max_pixels = 4; 1677 1678 switch (fetch_type) { 1679 case FetchType::kPatternAffineNNAny: 1680 case FetchType::kPatternAffineNNOpt: 1681 add_part_flags(PipePartFlags::kExpensive); 1682 break; 1683 1684 case FetchType::kPatternAffineBIAny: 1685 case FetchType::kPatternAffineBIOpt: 1686 // TODO: [JIT] OPTIMIZATION: Implement fetch4. 1687 _max_pixels = 1; 1688 add_part_flags(PipePartFlags::kExpensive); 1689 break; 1690 1691 default: 1692 BL_NOT_REACHED(); 1693 } 1694 1695 OpUtils::reset_var_struct(&f, sizeof(f)); 1696 1697 if (IntOps::is_power_of_2(_bpp)) 1698 _idx_shift = uint8_t(IntOps::ctz(_bpp)); 1699 } 1700 1701 // bl::Pipeline::JIT::FetchAffinePatternPart - Init & Fini 1702 // ======================================================= 1703 1704 void FetchAffinePatternPart::_init_part(const PipeFunction& fn, Gp& x, Gp& y) noexcept { 1705 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 1706 f->srctop = pc->new_gpz("f.srctop"); // Mem. 1707 f->stride = pc->new_gpz("f.stride"); // Mem. 1708 1709 f->xx_xy = pc->new_vec128("f.xx_xy"); // Reg. 1710 f->yx_yy = pc->new_vec128("f.yx_yy"); // Reg/Mem. 1711 f->tx_ty = pc->new_vec128("f.tx_ty"); // Reg/Mem. 1712 f->px_py = pc->new_vec128("f.px_py"); // Reg. 1713 f->ox_oy = pc->new_vec128("f.ox_oy"); // Reg/Mem. 1714 f->rx_ry = pc->new_vec128("f.rx_ry"); // Reg/Mem. 1715 f->qx_qy = pc->new_vec128("f.qx_qy"); // Reg [fetch4]. 1716 f->xx2_xy2 = pc->new_vec128("f.xx2_xy2"); // Reg/Mem [fetch4]. 1717 f->minx_miny = pc->new_vec128("f.minx_miny"); // Reg/Mem. 1718 f->maxx_maxy = pc->new_vec128("f.maxx_maxy"); // Reg/Mem. 1719 f->corx_cory = pc->new_vec128("f.corx_cory"); // Reg/Mem. 1720 f->tw_th = pc->new_vec128("f.tw_th"); // Reg/Mem. 1721 1722 f->v_idx = pc->new_vec128("f.v_idx"); // Reg/Tmp. 1723 f->vAddrMul = pc->new_vec128("f.vAddrMul"); // Reg/Tmp. 1724 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 1725 1726 pc->load(f->srctop, mem_ptr(fn.fetch_data(), REL_PATTERN(src.pixel_data))); 1727 pc->load(f->stride, mem_ptr(fn.fetch_data(), REL_PATTERN(src.stride))); 1728 1729 #if defined(BL_JIT_ARCH_A64) 1730 // Apply alpha offset to source pointers when on AArch64 as we cannot use offsets together with indexes. 1731 if (_alpha_fetch) { 1732 _fetch_info.apply_alpha_offset(); 1733 if (_fetch_info.applied_offset()) { 1734 pc->add(f->srctop, f->srctop, _fetch_info.applied_offset()); 1735 } 1736 } 1737 #endif // BL_JIT_ARCH_A64 1738 1739 pc->v_loadu128(f->xx_xy, mem_ptr(fn.fetch_data(), REL_PATTERN(affine.xx))); 1740 pc->v_loadu128(f->yx_yy, mem_ptr(fn.fetch_data(), REL_PATTERN(affine.yx))); 1741 1742 pc->s_mov_u32(f->tx_ty, y); 1743 pc->v_swizzle_u32x4(f->tx_ty, f->tx_ty, swizzle(1, 0, 1, 0)); 1744 pc->v_mul_u64_lo_u32(f->tx_ty, f->yx_yy, f->tx_ty); 1745 pc->v_add_i64(f->tx_ty, f->tx_ty, mem_ptr(fn.fetch_data(), REL_PATTERN(affine.tx))); 1746 1747 // RoR: `tw_th` and `rx_ry` are only used by repeated or reflected patterns. 1748 pc->v_loadu128(f->rx_ry, mem_ptr(fn.fetch_data(), REL_PATTERN(affine.rx))); 1749 pc->v_loadu128(f->tw_th, mem_ptr(fn.fetch_data(), REL_PATTERN(affine.tw))); 1750 1751 pc->v_loadu128(f->ox_oy, mem_ptr(fn.fetch_data(), REL_PATTERN(affine.ox))); 1752 pc->v_loadu128(f->xx2_xy2, mem_ptr(fn.fetch_data(), REL_PATTERN(affine.xx2))); 1753 1754 // Pad: [MaxY | MaxX | MinY | MinX] 1755 pc->v_loadu128(f->minx_miny, mem_ptr(fn.fetch_data(), REL_PATTERN(affine.min_x))); 1756 pc->v_loadu64(f->corx_cory, mem_ptr(fn.fetch_data(), REL_PATTERN(affine.cor_x))); 1757 1758 if (is_optimized()) { 1759 pc->v_packs_i32_i16(f->minx_miny, f->minx_miny, f->minx_miny); // [MaxY|MaxX|MinY|MinX|MaxY|MaxX|MinY|MinX] 1760 pc->v_swizzle_u32x4(f->maxx_maxy, f->minx_miny, swizzle(1, 1, 1, 1)); // [MaxY|MaxX|MaxY|MaxX|MaxY|MaxX|MaxY|MaxX] 1761 pc->v_swizzle_u32x4(f->minx_miny, f->minx_miny, swizzle(0, 0, 0, 0)); // [MinY|MinX|MinY|MinX|MinY|MinX|MinY|MinX] 1762 } 1763 else if (fetch_type() == FetchType::kPatternAffineNNAny) { 1764 // NOTE: This is a slightly different layout than others to match [V]PMADDWD instruction on X86. 1765 pc->v_swizzle_u32x4(f->maxx_maxy, f->minx_miny, swizzle(3, 2, 3, 2)); // [MaxY|MaxX|MaxY|MaxX] 1766 pc->v_swizzle_u32x4(f->minx_miny, f->minx_miny, swizzle(1, 0, 1, 0)); // [MinY|MinX|MinY|MinX] 1767 pc->v_swizzle_u32x4(f->corx_cory, f->corx_cory, swizzle(1, 0, 1, 0)); // [CorY|CorX|CorY|CorX] 1768 } 1769 else { 1770 pc->v_swizzle_u32x4(f->maxx_maxy, f->minx_miny, swizzle(3, 3, 2, 2)); // [MaxY|MaxY|MaxX|MaxX] 1771 pc->v_swizzle_u32x4(f->minx_miny, f->minx_miny, swizzle(1, 1, 0, 0)); // [MinY|MinY|MinX|MinX] 1772 pc->v_swizzle_u32x4(f->corx_cory, f->corx_cory, swizzle(1, 1, 0, 0)); // [CorY|CorY|CorX|CorX] 1773 } 1774 1775 if (is_optimized()) 1776 pc->v_broadcast_u32(f->vAddrMul, mem_ptr(fn.fetch_data(), REL_PATTERN(affine.addr_mul16))); 1777 else 1778 pc->v_broadcast_u64(f->vAddrMul, mem_ptr(fn.fetch_data(), REL_PATTERN(affine.addr_mul32))); 1779 1780 if (is_rect_fill()) { 1781 advance_px_py(f->tx_ty, x); 1782 normalize_px_py(f->tx_ty); 1783 } 1784 } 1785 1786 void FetchAffinePatternPart::_fini_part() noexcept {} 1787 1788 // bl::Pipeline::JIT::FetchAffinePatternPart - Advance 1789 // =================================================== 1790 1791 void FetchAffinePatternPart::advance_y() noexcept { 1792 pc->v_add_i64(f->tx_ty, f->tx_ty, f->yx_yy); 1793 1794 if (is_rect_fill()) 1795 normalize_px_py(f->tx_ty); 1796 } 1797 1798 void FetchAffinePatternPart::start_at_x(const Gp& x) noexcept { 1799 if (is_rect_fill()) { 1800 pc->v_mov(f->px_py, f->tx_ty); 1801 } 1802 else { 1803 // Similar to `advance_px_py()`, however, we don't need a temporary here... 1804 pc->s_mov_u32(f->px_py, x.r32()); 1805 pc->v_swizzle_u32x4(f->px_py, f->px_py, swizzle(1, 0, 1, 0)); 1806 pc->v_mul_u64_lo_u32(f->px_py, f->xx_xy, f->px_py); 1807 pc->v_add_i64(f->px_py, f->px_py, f->tx_ty); 1808 1809 normalize_px_py(f->px_py); 1810 } 1811 1812 if (pixel_granularity() > 1) 1813 enter_n(); 1814 } 1815 1816 void FetchAffinePatternPart::advance_x(const Gp& x, const Gp& diff) noexcept { 1817 bl_unused(x); 1818 BL_ASSERT(!is_rect_fill()); 1819 1820 if (pixel_granularity() > 1) 1821 leave_n(); 1822 1823 advance_px_py(f->px_py, diff); 1824 normalize_px_py(f->px_py); 1825 1826 if (pixel_granularity() > 1) 1827 enter_n(); 1828 } 1829 1830 void FetchAffinePatternPart::advance_px_py(Vec& px_py, const Gp& i) noexcept { 1831 Vec t = pc->new_vec128("@t"); 1832 1833 pc->s_mov_u32(t, i.r32()); 1834 pc->v_swizzle_u32x4(t, t, swizzle(1, 0, 1, 0)); 1835 pc->v_mul_u64_lo_u32(t, f->xx_xy, t); 1836 pc->v_add_i64(px_py, px_py, t); 1837 } 1838 1839 void FetchAffinePatternPart::normalize_px_py(Vec& px_py) noexcept { 1840 Vec v0 = pc->new_vec128("v0"); 1841 1842 pc->v_zero_i(v0); 1843 pc->xModI64HIxDouble(px_py, px_py, f->tw_th); 1844 pc->v_cmp_gt_i32(v0, v0, px_py); 1845 pc->v_and_i32(v0, v0, f->rx_ry); 1846 pc->v_add_i32(px_py, px_py, v0); 1847 1848 pc->v_cmp_gt_i32(v0, px_py, f->ox_oy); 1849 pc->v_and_i32(v0, v0, f->rx_ry); 1850 pc->v_sub_i32(px_py, px_py, v0); 1851 } 1852 1853 void FetchAffinePatternPart::clamp_vec_idx_32(Vec& dst, const Vec& src, ClampStep step) noexcept { 1854 switch (step) { 1855 // Step A - Handle a possible underflow (PAD). 1856 // 1857 // We know that `minx_miny` can contain these values (per vector element): 1858 // 1859 // a) `minx_miny == 0` to handle PAD case. 1860 // b) `minx_miny == INT32_MIN` to handle REPEAT & REFLECT cases. 1861 // 1862 // This means that we either clamp to zero if `src` is negative and `minx_miny == 0` 1863 // or we don't clamp at all in case that `minx_miny == INT32_MIN`. This means that 1864 // we don't need a pure `PMAXSD` replacement in pure SSE2 mode, just something that 1865 // works for the mentioned cases. 1866 case kClampStepA_NN: 1867 case kClampStepA_BI: { 1868 #if defined(BL_JIT_ARCH_X86) 1869 if (!pc->has_sse4_1()) { 1870 if (dst.id() == src.id()) { 1871 Vec tmp = pc->new_vec128("f.vIdxPad"); 1872 pc->v_mov(tmp, src); 1873 pc->v_cmp_gt_i32(dst, dst, f->minx_miny); // `-1` if `src` is greater than `minx_miny`. 1874 pc->v_and_i32(dst, dst, tmp); // Changes `dst` to `0` if clamped. 1875 } 1876 else { 1877 pc->v_mov(dst, src); 1878 pc->v_cmp_gt_i32(dst, dst, f->minx_miny); // `-1` if `src` is greater than `minx_miny`. 1879 pc->v_and_i32(dst, dst, src); // Changes `dst` to `0` if clamped. 1880 } 1881 break; 1882 } 1883 #endif // BL_JIT_ARCH_X86 1884 1885 pc->v_max_i32(dst, src, f->minx_miny); 1886 break; 1887 } 1888 1889 // Step B - Handle a possible overflow (PAD | Bilinear overflow). 1890 case kClampStepB_NN: 1891 case kClampStepB_BI: { 1892 // Always performed on the same register. 1893 BL_ASSERT(dst.id() == src.id()); 1894 1895 #if defined(BL_JIT_ARCH_X86) 1896 // TODO: [JIT] OPTIMIZATION: AVX-512 masking seems slower than AVX2. 1897 // if (pc->has_avx512()) { 1898 // x86::KReg k = cc->new_kw("f.kTmp"); 1899 // cc->vpcmpgtd(k, dst, f->maxx_maxy); 1900 // cc->k(k).vmovdqa32(dst, f->corx_cory); 1901 // } 1902 1903 if (!pc->has_sse4_1()) { 1904 // Blend(a, b, cond) == a ^ ((a ^ b) & cond) 1905 // == b ^ ((a ^ b) & ~cond) 1906 Vec tmp = pc->new_vec128("f.v_tmp"); 1907 pc->v_xor_i32(tmp, dst, f->corx_cory); 1908 pc->v_cmp_gt_i32(dst, dst, f->maxx_maxy); 1909 pc->v_andn_i32(dst, dst, tmp); 1910 pc->v_xor_i32(dst, dst, f->corx_cory); 1911 break; 1912 } 1913 #endif // BL_JIT_ARCH_X86 1914 1915 Vec tmp = pc->new_vec128("f.v_tmp"); 1916 pc->v_cmp_gt_i32(tmp, dst, f->maxx_maxy); 1917 pc->v_blendv_u8(dst, dst, f->corx_cory, tmp); 1918 break; 1919 } 1920 1921 // Step C - Handle a possible reflection (RoR). 1922 case kClampStepC_NN: 1923 case kClampStepC_BI: { 1924 // Always performed on the same register. 1925 BL_ASSERT(dst.id() == src.id()); 1926 1927 Vec tmp = pc->new_vec128("f.vIdxRoR"); 1928 pc->v_srai_i32(tmp, dst, 31); 1929 pc->v_xor_i32(dst, dst, tmp); 1930 break; 1931 } 1932 1933 default: 1934 BL_NOT_REACHED(); 1935 } 1936 } 1937 1938 // bl::Pipeline::JIT::FetchAffinePatternPart - Fetch 1939 // ================================================= 1940 1941 void FetchAffinePatternPart::enter_n() noexcept { 1942 Vec vMsk0 = pc->new_vec128("vMsk0"); 1943 1944 pc->v_add_i64(f->qx_qy, f->px_py, f->xx_xy); 1945 pc->v_cmp_gt_i32(vMsk0, f->qx_qy, f->ox_oy); 1946 pc->v_and_i32(vMsk0, vMsk0, f->rx_ry); 1947 pc->v_sub_i32(f->qx_qy, f->qx_qy, vMsk0); 1948 } 1949 1950 void FetchAffinePatternPart::leave_n() noexcept {} 1951 1952 void FetchAffinePatternPart::prefetch_n() noexcept {} 1953 void FetchAffinePatternPart::postfetch_n() noexcept {} 1954 1955 void FetchAffinePatternPart::fetch(Pixel& p, PixelCount n, PixelFlags flags, PixelPredicate& predicate) noexcept { 1956 p.set_count(n); 1957 1958 GatherMode gather_mode = predicate.gather_mode(); 1959 1960 switch (uint32_t(n)) { 1961 case 1: { 1962 BL_ASSERT(predicate.is_empty()); 1963 1964 switch (fetch_type()) { 1965 case FetchType::kPatternAffineNNAny: { 1966 Gp tex_ptr = pc->new_gpz("tex_ptr"); 1967 Gp tex_off = pc->new_gpz("tex_off"); 1968 1969 Vec v_idx = f->v_idx; 1970 Vec v_msk = pc->new_vec128("v_msk"); 1971 1972 clamp_vec_idx_32(v_idx, f->px_py, kClampStepA_NN); 1973 clamp_vec_idx_32(v_idx, v_idx, kClampStepB_NN); 1974 clamp_vec_idx_32(v_idx, v_idx, kClampStepC_NN); 1975 pc->v_add_i64(f->px_py, f->px_py, f->xx_xy); 1976 1977 FetchUtils::IndexExtractor iExt(pc); 1978 iExt.begin(FetchUtils::IndexExtractor::kTypeUInt32, v_idx); 1979 iExt.extract(tex_ptr, 3); 1980 iExt.extract(tex_off, 1); 1981 1982 pc->v_cmp_gt_i32(v_msk, f->px_py, f->ox_oy); 1983 pc->mul(tex_ptr, tex_ptr, f->stride); 1984 pc->v_and_i32(v_msk, v_msk, f->rx_ry); 1985 pc->v_sub_i32(f->px_py, f->px_py, v_msk); 1986 pc->add(tex_ptr, tex_ptr, f->srctop); 1987 1988 FetchUtils::fetch_pixel(pc, p, flags, fetch_info(), mem_ptr(tex_ptr, tex_off, _idx_shift)); 1989 FetchUtils::satisfy_pixels(pc, p, flags); 1990 break; 1991 } 1992 1993 case FetchType::kPatternAffineNNOpt: { 1994 Gp tex_ptr = pc->new_gpz("tex_ptr"); 1995 Vec v_idx = f->v_idx; 1996 Vec v_msk = pc->new_vec128("v_msk"); 1997 1998 pc->v_swizzle_u32x4(v_idx, f->px_py, swizzle(3, 1, 3, 1)); 1999 pc->v_packs_i32_i16(v_idx, v_idx, v_idx); 2000 pc->v_max_i16(v_idx, v_idx, f->minx_miny); 2001 pc->v_min_i16(v_idx, v_idx, f->maxx_maxy); 2002 2003 pc->v_srai_i16(v_msk, v_idx, 15); 2004 pc->v_xor_i32(v_idx, v_idx, v_msk); 2005 2006 pc->v_add_i64(f->px_py, f->px_py, f->xx_xy); 2007 pc->v_mhadd_i16_to_i32(v_idx, v_idx, f->vAddrMul); 2008 2009 pc->v_cmp_gt_i32(v_msk, f->px_py, f->ox_oy); 2010 pc->v_and_i32(v_msk, v_msk, f->rx_ry); 2011 pc->v_sub_i32(f->px_py, f->px_py, v_msk); 2012 pc->s_mov_u32(tex_ptr.r32(), v_idx); 2013 pc->add(tex_ptr, tex_ptr, f->srctop); 2014 2015 FetchUtils::fetch_pixel(pc, p, flags, fetch_info(), mem_ptr(tex_ptr)); 2016 FetchUtils::satisfy_pixels(pc, p, flags); 2017 break; 2018 } 2019 2020 case FetchType::kPatternAffineBIAny: { 2021 if (is_alpha_fetch()) { 2022 Vec v_idx = pc->new_vec128("v_idx"); 2023 Vec v_msk = pc->new_vec128("v_msk"); 2024 Vec v_weights = pc->new_vec128("v_weights"); 2025 2026 pc->v_swizzle_u32x4(v_idx, f->px_py, swizzle(3, 3, 1, 1)); 2027 pc->v_sub_i32(v_idx, v_idx, pc->simd_const(&ct.p_FFFFFFFF00000000, Bcst::kNA, v_idx)); 2028 2029 pc->v_swizzle_lo_u16x4(v_weights, f->px_py, swizzle(1, 1, 1, 1)); 2030 clamp_vec_idx_32(v_idx, v_idx, kClampStepA_BI); 2031 2032 pc->v_add_i64(f->px_py, f->px_py, f->xx_xy); 2033 clamp_vec_idx_32(v_idx, v_idx, kClampStepB_BI); 2034 2035 pc->v_cmp_gt_i32(v_msk, f->px_py, f->ox_oy); 2036 pc->v_swizzle_hi_u16x4(v_weights, v_weights, swizzle(1, 1, 1, 1)); 2037 2038 pc->v_and_i32(v_msk, v_msk, f->rx_ry); 2039 pc->v_srli_u16(v_weights, v_weights, 8); 2040 2041 pc->v_sub_i32(f->px_py, f->px_py, v_msk); 2042 pc->v_xor_i32(v_weights, v_weights, pc->simd_const(&ct.p_FFFF0000FFFF0000, Bcst::k32, v_weights)); 2043 2044 clamp_vec_idx_32(v_idx, v_idx, kClampStepC_BI); 2045 pc->v_add_u16(v_weights, v_weights, pc->simd_const(&ct.p_0101000001010000, Bcst::kNA, v_weights)); 2046 2047 Vec pixA = pc->new_vec128("pixA"); 2048 FetchUtils::filter_bilinear_a8_1x(pc, pixA, f->srctop, f->stride, fetch_info(), _idx_shift, v_idx, v_weights); 2049 2050 FetchUtils::x_assign_unpacked_alpha_values(pc, p, flags, pixA); 2051 FetchUtils::satisfy_pixels(pc, p, flags); 2052 } 2053 else if (p.isRGBA32()) { 2054 Vec v_idx = pc->new_vec128("v_idx"); 2055 Vec v_msk = pc->new_vec128("v_msk"); 2056 Vec v_weights = pc->new_vec128("v_weights"); 2057 2058 pc->v_swizzle_u32x4(v_idx, f->px_py, swizzle(3, 3, 1, 1)); 2059 pc->v_sub_i32(v_idx, v_idx, pc->simd_const(&ct.p_FFFFFFFF00000000, Bcst::kNA, v_idx)); 2060 2061 #if defined(BL_JIT_ARCH_X86) 2062 if (!pc->has_ssse3()) { 2063 pc->v_swizzle_u16x4(v_weights, f->px_py, swizzle(1, 1, 1, 1)); 2064 pc->v_srli_u16(v_weights, v_weights, 8); 2065 } 2066 else 2067 #endif 2068 { 2069 pc->v_swizzlev_u8(v_weights, f->px_py, pc->simd_const(&ct.swizu8_xxxx1xxxxxxx0xxx_to_z1z1z1z1z0z0z0z0, Bcst::kNA, v_weights)); 2070 } 2071 2072 pc->v_add_i64(f->px_py, f->px_py, f->xx_xy); 2073 clamp_vec_idx_32(v_idx, v_idx, kClampStepA_BI); 2074 pc->v_xor_i64(v_weights, v_weights, pc->simd_const(&ct.p_FFFFFFFF00000000, Bcst::k64, v_weights)); 2075 pc->v_cmp_gt_i32(v_msk, f->px_py, f->ox_oy); 2076 2077 clamp_vec_idx_32(v_idx, v_idx, kClampStepB_BI); 2078 pc->v_and_i32(v_msk, v_msk, f->rx_ry); 2079 2080 pc->v_add_u16(v_weights, v_weights, pc->simd_const(&ct.p_0101010100000000, Bcst::kNA, v_weights)); 2081 pc->v_sub_i32(f->px_py, f->px_py, v_msk); 2082 clamp_vec_idx_32(v_idx, v_idx, kClampStepC_BI); 2083 2084 p.uc.init(pc->new_vec128("pix0")); 2085 FetchUtils::filter_bilinear_argb32_1x(pc, p.uc[0], f->srctop, f->stride, v_idx, v_weights); 2086 FetchUtils::satisfy_pixels(pc, p, flags); 2087 } 2088 break; 2089 } 2090 2091 case FetchType::kPatternAffineBIOpt: { 2092 // TODO: [JIT] OPTIMIZATION: Not used at the moment. 2093 break; 2094 } 2095 2096 default: 2097 BL_NOT_REACHED(); 2098 } 2099 2100 break; 2101 } 2102 2103 case 4: { 2104 switch (fetch_type()) { 2105 case FetchType::kPatternAffineNNAny: { 2106 FetchUtils::FetchContext fCtx(pc, &p, PixelCount(4), flags, fetch_info()); 2107 FetchUtils::IndexExtractor iExt(pc); 2108 2109 Gp texPtr0 = pc->new_gpz("texPtr0"); 2110 Gp texOff0 = pc->new_gpz("texOff0"); 2111 Gp texPtr1 = pc->new_gpz("texPtr1"); 2112 Gp texOff1 = pc->new_gpz("texOff1"); 2113 2114 Vec vIdx0 = pc->new_vec128("vIdx0"); 2115 Vec vIdx1 = pc->new_vec128("vIdx1"); 2116 Vec vMsk0 = pc->new_vec128("vMsk0"); 2117 Vec vMsk1 = pc->new_vec128("vMsk1"); 2118 2119 pc->v_interleave_shuffle_u32x4(vIdx0, f->px_py, f->qx_qy, swizzle(3, 1, 3, 1)); 2120 pc->v_add_i64(f->px_py, f->px_py, f->xx2_xy2); 2121 2122 clamp_vec_idx_32(vIdx0, vIdx0, kClampStepA_NN); 2123 pc->v_add_i64(f->qx_qy, f->qx_qy, f->xx2_xy2); 2124 2125 clamp_vec_idx_32(vIdx0, vIdx0, kClampStepB_NN); 2126 pc->v_cmp_gt_i32(vMsk0, f->px_py, f->ox_oy); 2127 clamp_vec_idx_32(vIdx0, vIdx0, kClampStepC_NN); 2128 2129 pc->v_cmp_gt_i32(vMsk1, f->qx_qy, f->ox_oy); 2130 pc->v_and_i32(vMsk0, vMsk0, f->rx_ry); 2131 pc->v_and_i32(vMsk1, vMsk1, f->rx_ry); 2132 pc->v_sub_i32(f->px_py, f->px_py, vMsk0); 2133 pc->v_sub_i32(f->qx_qy, f->qx_qy, vMsk1); 2134 2135 iExt.begin(FetchUtils::IndexExtractor::kTypeUInt32, vIdx0); 2136 pc->v_interleave_shuffle_u32x4(vIdx1, f->px_py, f->qx_qy, swizzle(3, 1, 3, 1)); 2137 iExt.extract(texPtr0, 1); 2138 iExt.extract(texOff0, 0); 2139 2140 clamp_vec_idx_32(vIdx1, vIdx1, kClampStepA_NN); 2141 clamp_vec_idx_32(vIdx1, vIdx1, kClampStepB_NN); 2142 2143 iExt.extract(texPtr1, 3); 2144 iExt.extract(texOff1, 2); 2145 2146 pc->mul(texPtr0, texPtr0, f->stride); 2147 pc->mul(texPtr1, texPtr1, f->stride); 2148 2149 clamp_vec_idx_32(vIdx1, vIdx1, kClampStepC_NN); 2150 pc->v_add_i64(f->px_py, f->px_py, f->xx2_xy2); 2151 pc->v_add_i64(f->qx_qy, f->qx_qy, f->xx2_xy2); 2152 2153 pc->add(texPtr0, texPtr0, f->srctop); 2154 pc->add(texPtr1, texPtr1, f->srctop); 2155 iExt.begin(FetchUtils::IndexExtractor::kTypeUInt32, vIdx1); 2156 2157 fCtx.fetch_pixel(mem_ptr(texPtr0, texOff0, _idx_shift)); 2158 iExt.extract(texPtr0, 1); 2159 iExt.extract(texOff0, 0); 2160 2161 pc->v_cmp_gt_i32(vMsk0, f->px_py, f->ox_oy); 2162 pc->v_cmp_gt_i32(vMsk1, f->qx_qy, f->ox_oy); 2163 2164 fCtx.fetch_pixel(mem_ptr(texPtr1, texOff1, _idx_shift)); 2165 iExt.extract(texPtr1, 3); 2166 iExt.extract(texOff1, 2); 2167 pc->mul(texPtr0, texPtr0, f->stride); 2168 2169 pc->v_and_i32(vMsk0, vMsk0, f->rx_ry); 2170 pc->v_and_i32(vMsk1, vMsk1, f->rx_ry); 2171 2172 pc->mul(texPtr1, texPtr1, f->stride); 2173 pc->v_sub_i32(f->px_py, f->px_py, vMsk0); 2174 2175 pc->add(texPtr0, texPtr0, f->srctop); 2176 pc->add(texPtr1, texPtr1, f->srctop); 2177 fCtx.fetch_pixel(mem_ptr(texPtr0, texOff0, _idx_shift)); 2178 2179 pc->v_sub_i32(f->qx_qy, f->qx_qy, vMsk1); 2180 fCtx.fetch_pixel(mem_ptr(texPtr1, texOff1, _idx_shift)); 2181 fCtx.end(); 2182 2183 FetchUtils::satisfy_pixels(pc, p, flags); 2184 break; 2185 } 2186 2187 case FetchType::kPatternAffineNNOpt: { 2188 Vec v_idx = f->v_idx; 2189 Vec vMsk0 = pc->new_vec128("vMsk0"); 2190 Vec vMsk1 = pc->new_vec128("vMsk1"); 2191 2192 pc->v_interleave_shuffle_u32x4(v_idx, f->px_py, f->qx_qy, swizzle(3, 1, 3, 1)); 2193 pc->v_add_i64(f->px_py, f->px_py, f->xx2_xy2); 2194 pc->v_add_i64(f->qx_qy, f->qx_qy, f->xx2_xy2); 2195 2196 pc->v_cmp_gt_i32(vMsk0, f->px_py, f->ox_oy); 2197 pc->v_cmp_gt_i32(vMsk1, f->qx_qy, f->ox_oy); 2198 2199 pc->v_and_i32(vMsk0, vMsk0, f->rx_ry); 2200 pc->v_and_i32(vMsk1, vMsk1, f->rx_ry); 2201 2202 pc->v_sub_i32(f->px_py, f->px_py, vMsk0); 2203 pc->v_sub_i32(f->qx_qy, f->qx_qy, vMsk1); 2204 2205 pc->v_interleave_shuffle_u32x4(vMsk0, f->px_py, f->qx_qy, swizzle(3, 1, 3, 1)); 2206 pc->v_packs_i32_i16(v_idx, v_idx, vMsk0); 2207 2208 pc->v_max_i16(v_idx, v_idx, f->minx_miny); 2209 pc->v_min_i16(v_idx, v_idx, f->maxx_maxy); 2210 2211 pc->v_srai_i16(vMsk0, v_idx, 15); 2212 pc->v_xor_i32(v_idx, v_idx, vMsk0); 2213 2214 pc->v_mhadd_i16_to_i32(v_idx, v_idx, f->vAddrMul); 2215 FetchUtils::gather_pixels(pc, p, PixelCount(4), flags, fetch_info(), mem_ptr(f->srctop), v_idx, 0, FetchUtils::IndexLayout::kUInt32, gather_mode, [&](uint32_t step) noexcept { 2216 switch (step) { 2217 case 0: 2218 pc->v_add_i64(f->px_py, f->px_py, f->xx2_xy2); 2219 pc->v_add_i64(f->qx_qy, f->qx_qy, f->xx2_xy2); 2220 break; 2221 case 1: 2222 pc->v_cmp_gt_i32(vMsk0, f->px_py, f->ox_oy); 2223 pc->v_cmp_gt_i32(vMsk1, f->qx_qy, f->ox_oy); 2224 break; 2225 case 2: 2226 pc->v_and_i32(vMsk0, vMsk0, f->rx_ry); 2227 pc->v_and_i32(vMsk1, vMsk1, f->rx_ry); 2228 break; 2229 case 3: 2230 pc->v_sub_i32(f->px_py, f->px_py, vMsk0); 2231 pc->v_sub_i32(f->qx_qy, f->qx_qy, vMsk1); 2232 break; 2233 default: 2234 break; 2235 } 2236 }); 2237 2238 FetchUtils::satisfy_pixels(pc, p, flags); 2239 break; 2240 } 2241 2242 default: 2243 BL_NOT_REACHED(); 2244 } 2245 2246 break; 2247 } 2248 2249 default: 2250 BL_NOT_REACHED(); 2251 } 2252 } 2253 2254 } // {bl::Pipeline::JIT} 2255 2256 #endif // !BL_BUILD_NO_JIT