fillpart.cpp (60610B)
1 // This file is part of Blend2D project <https://blend2d.com> 2 // 3 // See blend2d.h or LICENSE.md for license and copyright information 4 // SPDX-License-Identifier: Zlib 5 6 #include "../../api-build_p.h" 7 #if !defined(BL_BUILD_NO_JIT) 8 9 #include "../../pipeline/jit/compoppart_p.h" 10 #include "../../pipeline/jit/fillpart_p.h" 11 #include "../../pipeline/jit/fetchpart_p.h" 12 #include "../../pipeline/jit/fetchpixelptrpart_p.h" 13 #include "../../pipeline/jit/fetchutilscoverage_p.h" 14 #include "../../pipeline/jit/fetchutilsinlineloops_p.h" 15 #include "../../pipeline/jit/fetchutilspixelaccess_p.h" 16 #include "../../pipeline/jit/pipecompiler_p.h" 17 18 namespace bl::Pipeline::JIT { 19 20 // bl::Pipeline::JIT::FillPart - Utilities 21 // ======================================= 22 23 static uint32_t calculate_coverage_byte_count(PixelCount pixel_count, PixelType pixel_type, PixelCoverageFormat coverage_format) noexcept { 24 DataWidth data_width = DataWidth::k8; 25 26 switch (coverage_format) { 27 case PixelCoverageFormat::kPacked: 28 data_width = DataWidth::k8; 29 break; 30 31 case PixelCoverageFormat::kUnpacked: 32 data_width = DataWidth::k16; 33 break; 34 35 default: 36 BL_NOT_REACHED(); 37 } 38 39 uint32_t count = uint32_t(pixel_count); 40 switch (pixel_type) { 41 case PixelType::kA8: 42 break; 43 44 case PixelType::kRGBA32: 45 count *= 4u; 46 break; 47 48 default: 49 BL_NOT_REACHED(); 50 } 51 52 return (1u << uint32_t(data_width)) * count; 53 } 54 55 static void init_vec_coverage( 56 PipeCompiler* pc, 57 VecArray& dst, 58 PixelCount max_pixel_count, 59 VecWidth acc_vec_width, 60 VecWidth max_vec_width, 61 PixelType pixel_type, 62 PixelCoverageFormat coverage_format) noexcept { 63 64 uint32_t coverage_byte_count = calculate_coverage_byte_count(max_pixel_count, pixel_type, coverage_format); 65 VecWidth vec_width = VecWidthUtils::vec_width_for_byte_count(max_vec_width, coverage_byte_count); 66 size_t vec_count = VecWidthUtils::vec_count_for_byte_count(vec_width, coverage_byte_count); 67 68 pc->new_vec_array(dst, vec_count, bl_max(vec_width, acc_vec_width), "vm"); 69 70 // The width of the register must match the accumulator (as otherwise AsmJit could 71 // spill and only load a part of it in case the vector width of `dst` is smaller). 72 dst.set_vec_width(vec_width); 73 } 74 75 static void pass_vec_coverage( 76 VecArray& dst, 77 const VecArray& src, 78 PixelCount pixel_count, 79 PixelType pixel_type, 80 PixelCoverageFormat coverage_format) noexcept { 81 82 uint32_t coverage_byte_count = calculate_coverage_byte_count(pixel_count, pixel_type, coverage_format); 83 VecWidth vec_width = VecWidthUtils::vec_width_for_byte_count(VecWidthUtils::vec_width_of(src[0]), coverage_byte_count); 84 size_t vec_count = VecWidthUtils::vec_count_for_byte_count(vec_width, coverage_byte_count); 85 86 // We can use at most what was given to us, or less in case that the current 87 // `pixel_count` is less than `max_pixel_count` passed to `init_vec_coverage()`. 88 BL_ASSERT(vec_count <= src.size()); 89 90 dst._size = vec_count; 91 for (size_t i = 0; i < vec_count; i++) { 92 dst.v[i].reset(); 93 dst.v[i].as<asmjit::Reg>().set_signature_and_id(VecWidthUtils::signature_of(vec_width), src.v[i].id()); 94 } 95 } 96 97 // bl::Pipeline::JIT::FillPart - Construction & Destruction 98 // ======================================================== 99 100 FillPart::FillPart(PipeCompiler* pc, FillType fill_type, FetchPixelPtrPart* dst_part, CompOpPart* comp_op_part) noexcept 101 : PipePart(pc, PipePartType::kFill), 102 _fill_type(fill_type) { 103 104 // Initialize the children of this part. 105 _children[kIndexDstPart] = dst_part; 106 _children[kIndexCompOpPart] = comp_op_part; 107 _child_count = 2; 108 } 109 110 // [[pure virtual]] 111 void FillPart::compile(const PipeFunction& fn) noexcept { 112 bl_unused(fn); 113 BL_NOT_REACHED(); 114 } 115 116 // bl::Pipeline::JIT::FillBoxAPart - Construction & Destruction 117 // ============================================================ 118 119 FillBoxAPart::FillBoxAPart(PipeCompiler* pc, FetchPixelPtrPart* dst_part, CompOpPart* comp_op_part) noexcept 120 : FillPart(pc, FillType::kBoxA, dst_part, comp_op_part) { 121 122 add_part_flags(PipePartFlags::kRectFill); 123 _max_vec_width_supported = kMaxPlatformWidth; 124 } 125 126 // bl::Pipeline::JIT::FillBoxAPart - Compile 127 // ========================================= 128 129 void FillBoxAPart::compile(const PipeFunction& fn) noexcept { 130 // Prepare 131 // ------- 132 133 _init_global_hook(cc->cursor()); 134 135 int dst_bpp = int(dst_part()->bpp()); 136 bool is_src_copy_fill = comp_op_part()->is_src_copy() && comp_op_part()->src_part()->is_solid(); 137 138 // Local Registers 139 // --------------- 140 141 Gp ctx_data = fn.ctx_data(); // Reg/Init. 142 Gp fill_data = fn.fill_data(); // Reg/Init. 143 144 Gp dst_ptr = pc->new_gpz("dst_ptr"); // Reg. 145 Gp dst_stride = pc->new_gpz("dst_stride"); // Reg/Mem. 146 147 Gp x = pc->new_gp32("x"); // Reg. 148 Gp y = pc->new_gp32("y"); // Reg/Mem. 149 Gp w = pc->new_gp32("w"); // Reg/Mem. 150 Gp ga_sm = pc->new_gp32("ga.sm"); // Reg/Tmp. 151 152 // Prolog 153 // ------ 154 155 pc->load(dst_stride, mem_ptr(ctx_data, BL_OFFSET_OF(ContextData, dst.stride))); 156 pc->load_u32(y, mem_ptr(fill_data, BL_OFFSET_OF(FillData::BoxA, box.y0))); 157 pc->load_u32(w, mem_ptr(fill_data, BL_OFFSET_OF(FillData::BoxA, box.x0))); 158 159 pc->mul(dst_ptr, dst_stride, y.clone_as(dst_ptr)); 160 161 dst_part()->init_ptr(dst_ptr); 162 comp_op_part()->init(fn, w, y, 1); 163 164 pc->add_ext(dst_ptr, dst_ptr, w, uint32_t(dst_bpp)); 165 pc->sub(w, mem_ptr(fill_data, BL_OFFSET_OF(FillData::BoxA, box.x1)), w); 166 pc->sub(y, mem_ptr(fill_data, BL_OFFSET_OF(FillData::BoxA, box.y1)), y); 167 pc->mul(x, w, dst_bpp); 168 pc->add(dst_ptr, dst_ptr, mem_ptr(ctx_data, BL_OFFSET_OF(ContextData, dst.pixel_data))); 169 170 if (is_src_copy_fill) { 171 Label L_NotStride = pc->new_label(); 172 173 pc->j(L_NotStride, cmp_ne(x.clone_as(dst_stride), dst_stride)); 174 pc->mul(w, w, y); 175 pc->mov(y, 1); 176 pc->bind(L_NotStride); 177 } 178 else { 179 // Only subtract from destination stride if this is not a solid rectangular fill. 180 pc->sub(dst_stride, dst_stride, x.clone_as(dst_stride)); 181 } 182 183 // Loop 184 // ---- 185 186 if (comp_op_part()->should_optimize_opaque_fill()) { 187 Label L_SemiAlphaInit = pc->new_label(); 188 Label L_End = pc->new_label(); 189 190 pc->load_u32(ga_sm, mem_ptr(fill_data, BL_OFFSET_OF(FillData::BoxA, alpha))); 191 pc->j(L_SemiAlphaInit, cmp_ne(ga_sm, 255)); 192 193 // Full Alpha 194 // ---------- 195 196 if (is_src_copy_fill) { 197 // Optimize fill rect if it can be implemented as a memset. The main reason is 198 // that if the width is reasonably small we want to only check that condition once. 199 comp_op_part()->c_mask_init_opaque(); 200 BL_ASSERT(comp_op_part()->_solid_opt.px.is_valid()); 201 202 FetchUtils::inline_fill_rect_loop(pc, dst_ptr, dst_stride, w, y, comp_op_part()->_solid_opt.px, dst_part()->bpp(), L_End); 203 comp_op_part()->c_mask_fini(); 204 } 205 else { 206 Label L_AdvanceY = pc->new_label(); 207 Label L_ProcessY = pc->new_label(); 208 209 comp_op_part()->c_mask_init_opaque(); 210 pc->j(L_ProcessY); 211 212 pc->bind(L_AdvanceY); 213 comp_op_part()->advance_y(); 214 pc->add(dst_ptr, dst_ptr, dst_stride); 215 216 pc->bind(L_ProcessY); 217 pc->mov(x, w); 218 comp_op_part()->start_at_x(pc->_gp_none); 219 comp_op_part()->c_mask_generic_loop(x); 220 pc->j(L_AdvanceY, sub_nz(y, 1)); 221 222 comp_op_part()->c_mask_fini(); 223 pc->j(L_End); 224 } 225 226 // Semi Alpha 227 // ---------- 228 229 { 230 Label L_AdvanceY = pc->new_label(); 231 Label L_ProcessY = pc->new_label(); 232 233 pc->bind(L_SemiAlphaInit); 234 235 if (is_src_copy_fill) { 236 // This was not accounted yet as `inline_fill_rect_loop()` expects full stride, so we have to account this now. 237 pc->sub(dst_stride, dst_stride, x.clone_as(dst_stride)); 238 } 239 240 comp_op_part()->c_mask_init(ga_sm, Vec()); 241 pc->j(L_ProcessY); 242 243 pc->bind(L_AdvanceY); 244 comp_op_part()->advance_y(); 245 pc->add(dst_ptr, dst_ptr, dst_stride); 246 247 pc->bind(L_ProcessY); 248 pc->mov(x, w); 249 comp_op_part()->start_at_x(pc->_gp_none); 250 comp_op_part()->c_mask_generic_loop(x); 251 pc->j(L_AdvanceY, sub_nz(y, 1)); 252 253 comp_op_part()->c_mask_fini(); 254 pc->bind(L_End); 255 } 256 } 257 else { 258 Label L_AdvanceY = pc->new_label(); 259 Label L_ProcessY = pc->new_label(); 260 261 comp_op_part()->c_mask_init(mem_ptr(fill_data, BL_OFFSET_OF(FillData::BoxA, alpha))); 262 pc->j(L_ProcessY); 263 264 pc->bind(L_AdvanceY); 265 comp_op_part()->advance_y(); 266 pc->add(dst_ptr, dst_ptr, dst_stride); 267 268 pc->bind(L_ProcessY); 269 pc->mov(x, w); 270 comp_op_part()->start_at_x(pc->_gp_none); 271 comp_op_part()->c_mask_generic_loop(x); 272 pc->j(L_AdvanceY, sub_nz(y, 1)); 273 274 comp_op_part()->c_mask_fini(); 275 } 276 277 // Epilog 278 // ------ 279 280 comp_op_part()->fini(); 281 _fini_global_hook(); 282 } 283 284 // bl::Pipeline::JIT::FillMaskPart - Construction & Destruction 285 // ============================================================ 286 287 FillMaskPart::FillMaskPart(PipeCompiler* pc, FetchPixelPtrPart* dst_part, CompOpPart* comp_op_part) noexcept 288 : FillPart(pc, FillType::kMask, dst_part, comp_op_part) { 289 290 _max_vec_width_supported = kMaxPlatformWidth; 291 } 292 293 // bl::Pipeline::JIT::FillMaskPart - Compile 294 // ========================================= 295 296 void FillMaskPart::compile(const PipeFunction& fn) noexcept { 297 // EndOrRepeat is expected to be zero for fast termination of the scanline. 298 BL_STATIC_ASSERT(uint32_t(MaskCommandType::kEndOrRepeat) == 0); 299 300 // Prepare 301 // ------- 302 303 _init_global_hook(cc->cursor()); 304 305 int dst_bpp = int(dst_part()->bpp()); 306 constexpr int kMaskCmdSize = int(sizeof(MaskCommand)); 307 308 #if defined(BL_JIT_ARCH_X86) 309 constexpr int label_alignment = 8; 310 #else 311 constexpr int label_alignment = 4; 312 #endif 313 314 // Local Labels 315 // ------------ 316 317 Label L_ScanlineInit = pc->new_label(); 318 Label L_ScanlineDone = pc->new_label(); 319 Label L_ScanlineSkip = pc->new_label(); 320 321 Label L_ProcessNext = pc->new_label(); 322 Label L_ProcessCmd = pc->new_label(); 323 Label L_CMaskInit = pc->new_label(); 324 Label L_VMaskA8WithoutGA = pc->new_label(); 325 Label L_End = pc->new_label(); 326 327 // Local Registers 328 // --------------- 329 330 Gp ctx_data = fn.ctx_data(); // Reg/Init. 331 Gp fill_data = fn.fill_data(); // Reg/Init. 332 333 Gp dst_ptr = pc->new_gpz("dst_ptr"); // Reg. 334 Gp dst_stride = pc->new_gpz("dst_stride"); // Reg/Mem. 335 336 Gp i = pc->new_gp32("i"); // Reg. 337 Gp x = pc->new_gp32("x"); // Reg. 338 Gp y = pc->new_gp32("y"); // Reg/Mem. 339 340 Gp cmd_type = pc->new_gp32("cmd_type"); // Reg/Tmp. 341 Gp cmd_ptr = pc->new_gpz("cmd_ptr"); // Reg/Mem. 342 Gp cmd_begin = pc->new_gpz("cmd_begin"); // Mem. 343 Gp mask_value = pc->new_gpz("mask_value"); // Reg. 344 Gp mask_advance = pc->new_gpz("mask_advance"); // Reg/Tmp 345 346 GlobalAlpha ga; 347 348 // Prolog 349 // ------ 350 351 // Initialize the destination. 352 pc->load(dst_stride, mem_ptr(ctx_data, BL_OFFSET_OF(ContextData, dst.stride))); 353 pc->load_u32(y, mem_ptr(fill_data, BL_OFFSET_OF(FillData, mask.box.y0))); 354 355 pc->mul(dst_ptr, dst_stride, y.clone_as(dst_ptr)); 356 pc->add(dst_ptr, dst_ptr, mem_ptr(ctx_data, BL_OFFSET_OF(ContextData, dst.pixel_data))); 357 358 // Initialize pipeline parts. 359 dst_part()->init_ptr(dst_ptr); 360 comp_op_part()->init(fn, pc->_gp_none, y, 1); 361 362 // Initialize mask pointers. 363 pc->load(cmd_ptr, mem_ptr(fill_data, BL_OFFSET_OF(FillData, mask.mask_command_data))); 364 365 // Initialize global alpha. 366 ga.init_from_mem(pc, mem_ptr(fill_data, BL_OFFSET_OF(FillData, mask.alpha))); 367 368 // y = fill_data->box.y1 - fill_data->box.y0; 369 pc->sub(y, mem_ptr(fill_data, BL_OFFSET_OF(FillData, mask.box.y1)), y); 370 pc->j(L_ScanlineInit); 371 372 // Scanline Done 373 // ------------- 374 375 Gp repeat = pc->new_gp32("repeat"); 376 377 pc->align(AlignMode::kCode, label_alignment); 378 pc->bind(L_ScanlineDone); 379 deadvance_dst_ptr(dst_ptr, x, int(dst_bpp)); 380 381 pc->bind(L_ScanlineSkip); 382 pc->load_u32(repeat, mem_ptr(cmd_ptr, BL_OFFSET_OF(MaskCommand, _x0))); 383 pc->j(L_End, sub_z(y, 1)); 384 385 pc->sub(repeat, repeat, 1); 386 pc->add(dst_ptr, dst_ptr, dst_stride); 387 pc->store_u32(mem_ptr(cmd_ptr, BL_OFFSET_OF(MaskCommand, _x0)), repeat); 388 pc->add(cmd_ptr, cmd_ptr, kMaskCmdSize); 389 comp_op_part()->advance_y(); 390 pc->cmov(cmd_ptr, cmd_begin, cmp_ne(repeat, 0)); 391 392 // Scanline Init 393 // ------------- 394 395 pc->bind(L_ScanlineInit); 396 pc->load_u32(cmd_type, mem_ptr(cmd_ptr, BL_OFFSET_OF(MaskCommand, _x1_and_type))); 397 pc->mov(cmd_begin, cmd_ptr); 398 pc->load_u32(x, mem_ptr(cmd_ptr, BL_OFFSET_OF(MaskCommand, _x0))); 399 // This is not really common, but it's possible to skip entire scanlines with `kEndOrRepeat`. 400 pc->j(L_ScanlineSkip, test_z(cmd_type, MaskCommand::kTypeMask)); 401 402 pc->add_scaled(dst_ptr, x.clone_as(dst_ptr), dst_bpp); 403 comp_op_part()->start_at_x(x); 404 pc->j(L_ProcessCmd); 405 406 // Process Command 407 // --------------- 408 409 pc->bind(L_ProcessNext); 410 pc->load_u32(cmd_type, mem_ptr(cmd_ptr, kMaskCmdSize + BL_OFFSET_OF(MaskCommand, _x1_and_type))); 411 pc->load_u32(i, mem_ptr(cmd_ptr, kMaskCmdSize + BL_OFFSET_OF(MaskCommand, _x0))); 412 pc->add(cmd_ptr, cmd_ptr, kMaskCmdSize); 413 pc->j(L_ScanlineDone, test_z(cmd_type, MaskCommand::kTypeMask)); 414 415 // Only emit the jump if there is something significant to skip. 416 if (comp_op_part()->has_part_flag(PipePartFlags::kAdvanceXIsSimple)) 417 pc->sub(i, i, x); 418 else 419 pc->j(L_ProcessCmd, sub_z(i, x)); 420 421 pc->add(x, x, i); 422 pc->add_scaled(dst_ptr, i.clone_as(dst_ptr), dst_bpp); 423 comp_op_part()->advance_x(x, i); 424 425 pc->bind(L_ProcessCmd); 426 427 #if defined(BL_JIT_ARCH_X86) 428 if (pc->has_bmi2() && pc->is_64bit()) 429 { 430 // This saves one instruction on X86_64 as RORX provides a non-destructive destination. 431 pc->ror(i.r64(), cmd_type.r64(), MaskCommand::kTypeBits); 432 } 433 else 434 #endif // BL_JIT_ARCH_X86 435 { 436 pc->shr(i, cmd_type, MaskCommand::kTypeBits); 437 } 438 439 pc->and_(cmd_type, cmd_type, MaskCommand::kTypeMask); 440 pc->sub(i, i, x); 441 pc->load(mask_value, mem_ptr(cmd_ptr, BL_OFFSET_OF(MaskCommand, _value.data))); 442 pc->add(x, x, i); 443 444 // We know the command is not kEndOrRepeat, which allows this little trick. 445 pc->j(L_CMaskInit, cmp_eq(cmd_type, uint32_t(MaskCommandType::kCMask))); 446 447 // VMask Command 448 // ------------- 449 450 // Increments the advance in the mask command in case it would be repeated. 451 pc->load(mask_advance, mem_ptr(cmd_ptr, BL_OFFSET_OF(MaskCommand, _mask_advance))); 452 pc->mem_add(mem_ptr(cmd_ptr, BL_OFFSET_OF(MaskCommand, _value.ptr)), mask_advance); 453 454 pc->j(L_VMaskA8WithoutGA, cmp_eq(cmd_type, uint32_t(MaskCommandType::kVMaskA8WithoutGA))); 455 comp_op_part()->v_mask_generic_loop(i, dst_ptr, mask_value, nullptr, L_ProcessNext); 456 457 pc->bind(L_VMaskA8WithoutGA); 458 comp_op_part()->v_mask_generic_loop(i, dst_ptr, mask_value, &ga, L_ProcessNext); 459 460 // CMask Command 461 // ------------- 462 463 pc->align(AlignMode::kCode, label_alignment); 464 pc->bind(L_CMaskInit); 465 if (comp_op_part()->should_optimize_opaque_fill()) { 466 Label L_CLoop_Msk = pc->new_label(); 467 pc->j(L_CLoop_Msk, cmp_ne(mask_value.r32(), 255)); 468 469 comp_op_part()->c_mask_init_opaque(); 470 comp_op_part()->c_mask_generic_loop(i); 471 comp_op_part()->c_mask_fini(); 472 pc->j(L_ProcessNext); 473 474 pc->align(AlignMode::kCode, label_alignment); 475 pc->bind(L_CLoop_Msk); 476 } 477 478 comp_op_part()->c_mask_init(mask_value.r32(), Vec()); 479 comp_op_part()->c_mask_generic_loop(i); 480 comp_op_part()->c_mask_fini(); 481 pc->j(L_ProcessNext); 482 483 // Epilog 484 // ------ 485 486 pc->bind(L_End); 487 comp_op_part()->fini(); 488 _fini_global_hook(); 489 } 490 491 void FillMaskPart::deadvance_dst_ptr(const Gp& dst_ptr, const Gp& x, int dst_bpp) noexcept { 492 Gp x_adv = x.clone_as(dst_ptr); 493 494 if (IntOps::is_power_of_2(dst_bpp)) { 495 if (dst_bpp > 1) 496 pc->shl(x_adv, x_adv, IntOps::ctz(dst_bpp)); 497 pc->sub(dst_ptr, dst_ptr, x_adv); 498 } 499 else { 500 Gp dst_adv = pc->new_gpz("dst_adv"); 501 pc->mul(dst_adv, x_adv, dst_bpp); 502 pc->sub(dst_ptr, dst_ptr, dst_adv); 503 } 504 } 505 506 // bl::Pipeline::JIT::FillAnalyticPart - Construction & Destruction 507 // ================================================================ 508 509 FillAnalyticPart::FillAnalyticPart(PipeCompiler* pc, FetchPixelPtrPart* dst_part, CompOpPart* comp_op_part) noexcept 510 : FillPart(pc, FillType::kAnalytic, dst_part, comp_op_part) { 511 512 _max_vec_width_supported = kMaxPlatformWidth; 513 } 514 515 // bl::Pipeline::JIT::FillAnalyticPart - Compile 516 // ============================================= 517 518 void FillAnalyticPart::compile(const PipeFunction& fn) noexcept { 519 // Prepare 520 // ------- 521 522 _init_global_hook(cc->cursor()); 523 524 PixelType pixel_type = comp_op_part()->pixel_type(); 525 PixelCoverageFormat coverage_format = comp_op_part()->coverage_format(); 526 527 uint32_t dst_bpp = dst_part()->bpp(); 528 uint32_t max_pixels = comp_op_part()->max_pixels(); 529 530 // v_proc SIMD width describes SIMD width used to accumulate coverages and then to calculate alpha masks. In 531 // general if we only calculate 4 coverages at once we only need 128-bit SIMD. However, 8 and more coverages 532 // need 256-bit SIMD or higher, if available. At the moment we use always a single register for this purpose, 533 // so SIMD width determines how many pixels we can process in a v_mask loop at a time. 534 uint32_t v_proc_pixel_count = 0; 535 VecWidth v_proc_width = pc->vec_width(); 536 537 if (pc->vec_width() >= VecWidth::k256 && max_pixels >= 8) { 538 v_proc_pixel_count = 8; 539 v_proc_width = VecWidth::k256; 540 } 541 else { 542 v_proc_pixel_count = bl_min<uint32_t>(max_pixels, 4); 543 v_proc_width = VecWidth::k128; 544 } 545 546 int bw_size = int(sizeof(BLBitWord)); 547 int bw_size_in_bits = bw_size * 8; 548 549 int pixels_per_one_bit = 4; 550 int pixels_per_one_bit_shift = int(IntOps::ctz(pixels_per_one_bit)); 551 552 int pixel_granularity = pixels_per_one_bit; 553 int pixels_per_bit_word = pixels_per_one_bit * bw_size_in_bits; 554 int pixels_per_bit_word_shift = int(IntOps::ctz(pixels_per_bit_word)); 555 556 if (comp_op_part()->max_pixels_of_children() < 4) 557 pixel_granularity = 1; 558 559 // Local Labels 560 // ------------ 561 562 Label L_BitScan_Init = pc->new_label(); 563 Label L_BitScan_Iter = pc->new_label(); 564 Label L_BitScan_Match = pc->new_label(); 565 Label L_BitScan_End = pc->new_label(); 566 567 Label L_VLoop_Init = pc->new_label(); 568 Label L_CLoop_Init = pc->new_label(); 569 570 Label L_VTail_Init; 571 572 if (max_pixels >= 4) { 573 L_VTail_Init = pc->new_label(); 574 } 575 576 Label L_Scanline_Done0 = pc->new_label(); 577 Label L_Scanline_Done1 = pc->new_label(); 578 Label L_Scanline_AdvY = pc->new_label(); 579 Label L_Scanline_Iter = pc->new_label(); 580 Label L_Scanline_Init = pc->new_label(); 581 582 Label L_End = pc->new_label(); 583 584 // Local Registers 585 // --------------- 586 587 Gp ctx_data = fn.ctx_data(); // Init. 588 Gp fill_data = fn.fill_data(); // Init. 589 590 Gp dst_ptr = pc->new_gpz("dst_ptr"); // Reg. 591 Gp dst_stride = pc->new_gpz("dst_stride"); // Mem. 592 593 Gp bit_ptr = pc->new_gpz("bit_ptr"); // Reg. 594 Gp bit_ptr_end = pc->new_gpz("bit_ptr_end"); // Reg/Mem. 595 596 Gp bit_ptr_run_len = pc->new_gpz("bit_ptr_run_len"); // Mem. 597 Gp bit_ptr_skip_len = pc->new_gpz("bit_ptr_skip_len"); // Mem. 598 599 Gp cell_ptr = pc->new_gpz("cell_ptr"); // Reg. 600 Gp cell_stride = pc->new_gpz("cell_stride"); // Mem. 601 602 Gp x0 = pc->new_gp32("x0"); // Reg 603 Gp x_off = pc->new_gp32("x_off"); // Reg/Mem. 604 Gp x_end = pc->new_gp32("x_end"); // Mem. 605 Gp x_start = pc->new_gp32("x_start"); // Mem. 606 607 Gp y = pc->new_gp32("y"); // Reg/Mem. 608 Gp i = pc->new_gp32("i"); // Reg. 609 Gp c_mask_alpha = pc->new_gp32("c_mask_alpha"); // Reg/Tmp. 610 611 Gp bit_word = pc->new_gpz("bit_word"); // Reg/Mem. 612 Gp bit_word_tmp = pc->new_gpz("bit_word_tmp"); // Reg/Tmp. 613 614 Vec acc = pc->new_vec_with_width(v_proc_width, "acc"); // Reg. 615 Vec global_alpha = pc->new_vec_with_width(v_proc_width, "global_alpha"); // Mem. 616 Vec fill_rule_mask = pc->new_vec_with_width(v_proc_width, "fill_rule_mask");// Mem. 617 Vec vec_zero; // Reg/Tmp. 618 619 Pixel d_pix("d", pixel_type); // Reg. 620 621 VecArray m; // Reg. 622 VecArray comp_cov; // Tmp (only for passing coverages to the compositor). 623 init_vec_coverage(pc, m, PixelCount(max_pixels), VecWidthUtils::vec_width_of(acc), pc->vec_width(), pixel_type, coverage_format); 624 625 // Prolog 626 // ------ 627 628 // Initialize the destination. 629 pc->load_u32(y, mem_ptr(fill_data, BL_OFFSET_OF(FillData::Analytic, box.y0))); 630 pc->load(dst_stride, mem_ptr(ctx_data, BL_OFFSET_OF(ContextData, dst.stride))); 631 632 pc->mul(dst_ptr, y.clone_as(dst_ptr), dst_stride); 633 pc->add(dst_ptr, dst_ptr, mem_ptr(ctx_data, BL_OFFSET_OF(ContextData, dst.pixel_data))); 634 635 // Initialize cell pointers. 636 pc->load(bit_ptr_skip_len, mem_ptr(fill_data, BL_OFFSET_OF(FillData::Analytic, bit_stride))); 637 pc->load(cell_stride, mem_ptr(fill_data, BL_OFFSET_OF(FillData::Analytic, cell_stride))); 638 639 pc->load(bit_ptr, mem_ptr(fill_data, BL_OFFSET_OF(FillData::Analytic, bit_top_ptr))); 640 pc->load(cell_ptr, mem_ptr(fill_data, BL_OFFSET_OF(FillData::Analytic, cell_top_ptr))); 641 642 // Initialize pipeline parts. 643 dst_part()->init_ptr(dst_ptr); 644 comp_op_part()->init(fn, pc->_gp_none, y, uint32_t(pixel_granularity)); 645 646 // y = fill_data->box.y1 - fill_data->box.y0; 647 pc->sub(y, mem_ptr(fill_data, BL_OFFSET_OF(FillData::Analytic, box.y1)), y); 648 649 // Decompose the original `bit_stride` to bit_ptr_run_len + bit_ptr_skip_len, where: 650 // - `bit_ptr_run_len` - Number of BitWords (in byte units) active in this band. 651 // - `bit_ptr_run_skip` - Number of BitWords (in byte units) to skip for this band. 652 pc->shr(x_start, mem_ptr(fill_data, BL_OFFSET_OF(FillData::Analytic, box.x0)), pixels_per_bit_word_shift); 653 pc->load_u32(x_end, mem_ptr(fill_data, BL_OFFSET_OF(FillData::Analytic, box.x1))); 654 pc->shr(bit_ptr_run_len.r32(), x_end, pixels_per_bit_word_shift); 655 656 pc->sub(bit_ptr_run_len.r32(), bit_ptr_run_len.r32(), x_start); 657 pc->inc(bit_ptr_run_len.r32()); 658 pc->shl(bit_ptr_run_len, bit_ptr_run_len, IntOps::ctz(bw_size)); 659 pc->sub(bit_ptr_skip_len, bit_ptr_skip_len, bit_ptr_run_len); 660 661 // Make `x_start` to become the X offset of the first active BitWord. 662 pc->lea(bit_ptr, mem_ptr(bit_ptr, x_start.clone_as(bit_ptr), IntOps::ctz(bw_size))); 663 pc->shl(x_start, x_start, pixels_per_bit_word_shift); 664 665 // Initialize global alpha and fill-rule. 666 pc->v_broadcast_u16(global_alpha, mem_ptr(fill_data, BL_OFFSET_OF(FillData::Analytic, alpha))); 667 pc->v_broadcast_u32(fill_rule_mask, mem_ptr(fill_data, BL_OFFSET_OF(FillData::Analytic, fill_rule_mask))); 668 669 #if defined(BL_JIT_ARCH_X86) 670 vec_zero = pc->new_vec128("vec_zero"); 671 // We shift left by 7 bits so we can use [V]PMULHUW in `calc_masks_from_cells()` on X86 ISA. In order to make that 672 // work, we have to also shift `fill_rule_mask` left by 1, so the total shift left is 8, which is what we want for 673 // [V]PMULHUW. 674 pc->v_slli_i16(global_alpha, global_alpha, 7); 675 pc->v_slli_i16(fill_rule_mask, fill_rule_mask, 1); 676 #else 677 // In non-x86 case we want to keep zero in `vec_zero` - no need to clear it every time we want to clear memory. 678 vec_zero = pc->simd_vec_zero(acc); 679 #endif 680 681 pc->j(L_Scanline_Init); 682 683 // BitScan 684 // ------- 685 686 // Called by Scanline iterator on the first non-zero BitWord it matches. The responsibility of BitScan is to find 687 // the first bit in the passed BitWord followed by matching the bit that ends this match. This would essentially 688 // produce the first [x0, x1) span that has to be composited as 'VMask' loop. 689 690 pc->bind(L_BitScan_Init); // L_BitScan_Init: 691 692 count_zeros(x0.clone_as(bit_word), bit_word); // x0 = ctz(bit_word) or clz(bit_word); 693 pc->store_zero_reg(mem_ptr(bit_ptr, -bw_size)); // bit_ptr[-1] = 0; 694 pc->mov(bit_word_tmp, -1); // bit_word_tmp = -1; (all ones). 695 shift_mask(bit_word_tmp, bit_word_tmp, x0); // bit_word_tmp = bit_word_tmp << x0 or bit_word_tmp >> x0 696 697 // Convert bit offset `x0` into a pixel offset. We must consider `x_off` as it's only zero for the very first 698 // BitWord (all others are multiplies of `pixels_per_bit_word`). 699 pc->add_ext(x0, x_off, x0, 1 << pixels_per_one_bit_shift); // x0 = x_off + (x0 << pixels_per_one_bit_shift); 700 701 // Load the given cells to `m0` and clear the BitWord and all cells it represents in memory. This is important as 702 // the compositor has to clear the memory during composition. If this is a rare case where `x0` points at the end 703 // of the raster there is still one cell that is non-zero. This makes sure it's cleared. 704 705 pc->add_scaled(dst_ptr, x0.clone_as(dst_ptr), int(dst_bpp)); // dst_ptr += x0 * dst_bpp; 706 pc->add_scaled(cell_ptr, x0.clone_as(cell_ptr), 4); // cell_ptr += x0 * sizeof(uint32_t); 707 708 // Rare case - line rasterized at the end of the raster boundary. In 99% cases this is a clipped line that was 709 // rasterized as vertical-only line at the end of the render box. This is a completely valid case that produces 710 // nothing. 711 712 pc->j(L_Scanline_Done0, ucmp_ge(x0, x_end)); // if (x0 >= x_end) goto L_Scanline_Done0; 713 714 // Setup compositor and source/destination parts. This is required as the fetcher needs to know where to start. 715 // And since `start_at_x()` can only be called once per scanline we must do it here. 716 717 comp_op_part()->start_at_x(x0); // <CompOpPart::StartAtX> 718 719 if (max_pixels > 1) 720 comp_op_part()->prefetch_n(); // <CompOpPart::PrefetchN> 721 else if (pixel_granularity > 1) 722 comp_op_part()->src_part()->prefetch_n(); 723 724 pc->v_loada32(acc, pc->_get_mem_const(&ct.p_0002000000020000)); 725 726 // If `bit_word ^ bit_word_tmp` results in non-zero value it means that the current span ends within the same BitWord, 727 // otherwise the span crosses multiple BitWords. 728 729 pc->j(L_BitScan_Match, xor_nz(bit_word, bit_word_tmp)); // if ((bit_word ^= bit_word_tmp) != 0) goto L_BitScan_Match; 730 731 // Okay, so the span crosses multiple BitWords. Firstly we have to make sure this was not the last one. If that's 732 // the case we must terminate the scanning immediately. 733 734 pc->mov(i, bw_size_in_bits); // i = bw_size_in_bits; 735 pc->j(L_BitScan_End, cmp_eq(bit_ptr, bit_ptr_end)); // if (bit_ptr == bit_ptr_end) goto L_BitScan_End; 736 737 // A BitScan loop - iterates over all consecutive BitWords and finds those that don't have all bits set to 1. 738 739 pc->bind(L_BitScan_Iter); // L_BitScan_Iter: 740 pc->load(bit_word, mem_ptr(bit_ptr)); // bit_word = bit_ptr[0]; 741 pc->store_zero_reg(mem_ptr(bit_ptr)); // bit_ptr[0] = 0; 742 pc->add(x_off, x_off, pixels_per_bit_word); // x_off += pixels_per_bit_word; 743 pc->add(bit_ptr, bit_ptr, bw_size); // bit_ptr += bw_size; 744 pc->j(L_BitScan_Match, xor_nz(bit_word, -1)); // if ((bit_word ^= -1) != 0) goto L_BitScan_Match; 745 pc->j(L_BitScan_End, cmp_eq(bit_ptr, bit_ptr_end)); // if (bit_ptr == bit_ptr_end) goto L_BitScan_End; 746 pc->j(L_BitScan_Iter); // goto L_BitScan_Iter; 747 748 pc->bind(L_BitScan_Match); // L_BitScan_Match: 749 count_zeros(i.clone_as(bit_word), bit_word); // i = ctz(bit_word) or clz(bit_word); 750 751 pc->bind(L_BitScan_End); // L_BitScan_End: 752 753 #if defined(BL_JIT_ARCH_X86) 754 if (v_proc_pixel_count == 8) { 755 pc->v_add_i32(acc.v256(), acc.v256(), mem_ptr(cell_ptr)); // acc[7:0] += cell_ptr[7:0]; 756 } 757 else 758 #endif // BL_JIT_ARCH_X86 759 { 760 pc->v_add_i32(acc.v128(), acc.v128(), mem_ptr(cell_ptr)); // acc[3:0] += cell_ptr[3:0]; 761 } 762 763 pc->mov(bit_word_tmp, -1); // bit_word_tmp = -1; (all ones). 764 shift_mask(bit_word_tmp, bit_word_tmp, i); // bit_word_tmp = bit_word_tmp << i or bit_word_tmp >> i; 765 pc->shl(i, i, pixels_per_one_bit_shift); // i <<= pixels_per_one_bit_shift; 766 767 pc->xor_(bit_word, bit_word, bit_word_tmp); // bit_word ^= bit_word_tmp; 768 pc->add(i, i, x_off); // i += x_off; 769 770 // In cases where the raster width is not a multiply of `pixels_per_one_bit` we must make sure we won't overflow it. 771 772 pc->umin(i, i, x_end); // i = min(i, x_end); 773 #if defined(BL_JIT_ARCH_X86) 774 pc->v_zero_i(vec_zero); // vec_zero = 0; 775 #endif // BL_JIT_ARCH_X86 776 pc->v_storea128(mem_ptr(cell_ptr), vec_zero); // cell_ptr[3:0] = 0; 777 778 // `i` is now the number of pixels (and cells) to composite by using `v_mask`. 779 780 pc->sub(i, i, x0); // i -= x0; 781 pc->add(x0, x0, i); // x0 += i; 782 pc->j(L_VLoop_Init); // goto L_VLoop_Init; 783 784 // VMaskLoop - Main VMask Loop - 8 Pixels (256-bit SIMD) 785 // ----------------------------------------------------- 786 787 #if defined(BL_JIT_ARCH_X86) 788 if (v_proc_pixel_count == 8u) { 789 Label L_VLoop_Iter8 = pc->new_label(); 790 Label L_VLoop_End = pc->new_label(); 791 792 pc->bind(L_VLoop_Iter8); // L_VLoop_Iter8: 793 pc->v_extract_v128(acc, acc, 1); 794 795 pass_vec_coverage(comp_cov, m, PixelCount(8), pixel_type, coverage_format); 796 comp_op_part()->v_mask_proc_store_advance(dst_ptr, PixelCount(8), comp_cov, PixelCoverageFlags::kNone); 797 798 pc->add(cell_ptr, cell_ptr, 8 * 4); // cell_ptr += 8 * sizeof(uint32_t); 799 pc->v_add_i32(acc, acc, mem_ptr(cell_ptr)); // acc[7:0] += cell_ptr[7:0] 800 pc->v_zero_i(vec_zero); // vec_zero = 0; 801 pc->v_storeu256(mem_ptr(cell_ptr, -16), vec_zero.v256()); // cell_ptr[3:-4] = 0; 802 803 pc->bind(L_VLoop_Init); // L_VLoop_Init: 804 accumulate_coverages(acc); 805 calc_masks_from_cells(m[0], acc, fill_rule_mask, global_alpha); 806 normalize_coverages(acc); 807 expand_mask(m, PixelCount(8)); 808 809 pc->j(L_VLoop_Iter8, sub_nc(i, 8)); // if ((i -= 8) >= 0) goto L_VLoop_Iter8; 810 pc->j(L_VLoop_End, add_z(i, 8)); // if ((i += 8) == 0) goto L_VLoop_End; 811 pc->j(L_VTail_Init, ucmp_lt(i, 4)); // if (i < 4) goto L_VTail_Init; 812 813 pc->add(cell_ptr, cell_ptr, 4 * 4); // cell_ptr += 4 * sizeof(uint32_t); 814 pc->v_zero_i(vec_zero); // vec_zero = 0; 815 pc->v_storea128(mem_ptr(cell_ptr), vec_zero.v128()); // cell_ptr[3:0] = 0; 816 817 pass_vec_coverage(comp_cov, m, PixelCount(4), pixel_type, coverage_format); 818 comp_op_part()->v_mask_proc_store_advance(dst_ptr, PixelCount(4), comp_cov, PixelCoverageFlags::kImmutable); 819 if (pixel_type == PixelType::kRGBA32) { 820 if (m[0].is_vec512()) 821 pc->cc->vshufi32x4(m[0], m[0], m[0], x86::shuffle_imm(3, 2, 3, 2)); // m[0] = [a7 a7 a7 a7 a6 a6 a6 a6|a5 a5 a5 a5 a4 a4 a4 a4] 822 else 823 pc->v_mov(m[0], m[1]); // m[0] = [a7 a7 a7 a7 a6 a6 a6 a6|a5 a5 a5 a5 a4 a4 a4 a4] 824 } 825 else if (pixel_type == PixelType::kA8) { 826 pc->v_swizzle_u32x4(m[0], m[0], swizzle(3, 2, 3, 2)); // m[0] = [?? ?? ?? ?? ?? ?? ?? ??|a7 a6 a5 a4 a7 a6 a5 a4] 827 } 828 else { 829 BL_NOT_REACHED(); 830 } 831 832 pc->v_extract_v128(acc, acc, 1); 833 pc->j(L_VTail_Init, sub_nz(i, 4)); // if ((i -= 4) > 0) goto L_VTail_Init; 834 835 pc->bind(L_VLoop_End); // L_VLoop_End: 836 pc->v_extract_v128(acc, acc, 0); 837 pc->j(L_Scanline_Done1, ucmp_ge(x0, x_end)); // if (x0 >= x_end) goto L_Scanline_Done1; 838 } 839 else 840 #endif 841 842 // VMask Loop - Main VMask Loop - 4 Pixels 843 // --------------------------------------- 844 845 if (v_proc_pixel_count == 4u) { 846 Label L_VLoop_Cont = pc->new_label(); 847 848 pc->bind(L_VLoop_Cont); // L_VLoop_Cont: 849 850 pass_vec_coverage(comp_cov, m, PixelCount(4), pixel_type, coverage_format); 851 comp_op_part()->v_mask_proc_store_advance(dst_ptr, PixelCount(4), comp_cov, PixelCoverageFlags::kNone); 852 853 pc->add(cell_ptr, cell_ptr, 4 * 4); // cell_ptr += 4 * sizeof(uint32_t); 854 pc->v_add_i32(acc, acc, mem_ptr(cell_ptr)); // acc[3:0] += cell_ptr[3:0]; 855 #if defined(BL_JIT_ARCH_X86) 856 pc->v_zero_i(vec_zero); // vec_zero = 0; 857 #endif // BL_JIT_ARCH_X86 858 pc->v_storea128(mem_ptr(cell_ptr), vec_zero); // cell_ptr[3:0] = 0; 859 d_pix.reset_all_except_type_and_name(); 860 861 pc->bind(L_VLoop_Init); // L_VLoop_Init: 862 accumulate_coverages(acc); 863 calc_masks_from_cells(m[0], acc, fill_rule_mask, global_alpha); 864 normalize_coverages(acc); 865 expand_mask(m, PixelCount(4)); 866 867 pc->j(L_VLoop_Cont, sub_nc(i, 4)); // if ((i -= 4) >= 0) goto L_VLoop_Cont; 868 pc->j(L_VTail_Init, add_nz(i, 4)); // if ((i += 4) != 0) goto L_VTail_Init; 869 pc->j(L_Scanline_Done1, ucmp_ge(x0, x_end)); // if (x0 >= x_end) goto L_Scanline_Done1; 870 } 871 872 // VMask Loop - Main VMask Loop - 1 Pixel 873 // -------------------------------------- 874 875 else { 876 Label L_VLoop_Iter = pc->new_label(); 877 Label L_VLoop_Step = pc->new_label(); 878 879 Gp n = pc->new_gp32("n"); 880 881 pc->bind(L_VLoop_Iter); // L_VLoop_Iter: 882 pc->umin(n, i, 4); // n = umin(i, 4); 883 pc->sub(i, i, n); // i -= n; 884 pc->add_scaled(cell_ptr, n, 4); // cell_ptr += n * 4; 885 886 if (pixel_granularity >= 4) 887 comp_op_part()->enter_partial_mode(); // <CompOpPart::enter_partial_mode> 888 889 if (pixel_type == PixelType::kRGBA32) { 890 constexpr PixelFlags kPC_Immutable = PixelFlags::kPC | PixelFlags::kImmutable; 891 892 #if defined(BL_JIT_ARCH_X86) 893 if (!pc->has_avx2()) { 894 // Broadcasts were introduced by AVX2, so we generally don't want to use code that relies on them as they 895 // would expand to more than a single instruction. So instead of a broadcast, we pre-shift the input in a 896 // way so we can use a single [V]PSHUFLW to shuffle the components to places where the compositor needs them. 897 pc->v_sllb_u128(m[0], m[0], 6); // m0[7:0] = [__ a3 a2 a1 a0 __ __ __] 898 899 pc->bind(L_VLoop_Step); // L_VLoop_Step: 900 pc->v_swizzle_lo_u16x4(m[0], m[0], swizzle(3, 3, 3, 3)); // m0[7:0] = [__ a3 a2 a1 a0 a0 a0 a0] 901 902 comp_cov.init(m[0].v128()); 903 comp_op_part()->v_mask_proc_rgba32_vec(d_pix, PixelCount(1), kPC_Immutable, comp_cov, PixelCoverageFlags::kImmutable, pc->empty_predicate()); 904 } 905 else 906 #endif 907 { 908 Vec vm_tmp = pc->new_vec128("@vm_tmp"); 909 pc->bind(L_VLoop_Step); // L_VLoop_Step: 910 911 if (coverage_format == PixelCoverageFormat::kPacked) 912 pc->v_broadcast_u8(vm_tmp, m[0].v128()); // vm_tmp[15:0] = [a0 a0 a0 a0 a0 a0 a0 a0|a0 a0 a0 a0 a0 a0 a0 a0] 913 else 914 pc->v_broadcast_u16(vm_tmp, m[0].v128()); // vm_tmp[15:0] = [_0 a0 _0 a0 _0 a0 _0 a0|_0 a0 _0 a0 _0 a0 _0 a0] 915 916 comp_cov.init(vm_tmp); 917 comp_op_part()->v_mask_proc_rgba32_vec(d_pix, PixelCount(1), kPC_Immutable, comp_cov, PixelCoverageFlags::kNone, pc->empty_predicate()); 918 } 919 920 pc->xStorePixel(dst_ptr, d_pix.pc[0], 1, dst_bpp, Alignment(1)); 921 d_pix.reset_all_except_type_and_name(); 922 } 923 else if (pixel_type == PixelType::kA8) { 924 pc->bind(L_VLoop_Step); // L_VLoop_Step: 925 926 Gp msk = pc->new_gp32("@msk"); 927 pc->s_extract_u16(msk, m[0], 0); 928 929 comp_op_part()->v_mask_proc_a8_gp(d_pix, PixelFlags::kSA | PixelFlags::kImmutable, msk, PixelCoverageFlags::kNone); 930 931 pc->store_u8(mem_ptr(dst_ptr), d_pix.sa); 932 d_pix.reset_all_except_type_and_name(); 933 } 934 935 pc->add(dst_ptr, dst_ptr, dst_bpp); // dst_ptr += dst_bpp; 936 pc->shift_or_rotate_right(m[0], m[0], 2); // m0[15:0] = [??, m[15:2]] 937 938 if (pixel_granularity >= 4) // if (pixel_granularity >= 4) 939 comp_op_part()->next_partial_pixel(); // <CompOpPart::next_partial_pixel> 940 941 pc->j(L_VLoop_Step, sub_nz(n, 1)); // if (--n != 0) goto L_VLoop_Step; 942 943 if (pixel_granularity >= 4) // if (pixel_granularity >= 4) 944 comp_op_part()->exit_partial_mode(); // <CompOpPart::exit_partial_mode> 945 946 #if defined(BL_JIT_ARCH_X86) 947 if (!pc->has_avx()) { 948 // We must use unaligned loads here as we don't know whether we are at the end of the scanline. 949 // In that case `cell_ptr` could already be misaligned if the image width is not divisible by 4. 950 Vec cov_tmp = pc->new_vec128("@cov_tmp"); 951 pc->v_loadu128(cov_tmp, mem_ptr(cell_ptr)); // cov_tmp[3:0] = cell_ptr[3:0]; 952 pc->v_add_i32(acc, acc, cov_tmp); // acc[3:0] += cov_tmp 953 } 954 else 955 #endif // BL_JIT_ARCH_X86 956 { 957 pc->v_add_i32(acc, acc, mem_ptr(cell_ptr)); // acc[3:0] += cell_ptr[3:0] 958 } 959 960 #if defined(BL_JIT_ARCH_X86) 961 pc->v_zero_i(vec_zero); // vec_zero = 0; 962 #endif 963 pc->v_storeu128(mem_ptr(cell_ptr), vec_zero); // cell_ptr[3:0] = 0; 964 965 pc->bind(L_VLoop_Init); // L_VLoop_Init: 966 967 accumulate_coverages(acc); 968 calc_masks_from_cells(m[0], acc, fill_rule_mask, global_alpha); 969 normalize_coverages(acc); 970 971 pc->j(L_VLoop_Iter, test_nz(i)); // if (i != 0) goto L_VLoop_Iter; 972 pc->j(L_Scanline_Done1, ucmp_ge(x0, x_end)); // if (x0 >= x_end) goto L_Scanline_Done1; 973 } 974 975 // BitGap 976 // ------ 977 978 // If we are here we are at the end of `v_mask` loop. There are two possibilities: 979 // 980 // 1. There is a gap between bits in a single or multiple BitWords. This means that there is a possibility 981 // for a `c_mask` loop which could be fully opaque, semi-transparent, or fully transparent (a real gap). 982 // 983 // 2. This was the last span and there are no more bits in consecutive BitWords. We will not consider this as 984 // a special case and just process the remaining BitWords in a normal way (scanning until the end of the 985 // current scanline). 986 987 Label L_BitGap_Match = pc->new_label(); 988 Label L_BitGap_Cont = pc->new_label(); 989 990 pc->j(L_BitGap_Match, test_nz(bit_word)); // if (bit_word != 0) goto L_BitGap_Match; 991 992 // Loop unrolled 2x as we could be inside a larger span. 993 994 pc->bind(L_BitGap_Cont); // L_BitGap_Cont: 995 pc->add(x_off, x_off, pixels_per_bit_word); // x_off += pixels_per_bit_word; 996 pc->j(L_Scanline_Done1, cmp_eq(bit_ptr, bit_ptr_end)); // if (bit_ptr == bit_ptr_end) goto L_Scanline_Done1; 997 998 pc->load(bit_word, mem_ptr(bit_ptr)); // bit_word = bit_ptr[0]; 999 pc->add(bit_ptr, bit_ptr, bw_size); // bit_ptr += bw_size; 1000 pc->j(L_BitGap_Match, test_nz(bit_word)); // if (bit_word != 0) goto L_BitGap_Match; 1001 1002 pc->add(x_off, x_off, pixels_per_bit_word); // x_off += pixels_per_bit_word; 1003 pc->j(L_Scanline_Done1, cmp_eq(bit_ptr, bit_ptr_end)); // if (bit_ptr == bit_ptr_end) goto L_Scanline_Done1; 1004 1005 pc->load(bit_word, mem_ptr(bit_ptr)); // bit_word = bit_ptr[0]; 1006 pc->add(bit_ptr, bit_ptr, bw_size); // bit_ptr += bw_size; 1007 pc->j(L_BitGap_Cont, test_z(bit_word)); // if (bit_word == 0) goto L_BitGap_Cont; 1008 1009 pc->bind(L_BitGap_Match); // L_BitGap_Match: 1010 pc->store_zero_reg(mem_ptr(bit_ptr, -bw_size)); // bit_ptr[-1] = 0; 1011 count_zeros(i.clone_as(bit_word), bit_word); // i = ctz(bit_word) or clz(bit_word); 1012 pc->mov(bit_word_tmp, -1); // bit_word_tmp = -1; (all ones) 1013 1014 if (coverage_format == PixelCoverageFormat::kPacked) 1015 pc->s_extract_u8(c_mask_alpha, m[0], 0); // c_mask_alpha = s_extract_u8(m0, 0); 1016 else 1017 pc->s_extract_u16(c_mask_alpha, m[0], 0); // c_mask_alpha = s_extract_u16(m0, 0); 1018 1019 shift_mask(bit_word_tmp, bit_word_tmp, i); // bit_word_tmp = bit_word_tmp << i or bit_word_tmp >> i; 1020 pc->shl(i, i, imm(pixels_per_one_bit_shift)); // i <<= pixels_per_one_bit_shift; 1021 1022 pc->xor_(bit_word, bit_word, bit_word_tmp); // bit_word ^= bit_word_tmp; 1023 pc->add(i, i, x_off); // i += x_off; 1024 pc->sub(i, i, x0); // i -= x0; 1025 pc->add(x0, x0, i); // x0 += i; 1026 pc->add_scaled(cell_ptr, i.clone_as(cell_ptr), 4); // cell_ptr += i * sizeof(uint32_t); 1027 pc->j(L_CLoop_Init, test_nz(c_mask_alpha)); // if (c_mask_alpha != 0) goto L_CLoop_Init; 1028 1029 // Fully-Transparent span where `c_mask_alpha == 0`. 1030 1031 pc->add_scaled(dst_ptr, i.clone_as(dst_ptr), int(dst_bpp)); // dst_ptr += i * dst_bpp; 1032 1033 if (v_proc_pixel_count >= 4) 1034 comp_op_part()->postfetch_n(); 1035 1036 comp_op_part()->advance_x(x0, i); 1037 1038 if (v_proc_pixel_count >= 4) 1039 comp_op_part()->prefetch_n(); 1040 1041 pc->j(L_BitScan_Match, test_nz(bit_word)); // if (bit_word != 0) goto L_BitScan_Match; 1042 pc->j(L_BitScan_Iter); // goto L_BitScan_Iter; 1043 1044 // CMask - Loop 1045 // ------------ 1046 1047 pc->bind(L_CLoop_Init); // L_CLoop_Init: 1048 if (comp_op_part()->should_optimize_opaque_fill()) { 1049 Label L_CLoop_Msk = pc->new_label(); 1050 pc->j(L_CLoop_Msk, cmp_ne(c_mask_alpha, 255)); // if (c_mask_alpha != 255) goto L_CLoop_Msk 1051 1052 comp_op_part()->c_mask_init_opaque(); 1053 if (pixel_granularity >= 4) 1054 comp_op_part()->c_mask_granular_loop(i); 1055 else 1056 comp_op_part()->c_mask_generic_loop(i); 1057 comp_op_part()->c_mask_fini(); 1058 1059 pc->j(L_BitScan_Match, test_nz(bit_word)); // if (bit_word != 0) goto L_BitScan_Match; 1060 pc->j(L_BitScan_Iter); // goto L_BitScan_Iter; 1061 1062 pc->bind(L_CLoop_Msk); // L_CLoop_Msk: 1063 } 1064 1065 if (coverage_format == PixelCoverageFormat::kPacked) { 1066 pc->v_broadcast_u8(m[0], m[0]); // m0 = [a0 a0 a0 a0 a0 a0 a0 a0|a0 a0 a0 a0 a0 a0 a0 a0] 1067 } 1068 #if defined(BL_JIT_ARCH_X86) 1069 else if (!pc->has_avx2()) { 1070 pc->v_swizzle_u32x4(m[0], m[0], swizzle(0, 0, 0, 0)); // m0 = [_0 a0 _0 a0 _0 a0 _0 a0|_0 a0 _0 a0 _0 a0 _0 a0] 1071 } 1072 #endif 1073 else { 1074 pc->v_broadcast_u16(m[0], m[0]); // m0 = [_0 a0 _0 a0 _0 a0 _0 a0|_0 a0 _0 a0 _0 a0 _0 a0] 1075 } 1076 1077 comp_op_part()->c_mask_init(c_mask_alpha, m[0]); 1078 if (pixel_granularity >= 4) 1079 comp_op_part()->c_mask_granular_loop(i); 1080 else 1081 comp_op_part()->c_mask_generic_loop(i); 1082 comp_op_part()->c_mask_fini(); 1083 1084 pc->j(L_BitScan_Match, test_nz(bit_word)); // if (bit_word != 0) goto L_BitScan_Match; 1085 pc->j(L_BitScan_Iter); // goto L_BitScan_Iter; 1086 1087 // VMask - Tail - Tail `v_mask` loop for pixels near the end of the scanline 1088 // ------------------------------------------------------------------------ 1089 1090 if (max_pixels >= 4u) { 1091 Label L_VTail_Cont = pc->new_label(); 1092 1093 Vec m128 = m[0].v128(); 1094 VecArray msk(m128); 1095 1096 // Tail loop can handle up to `pixels_per_one_bit - 1`. 1097 if (pixel_type == PixelType::kRGBA32) { 1098 bool hasV256Mask = m[0].size() >= 32u; 1099 1100 pc->bind(L_VTail_Init); // L_VTail_Init: 1101 pc->add_scaled(cell_ptr, i, 4); // cell_ptr += i * sizeof(uint32_t); 1102 1103 if (coverage_format == PixelCoverageFormat::kUnpacked && !hasV256Mask) { 1104 pc->v_swap_u64(m[1], m[1]); 1105 } 1106 comp_op_part()->enter_partial_mode(); // <CompOpPart::enter_partial_mode> 1107 1108 pc->bind(L_VTail_Cont); // L_VTail_Cont: 1109 comp_op_part()->v_mask_proc_rgba32_vec(d_pix, PixelCount(1), PixelFlags::kPC | PixelFlags::kImmutable, msk, PixelCoverageFlags::kImmutable, pc->empty_predicate()); 1110 1111 pc->xStorePixel(dst_ptr, d_pix.pc[0], 1, dst_bpp, Alignment(1)); 1112 pc->add(dst_ptr, dst_ptr, dst_bpp); // dst_ptr += dst_bpp; 1113 1114 if (coverage_format == PixelCoverageFormat::kPacked) { 1115 pc->shift_or_rotate_right(m[0], m[0], 4); // m0[15:0] = [????, m[15:4]] 1116 } 1117 else { 1118 #if defined(BL_JIT_ARCH_X86) 1119 if (hasV256Mask) { 1120 // All 4 expanded masks for ARGB channels are in a single register, so just permute. 1121 pc->v_swizzle_u64x4(m[0], m[0], swizzle(0, 3, 2, 1)); 1122 } 1123 else 1124 #endif 1125 { 1126 pc->v_interleave_hi_u64(m[0], m[0], m[1]); 1127 } 1128 } 1129 1130 comp_op_part()->next_partial_pixel(); // <CompOpPart::next_partial_pixel> 1131 d_pix.reset_all_except_type_and_name(); 1132 pc->j(L_VTail_Cont, sub_nz(i, 1)); // if (--i) goto L_VTail_Cont; 1133 1134 comp_op_part()->exit_partial_mode(); // <CompOpPart::exit_partial_mode> 1135 } 1136 else if (pixel_type == PixelType::kA8) { 1137 Gp mScalar = pc->new_gp32("mScalar"); 1138 1139 pc->bind(L_VTail_Init); // L_VTail_Init: 1140 pc->add_scaled(cell_ptr, i, 4); // cell_ptr += i * sizeof(uint32_t); 1141 comp_op_part()->enter_partial_mode(); // <CompOpPart::enter_partial_mode> 1142 1143 pc->bind(L_VTail_Cont); // L_VTail_Cont: 1144 if (coverage_format == PixelCoverageFormat::kPacked) 1145 pc->s_extract_u8(mScalar, m128, 0); 1146 else 1147 pc->s_extract_u16(mScalar, m128, 0); 1148 comp_op_part()->v_mask_proc_a8_gp(d_pix, PixelFlags::kSA | PixelFlags::kImmutable, mScalar, PixelCoverageFlags::kNone); 1149 1150 pc->store_u8(mem_ptr(dst_ptr), d_pix.sa); 1151 pc->add(dst_ptr, dst_ptr, dst_bpp); // dst_ptr += dst_bpp; 1152 if (coverage_format == PixelCoverageFormat::kPacked) 1153 pc->shift_or_rotate_right(m128, m128, 1); // m0[15:0] = [?, m[15:1]] 1154 else 1155 pc->shift_or_rotate_right(m128, m128, 2); // m0[15:0] = [??, m[15:2]] 1156 comp_op_part()->next_partial_pixel(); // <CompOpPart::next_partial_pixel> 1157 d_pix.reset_all_except_type_and_name(); 1158 pc->j(L_VTail_Cont, sub_nz(i, 1)); // if (--i) goto L_VTail_Cont; 1159 1160 comp_op_part()->exit_partial_mode(); // <CompOpPart::exit_partial_mode> 1161 } 1162 1163 // Since this was a tail loop we know that there is nothing to be processed afterwards, because tail loop is only 1164 // possible at the end of the scanline boundary / clip region. 1165 } 1166 1167 // Scanline Iterator 1168 // ----------------- 1169 1170 // This loop is used to quickly test bit_words in `bit_ptr`. In some cases the whole scanline could be empty, so this 1171 // loop makes sure we won't enter more complicated loops if this happens. It's also used to quickly find the first 1172 // bit, which is non-zero - in that case it jumps directly to BitScan section. 1173 // 1174 // NOTE: Storing zeros to `cell_ptr` must be unaligned here as we may be at the end of the scanline. 1175 1176 pc->bind(L_Scanline_Done0); // L_Scanline_Done0: 1177 #if defined(BL_JIT_ARCH_X86) 1178 pc->v_zero_i(vec_zero); // vec_zero = 0; 1179 #endif // BL_JIT_ARCH_X86 1180 pc->v_storeu128(mem_ptr(cell_ptr), vec_zero); // cell_ptr[3:0] = 0; 1181 1182 pc->bind(L_Scanline_Done1); // L_Scanline_Done1: 1183 deadvance_dst_ptr_and_cell_ptr(dst_ptr, // dst_ptr -= x0 * dst_bpp; 1184 cell_ptr, x0, dst_bpp); // cell_ptr -= x0 * sizeof(uint32_t); 1185 pc->j(L_End, sub_z(y, 1)); // if (--y == 0) goto L_End; 1186 pc->mov(bit_ptr, bit_ptr_end); // bit_ptr = bit_ptr_end; 1187 1188 pc->bind(L_Scanline_AdvY); // L_Scanline_AdvY: 1189 pc->add(dst_ptr, dst_ptr, dst_stride); // dst_ptr += dst_stride; 1190 pc->add(bit_ptr, bit_ptr, bit_ptr_skip_len); // bit_ptr += bit_ptr_skip_len; 1191 pc->add(cell_ptr, cell_ptr, cell_stride); // cell_ptr += cell_stride; 1192 comp_op_part()->advance_y(); // <CompOpPart::AdvanceY> 1193 1194 pc->bind(L_Scanline_Init); // L_Scanline_Init: 1195 pc->mov(x_off, x_start); // x_off = x_start; 1196 pc->add(bit_ptr_end, bit_ptr, bit_ptr_run_len); // bit_ptr_end = bit_ptr + bit_ptr_run_len; 1197 1198 pc->bind(L_Scanline_Iter); // L_Scanline_Iter: 1199 pc->load(bit_word, mem_ptr(bit_ptr)); // bit_word = bit_ptr[0]; 1200 pc->add(bit_ptr, bit_ptr, bw_size); // bit_ptr += bw_size; 1201 pc->j(L_BitScan_Init, test_nz(bit_word)); // if (bit_word != 0) goto L_BitScan_Init; 1202 1203 pc->add(x_off, x_off, pixels_per_bit_word); // x_off += pixels_per_bit_word; 1204 pc->j(L_Scanline_Iter, cmp_ne(bit_ptr, bit_ptr_end)); // if (bit_ptr != bit_ptr_end) goto L_Scanline_Iter; 1205 pc->j(L_Scanline_AdvY, sub_nz(y, 1)); // if (--y) goto L_Scanline_AdvY; 1206 1207 // Epilog 1208 // ------ 1209 1210 pc->bind(L_End); 1211 comp_op_part()->fini(); 1212 _fini_global_hook(); 1213 } 1214 1215 void FillAnalyticPart::accumulate_coverages(const Vec& acc) noexcept { 1216 Vec tmp = pc->new_similar_reg<Vec>(acc, "vCovTmp"); 1217 1218 pc->v_sllb_u128(tmp, acc, 4); // tmp[7:0] = [ c6 c5 c4 0 | c2 c1 c0 0 ]; 1219 pc->v_add_i32(acc, acc, tmp); // acc[7:0] = [c7:c6 c6:c5 c5:c4 c4 |c3:c2 c2:c1 c1:c0 c0 ]; 1220 pc->v_sllb_u128(tmp, acc, 8); // tmp[7:0] = [c5:c4 c4 0 0 |c1:c0 c0 0 0 ]; 1221 pc->v_add_i32(acc, acc, tmp); // acc[7:0] = [c7:c4 c6:c4 c5:c4 c4 |c3:c0 c2:c0 c1:c0 c0 ]; 1222 1223 #if defined(BL_JIT_ARCH_X86) 1224 if (acc.is_vec256()) { 1225 pc->v_swizzle_u32x4(tmp.v128(), acc.v128(), swizzle(3, 3, 3, 3)); 1226 cc->vperm2i128(tmp, tmp, tmp, perm_2x128_imm(Perm2x128::kALo, Perm2x128::kZero)); 1227 pc->v_add_i32(acc, acc, tmp); // acc[7:0] = [c7:c0 c6:c0 c5:c0 c4:c0|c3:c0 c2:c0 c1:c0 c0 ]; 1228 } 1229 #endif // BL_JIT_ARCH_X86 1230 } 1231 1232 void FillAnalyticPart::normalize_coverages(const Vec& acc) noexcept { 1233 pc->v_srlb_u128(acc, acc, 12); // acc[3:0] = [ 0 0 0 c0 ]; 1234 } 1235 1236 // Calculate masks from cell and store them to a vector of the following layout: 1237 // 1238 // [__ __ __ __ a7 a6 a5 a4|__ __ __ __ a3 a2 a1 a0] 1239 // 1240 // NOTE: Depending on the vector size the output mask is for either 4 or 8 pixels. 1241 void FillAnalyticPart::calc_masks_from_cells(const Vec& msk_, const Vec& acc, const Vec& fill_rule_mask, const Vec& global_alpha) noexcept { 1242 Vec msk = msk_.clone_as(acc); 1243 1244 #if defined(BL_JIT_ARCH_X86) 1245 // This implementation is a bit tricky. In the original AGG and FreeType `A8_SHIFT + 1` is used. However, we don't do 1246 // that and mask out the last bit through `fill_rule_mask`. The reason we do this is that our `global_alpha` is already 1247 // pre-shifted by `7` bits left and we only need to shift the final mask by one bit left after it's been calculated. 1248 // So instead of shifting it left later we clear the LSB bit now and that's it, we saved one instruction. 1249 pc->v_srai_i32(msk, acc, A8Info::kShift); 1250 pc->v_and_i32(msk, msk, fill_rule_mask); 1251 1252 // We have to make sure that the cleared LSB bit stays zero. Since we only use SUB with even value and abs we are 1253 // fine. However, that packing would not be safe if there was no "v_min_i16", which makes sure we are always safe. 1254 Operand i_0x00000200 = pc->simd_const(&ct.p_0000020000000200, Bcst::k32, msk); 1255 pc->v_sub_i32(msk, msk, i_0x00000200); 1256 pc->v_abs_i32(msk, msk); 1257 1258 if (pc->has_sse4_1()) { 1259 // This is not really faster, but it uses the same constant as one of the previous operations, potentially saving 1260 // us a register. 1261 pc->v_min_u32(msk, msk, i_0x00000200); 1262 pc->v_packs_i32_i16(msk, msk, msk); 1263 } 1264 else { 1265 pc->v_packs_i32_i16(msk, msk, msk); 1266 pc->v_min_i16(msk, msk, pc->simd_const(&ct.p_0200020002000200, Bcst::kNA, msk)); 1267 } 1268 1269 // Multiply masks by global alpha, this would output masks in [0, 255] range. 1270 pc->v_mulh_u16(msk, msk, global_alpha); 1271 #else 1272 // This implementation doesn't need any tricks as a lot of SIMD primitives are just provided natively. 1273 pc->v_srai_i32(msk, acc, A8Info::kShift + 1); 1274 pc->v_and_i32(msk, msk, fill_rule_mask); 1275 1276 pc->v_sub_i32(msk, msk, pc->simd_const(&ct.p_0000010000000100, Bcst::k32, msk)); 1277 pc->v_abs_i32(msk, msk); 1278 pc->v_min_u32(msk, msk, pc->simd_const(&ct.p_0000010000000100, Bcst::kNA, msk)); 1279 1280 pc->v_mul_u16(msk, msk, global_alpha); 1281 pc->cc->shrn(msk.h4(), msk.s4(), 8); 1282 #endif 1283 } 1284 1285 void FillAnalyticPart::expand_mask(const VecArray& msk, PixelCount pixel_count) noexcept { 1286 PixelType pixel_type = comp_op_part()->pixel_type(); 1287 PixelCoverageFormat coverage_format = comp_op_part()->coverage_format(); 1288 1289 if (pixel_type == PixelType::kRGBA32) { 1290 switch (coverage_format) { 1291 #if defined(BL_JIT_ARCH_A64) 1292 case PixelCoverageFormat::kPacked: { 1293 uint32_t n_regs = (uint32_t(pixel_count) + 3u) / 4u; 1294 for (uint32_t i = 0; i < n_regs; i++) { 1295 Vec v = msk[i].v128(); 1296 pc->v_swizzlev_u8(v, v, pc->simd_const(&ct.swizu8_xxxxxxxxx3x2x1x0_to_3333222211110000, Bcst::kNA, v)); 1297 } 1298 return; 1299 } 1300 #endif // BL_JIT_ARCH_A64 1301 1302 case PixelCoverageFormat::kUnpacked: { 1303 if (pixel_count == PixelCount(4)) { 1304 Vec cov0_128 = msk[0].v128(); 1305 1306 pc->v_interleave_lo_u16(cov0_128, cov0_128, cov0_128); // msk[0] = [a3 a3 a2 a2 a1 a1 a0 a0] 1307 #if defined(BL_JIT_ARCH_X86) 1308 if (msk[0].is_vec256()) { 1309 pc->v_swizzle_u64x4(msk[0], msk[0], swizzle(1, 1, 0, 0)); // msk[0] = [a3 a3 a2 a2 a3 a3 a2 a2|a1 a1 a0 a0 a1 a1 a0 a0] 1310 pc->v_swizzle_u32x4(msk[0], msk[0], swizzle(1, 1, 0, 0)); // msk[0] = [a3 a3 a3 a3 a2 a2 a2 a2|a1 a1 a1 a1 a0 a0 a0 a0] 1311 } 1312 else 1313 #endif // BL_JIT_ARCH_X86 1314 { 1315 pc->v_swizzle_u32x4(msk[1], msk[0], swizzle(3, 3, 2, 2)); // msk[0] = [a3 a3 a3 a3 a2 a2 a2 a2] 1316 pc->v_swizzle_u32x4(msk[0], msk[0], swizzle(1, 1, 0, 0)); // msk[0] = [a1 a1 a1 a1 a0 a0 a0 a0] 1317 } 1318 1319 return; 1320 } 1321 1322 #if defined(BL_JIT_ARCH_X86) 1323 if (pixel_count == PixelCount(8)) { 1324 if (msk[0].is_vec512()) { 1325 if (pc->has_avx512_vbmi()) { 1326 Vec pred = pc->simd_vec_const(&ct.permu8_4xa8_lo_to_rgba32_uc, Bcst::kNA_Unique, msk[0]); 1327 pc->v_permute_u8(msk[0], pred, msk[0]); // msk[0] = [4x00a7 4x00a6 4x00a5 4x00a4|4x00a3 4x00a2 4x00a1 4x00a0] 1328 } 1329 else { 1330 Vec msk_256 = msk[0].v256(); 1331 Operand pred = pc->simd_const(&ct.swizu8_xxxxxxxxx3x2x1x0_to_3333222211110000, Bcst::kNA, msk_256); 1332 pc->v_swizzlev_u8(msk_256, msk_256, pred); // msk[0] = [2xa7a7 3xa6a6 3xa5a5 2xa4a4|2xa3a3 2xa2a2 2xa1a1 2xa0a0] 1333 pc->v_cvt_u8_lo_to_u16(msk[0], msk_256); // msk[0] = [4x00a7 4x00a6 4x00a5 4x00a4|4x00a3 4x00a2 4x00a1 4x00a0] 1334 } 1335 } 1336 else { 1337 // msk[0] = [__ __ __ __ a7 a6 a5 a4|__ __ __ __ a3 a2 a1 a0] 1338 pc->v_interleave_lo_u16(msk[0], msk[0], msk[0]); // msk[0] = [a7 a7 a6 a6 a5 a5 a4 a4|a3 a3 a2 a2 a1 a1 a0 a0] 1339 pc->v_swizzle_u64x4(msk[1], msk[0], swizzle(3, 3, 2, 2)); // msk[1] = [a7 a7 a6 a6 a7 a7 a6 a6|a5 a5 a4 a4 a5 a5 a4 a4] 1340 pc->v_swizzle_u64x4(msk[0], msk[0], swizzle(1, 1, 0, 0)); // msk[0] = [a3 a3 a2 a2 a3 a3 a2 a2|a1 a1 a0 a0 a1 a1 a0 a0] 1341 pc->v_interleave_lo_u32(msk[0], msk[0], msk[0]); // msk[0] = [a3 a3 a3 a3 a2 a2 a2 a2|a1 a1 a1 a1 a0 a0 a0 a0] 1342 pc->v_interleave_lo_u32(msk[1], msk[1], msk[1]); // msk[1] = [a7 a7 a7 a7 a6 a6 a6 a6|a5 a5 a5 a5 a4 a4 a4 a4] 1343 } 1344 return; 1345 } 1346 #endif // BL_JIT_ARCH_X86 1347 1348 break; 1349 } 1350 1351 default: 1352 BL_NOT_REACHED(); 1353 } 1354 } 1355 else if (pixel_type == PixelType::kA8) { 1356 switch (coverage_format) { 1357 case PixelCoverageFormat::kPacked: { 1358 if (pixel_count <= PixelCount(8)) { 1359 Vec v = msk[0].v128(); 1360 pc->v_packs_i16_u8(v, v, v); 1361 return; 1362 } 1363 1364 break; 1365 } 1366 1367 case PixelCoverageFormat::kUnpacked: { 1368 if (pixel_count <= PixelCount(4)) 1369 return; 1370 1371 #if defined(BL_JIT_ARCH_X86) 1372 // We have to convert from: 1373 // msk = [?? ?? ?? ?? a7 a6 a5 a4|?? ?? ?? ?? a3 a2 a1 a0] 1374 // To: 1375 // msk = [a7 a6 a5 a4 a3 a2 a1 a0|a7 a6 a5 a4 a3 a2 a1 a0] 1376 pc->v_swizzle_u64x4(msk[0].ymm(), msk[0].ymm(), swizzle(2, 0, 2, 0)); 1377 #endif // BL_JIT_ARCH_X86 1378 1379 return; 1380 } 1381 1382 default: 1383 BL_NOT_REACHED(); 1384 } 1385 } 1386 1387 BL_NOT_REACHED(); 1388 } 1389 1390 void FillAnalyticPart::deadvance_dst_ptr_and_cell_ptr(const Gp& dst_ptr, const Gp& cell_ptr, const Gp& x, uint32_t dst_bpp) noexcept { 1391 Gp x_adv = x.clone_as(dst_ptr); 1392 1393 #if defined(BL_JIT_ARCH_A64) 1394 pc->cc->sub(cell_ptr, cell_ptr, x_adv, a64::lsl(2)); 1395 if (asmjit::Support::is_power_of_2(dst_bpp)) { 1396 uint32_t shift = asmjit::Support::ctz(dst_bpp); 1397 pc->cc->sub(dst_ptr, dst_ptr, x_adv, a64::lsl(shift)); 1398 } 1399 else { 1400 pc->mul(x_adv, x_adv, dst_bpp); 1401 pc->sub(dst_ptr, dst_ptr, x_adv); 1402 } 1403 #else 1404 if (dst_bpp == 1) { 1405 pc->sub(dst_ptr, dst_ptr, x_adv); 1406 pc->shl(x_adv, x_adv, 2); 1407 pc->sub(cell_ptr, cell_ptr, x_adv); 1408 } 1409 else if (dst_bpp == 2) { 1410 pc->shl(x_adv, x_adv, 1); 1411 pc->sub(dst_ptr, dst_ptr, x_adv); 1412 pc->shl(x_adv, x_adv, 1); 1413 pc->sub(cell_ptr, cell_ptr, x_adv); 1414 } 1415 else if (dst_bpp == 4) { 1416 pc->shl(x_adv, x_adv, 2); 1417 pc->sub(dst_ptr, dst_ptr, x_adv); 1418 pc->sub(cell_ptr, cell_ptr, x_adv); 1419 } 1420 else { 1421 Gp dst_adv = pc->new_gpz("dst_adv"); 1422 pc->mul(dst_adv, x_adv, dst_bpp); 1423 pc->shl(x_adv, x_adv, 2); 1424 pc->sub(dst_ptr, dst_ptr, dst_adv); 1425 pc->sub(cell_ptr, cell_ptr, x_adv); 1426 } 1427 #endif 1428 } 1429 1430 } // {bl::Pipeline::JIT} 1431 1432 #endif // !BL_BUILD_NO_JIT