pixelconverter_avx2.cpp (25262B)
1 // This file is part of Blend2D project <https://blend2d.com> 2 // 3 // See blend2d.h or LICENSE.md for license and copyright information 4 // SPDX-License-Identifier: Zlib 5 6 #include "api-build_p.h" 7 #ifdef BL_TARGET_OPT_AVX2 8 9 #include "pixelconverter_p.h" 10 #include "simd/simd_p.h" 11 12 // PixelConverter - Copy (AVX2) 13 // ============================ 14 15 BLResult bl_convert_copy_avx2( 16 const BLPixelConverterCore* self, 17 uint8_t* dst_data, intptr_t dst_stride, 18 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 19 20 using namespace SIMD; 21 22 const size_t bytes_per_pixel = bl_pixel_converter_get_data(self)->mem_copy_data.bytes_per_pixel; 23 const size_t byte_width = size_t(w) * bytes_per_pixel; 24 25 // Use a generic copy if `byte_width` is small as we would not be able to 26 // utilize SIMD properly - in general we want to use at least 16-byte RW. 27 if (byte_width < 16) 28 return bl_convert_copy(self, dst_data, dst_stride, src_data, src_stride, w, h, options); 29 30 if (!options) 31 options = &bl_pixel_converter_default_options; 32 33 const size_t gap = options->gap; 34 dst_stride -= intptr_t(byte_width + gap); 35 src_stride -= intptr_t(byte_width); 36 37 for (uint32_t y = h; y != 0; y--) { 38 size_t i = byte_width; 39 40 BL_NOUNROLL 41 while (i >= 64) { 42 Vec32xU8 p0 = loadu<Vec32xU8>(src_data + 0); 43 Vec32xU8 p1 = loadu<Vec32xU8>(src_data + 32); 44 45 storeu(dst_data + 0, p0); 46 storeu(dst_data + 32, p1); 47 48 dst_data += 64; 49 src_data += 64; 50 i -= 64; 51 } 52 53 BL_NOUNROLL 54 while (i >= 16) { 55 storeu(dst_data, loadu<Vec16xU8>(src_data)); 56 57 dst_data += 16; 58 src_data += 16; 59 i -= 16; 60 } 61 62 if (i) { 63 dst_data += i; 64 src_data += i; 65 storeu(dst_data - 16, loadu<Vec16xU8>(src_data - 16)); 66 } 67 68 dst_data = bl_pixel_converter_fill_gap(dst_data, gap); 69 dst_data += dst_stride; 70 src_data += src_stride; 71 } 72 73 return BL_SUCCESS; 74 } 75 76 // PixelConverter - Copy|Or (AVX2) 77 // =============================== 78 79 BLResult bl_convert_copy_or_8888_avx2( 80 const BLPixelConverterCore* self, 81 uint8_t* dst_data, intptr_t dst_stride, 82 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 83 84 using namespace SIMD; 85 86 if (!options) { 87 options = &bl_pixel_converter_default_options; 88 } 89 90 const size_t gap = options->gap; 91 dst_stride -= uintptr_t(w) * 4 + gap; 92 src_stride -= uintptr_t(w) * 4; 93 94 Vec32xU8 fill_mask = make256_u32<Vec32xU8>(bl_pixel_converter_get_data(self)->mem_copy_data.fill_mask); 95 Vec32xU8 load_store_mask = loada_64_i8_i32<Vec32xU8>(bl::common_table.loadstore16_lo8_msk8() + (w & 7u)); 96 97 for (uint32_t y = h; y != 0; y--) { 98 uint32_t i = w; 99 100 BL_NOUNROLL 101 while (i >= 32) { 102 Vec32xU8 p0 = fill_mask | loadu<Vec32xU8>(src_data + 0); 103 Vec32xU8 p1 = fill_mask | loadu<Vec32xU8>(src_data + 32); 104 Vec32xU8 p2 = fill_mask | loadu<Vec32xU8>(src_data + 64); 105 Vec32xU8 p3 = fill_mask | loadu<Vec32xU8>(src_data + 96); 106 107 storeu(dst_data + 0, p0); 108 storeu(dst_data + 32, p1); 109 storeu(dst_data + 64, p2); 110 storeu(dst_data + 96, p3); 111 112 dst_data += 128; 113 src_data += 128; 114 i -= 32; 115 } 116 117 BL_NOUNROLL 118 while (i >= 8) { 119 Vec32xU8 p0 = fill_mask | loadu<Vec32xU8>(src_data); 120 storeu(dst_data, p0); 121 122 dst_data += 32; 123 src_data += 32; 124 i -= 8; 125 } 126 127 if (i) { 128 Vec32xU8 p0 = fill_mask | loadu_256_mask32<Vec32xU8>(src_data, load_store_mask); 129 storeu_256_mask32(dst_data, p0, load_store_mask); 130 131 dst_data += i * 4; 132 src_data += i * 4; 133 } 134 135 dst_data = bl_pixel_converter_fill_gap(dst_data, gap); 136 dst_data += dst_stride; 137 src_data += src_stride; 138 } 139 140 return BL_SUCCESS; 141 } 142 143 // PixelConverter - Copy|Shufb (AVX2) 144 // ================================== 145 146 BLResult bl_convert_copy_shufb_8888_avx2( 147 const BLPixelConverterCore* self, 148 uint8_t* dst_data, intptr_t dst_stride, 149 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 150 151 using namespace SIMD; 152 153 if (!options) { 154 options = &bl_pixel_converter_default_options; 155 } 156 157 const size_t gap = options->gap; 158 dst_stride -= uintptr_t(w) * 4 + gap; 159 src_stride -= uintptr_t(w) * 4; 160 161 const BLPixelConverterData::ShufbData& d = bl_pixel_converter_get_data(self)->shufb_data; 162 163 Vec32xU8 fill_mask = make256_u32<Vec32xU8>(bl_pixel_converter_get_data(self)->mem_copy_data.fill_mask); 164 Vec32xU8 predicate = broadcast_i128<Vec32xU8>(loadu<Vec16xU8>(d.shufb_predicate)); 165 Vec32xU8 load_store_mask = loada_64_i8_i32<Vec32xU8>(bl::common_table.loadstore16_lo8_msk8() + (w & 7u)); 166 167 for (uint32_t y = h; y != 0; y--) { 168 uint32_t i = w; 169 170 BL_NOUNROLL 171 while (i >= 32) { 172 Vec32xU8 p0 = loadu<Vec32xU8>(src_data + 0); 173 Vec32xU8 p1 = loadu<Vec32xU8>(src_data + 32); 174 Vec32xU8 p2 = loadu<Vec32xU8>(src_data + 64); 175 Vec32xU8 p3 = loadu<Vec32xU8>(src_data + 96); 176 177 storeu(dst_data + 0, swizzlev_u8(p0, predicate) | fill_mask); 178 storeu(dst_data + 32, swizzlev_u8(p1, predicate) | fill_mask); 179 storeu(dst_data + 64, swizzlev_u8(p2, predicate) | fill_mask); 180 storeu(dst_data + 96, swizzlev_u8(p3, predicate) | fill_mask); 181 182 dst_data += 128; 183 src_data += 128; 184 i -= 32; 185 } 186 187 BL_NOUNROLL 188 while (i >= 8) { 189 Vec32xU8 p0 = loadu<Vec32xU8>(src_data); 190 storeu(dst_data, swizzlev_u8(p0, predicate) | fill_mask); 191 192 dst_data += 32; 193 src_data += 32; 194 i -= 8; 195 } 196 197 if (i) { 198 Vec32xU8 p0 = loadu_256_mask32<Vec32xU8>(src_data, load_store_mask); 199 storeu_256_mask32(dst_data, swizzlev_u8(p0, predicate) | fill_mask, load_store_mask); 200 201 dst_data += i * 4; 202 src_data += i * 4; 203 } 204 205 dst_data = bl_pixel_converter_fill_gap(dst_data, gap); 206 dst_data += dst_stride; 207 src_data += src_stride; 208 } 209 210 return BL_SUCCESS; 211 } 212 213 // PixelConverter - RGB32 <- RGB24 (AVX2) 214 // ====================================== 215 216 BLResult bl_convert_rgb32_from_rgb24_shufb_avx2( 217 const BLPixelConverterCore* self, 218 uint8_t* dst_data, intptr_t dst_stride, 219 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 220 221 using namespace SIMD; 222 223 if (!options) 224 options = &bl_pixel_converter_default_options; 225 226 const size_t gap = options->gap; 227 dst_stride -= uintptr_t(w) * 4 + gap; 228 src_stride -= uintptr_t(w) * 3; 229 230 const BLPixelConverterData::ShufbData& d = bl_pixel_converter_get_data(self)->shufb_data; 231 232 Vec32xU8 fill_mask = make256_u32<Vec32xU8>(bl_pixel_converter_get_data(self)->mem_copy_data.fill_mask); 233 Vec32xU8 predicate = broadcast_i128<Vec32xU8>(loadu<Vec16xU8>(d.shufb_predicate)); 234 Vec16xU8 load_store_mask = loada_32_i8_i32<Vec16xU8>(bl::common_table.loadstore16_lo8_msk8() + (w & 3u)); 235 236 for (uint32_t y = h; y != 0; y--) { 237 uint32_t i = w; 238 239 BL_NOUNROLL 240 while (i >= 32) { 241 Vec32xU8 p0, p1, p2, p3; 242 Vec32xU8 q0, q1, q2, q3; 243 244 p0 = loadu_128<Vec32xU8>(src_data + 0); // [x5|z4 y4 x4|z3 y3 x3 z2|y2 x2 z1 y1|x1 z0 y0 x0] 245 p1 = loadu_128<Vec32xU8>(src_data + 16); // [yA|xA|z9 y9|x9 z8 y8 x8|z7 y7 x7 z6|y6 x6 z5 y5] 246 p3 = loadu_128<Vec32xU8>(src_data + 32); // [zF yF xF zE|yE xE zD yD|xD zC yC xC|zB yB xB zA] 247 248 p2 = alignr_u128<8>(p3, p1); // [-- -- -- --|zB yB xB zA|yA|xA|z9 y9|x9 z8 y8 x8] 249 p1 = alignr_u128<12>(p1, p0); // [-- -- -- --|z7 y7 x7 z6|y6 x6 z5 y5|x5|z4 y4 x4] 250 p3 = srlb_u128<4>(p3); // [-- -- -- --|zF yF xF zE|yE xE zD yD|xD zC yC xC] 251 252 p0 = interleave_i128(p0, p1); 253 p2 = interleave_i128(p2, p3); 254 255 q0 = loadu_128<Vec32xU8>(src_data + 48); // [x5|z4 y4 x4|z3 y3 x3 z2|y2 x2 z1 y1|x1 z0 y0 x0] 256 q1 = loadu_128<Vec32xU8>(src_data + 64); // [yA|xA|z9 y9|x9 z8 y8 x8|z7 y7 x7 z6|y6 x6 z5 y5] 257 q3 = loadu_128<Vec32xU8>(src_data + 80); // [zF yF xF zE|yE xE zD yD|xD zC yC xC|zB yB xB zA] 258 259 q2 = alignr_u128<8>(q3, q1); // [-- -- -- --|zB yB xB zA|yA|xA|z9 y9|x9 z8 y8 x8] 260 q1 = alignr_u128<12>(q1, q0); // [-- -- -- --|z7 y7 x7 z6|y6 x6 z5 y5|x5|z4 y4 x4] 261 q3 = srlb_u128<4>(q3); // [-- -- -- --|zF yF xF zE|yE xE zD yD|xD zC yC xC] 262 263 q0 = interleave_i128(q0, q1); 264 q2 = interleave_i128(q2, q3); 265 266 storeu(dst_data + 0, swizzlev_u8(p0, predicate) | fill_mask); 267 storeu(dst_data + 32, swizzlev_u8(p2, predicate) | fill_mask); 268 storeu(dst_data + 64, swizzlev_u8(q0, predicate) | fill_mask); 269 storeu(dst_data + 96, swizzlev_u8(q2, predicate) | fill_mask); 270 271 dst_data += 128; 272 src_data += 96; 273 i -= 32; 274 } 275 276 BL_NOUNROLL 277 while (i >= 8) { 278 Vec16xU8 p0, p1; 279 280 p0 = loadu<Vec16xU8>(src_data); // [x5|z4 y4 x4|z3 y3 x3 z2|y2 x2 z1 y1|x1 z0 y0 x0] 281 p1 = loadu_64<Vec16xU8>(src_data + 16); // [-- -- -- --|-- -- -- --|z7 y7 x7 z6|y6 x6 z5 y5] 282 p1 = alignr_u128<12>(p1, p0); // [-- -- -- --|z7 y7 x7 z6|y6 x6 z5 y5|x5|z4 y4 x4] 283 284 storeu(dst_data + 0, swizzlev_u8(p0, vec_128(predicate)) | vec_128(fill_mask)); 285 storeu(dst_data + 16, swizzlev_u8(p1, vec_128(predicate)) | vec_128(fill_mask)); 286 287 dst_data += 32; 288 src_data += 24; 289 i -= 8; 290 } 291 292 if (i >= 4) { 293 Vec16xU8 p0; 294 295 p0 = loadu_64<Vec16xU8>(src_data); // [-- -- -- --|-- -- -- --|y2 x2 z1 y1|x1 z0 y0 x0] 296 p0 = insert_m32<2>(p0, src_data + 8); // [-- -- -- --|z3 y3 x3 z2|y2 x2 z1 y1|x1 z0 y0 x0] 297 298 storeu(dst_data, swizzlev_u8(p0, vec_128(predicate)) | vec_128(fill_mask)); 299 300 dst_data += 16; 301 src_data += 12; 302 i -= 4; 303 } 304 305 if (i) { 306 Vec16xU8 p0 = make_zero<Vec16xU8>(); 307 p0 = insert_m24<0>(p0, src_data + 0); // [-- -- -- --|-- -- -- --|-- -- -- --|-- z0 y0 x0] 308 if (i >= 2) { 309 p0 = insert_m24<3>(p0, src_data + 3); // [-- -- -- --|-- -- -- --|-- -- z1 y1|x1 z0 y0 x0] 310 if (i >= 3) { 311 p0 = insert_m24<6>(p0, src_data + 6); // [-- -- -- --|-- -- -- z2|y2 x2 z1 y1|x1 z0 y0 x0] 312 } 313 } 314 315 storeu_128_mask32(dst_data, swizzlev_u8(p0, vec_128(predicate)) | vec_128(fill_mask), load_store_mask); 316 317 dst_data += i * 4; 318 src_data += i * 3; 319 } 320 321 dst_data = bl_pixel_converter_fill_gap(dst_data, gap); 322 dst_data += dst_stride; 323 src_data += src_stride; 324 } 325 326 return BL_SUCCESS; 327 } 328 329 // PixelConverter - Premultiply (AVX2) 330 // =================================== 331 332 template<uint32_t A_Shift, bool UseShufB> 333 static BL_INLINE BLResult bl_convert_premultiply_8888_template_avx2( 334 const BLPixelConverterCore* self, 335 uint8_t* dst_data, intptr_t dst_stride, 336 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 337 338 using namespace SIMD; 339 340 if (!options) 341 options = &bl_pixel_converter_default_options; 342 343 const size_t gap = options->gap; 344 dst_stride -= uintptr_t(w) * 4 + gap; 345 src_stride -= uintptr_t(w) * 4; 346 347 const BLPixelConverterData::PremultiplyData& d = bl_pixel_converter_get_data(self)->premultiply_data; 348 349 Vec32xU8 zero = make_zero<Vec32xU8>(); 350 Vec32xU8 fill_mask = make256_u32<Vec32xU8>(d.fill_mask); 351 Vec16xU16 alpha_mask = make256_u64<Vec16xU16>(uint64_t(0xFFu) << (A_Shift * 2)); 352 353 Vec32xU8 predicate; 354 if (UseShufB) 355 predicate = broadcast_i128<Vec32xU8>(loadu<Vec16xU8>(d.shufb_predicate)); 356 357 Vec32xU8 load_store_mask_lo = loada_64_i8_i32<Vec32xU8>(&bl::common_table.loadstore16_lo8_msk8()[w & 15]); 358 Vec32xU8 load_store_mask_hi = loada_64_i8_i32<Vec32xU8>(&bl::common_table.loadstore16_hi8_msk8()[w & 15]); 359 360 // Alpha byte-index that can be used by instructions that perform shuffling. 361 constexpr uint32_t AI = A_Shift / 8u; 362 363 for (uint32_t y = h; y != 0; y--) { 364 uint32_t i = w; 365 366 BL_NOUNROLL 367 while (i >= 16) { 368 Vec32xU8 packed0 = loadu<Vec32xU8>(src_data + 0); 369 Vec32xU8 packed1 = loadu<Vec32xU8>(src_data + 32); 370 371 if (UseShufB) { 372 packed0 = swizzlev_u8(packed0, predicate); 373 packed1 = swizzlev_u8(packed1, predicate); 374 } 375 376 Vec16xU16 p1 = vec_u16(interleave_hi_u8(packed0, zero)); 377 Vec16xU16 p0 = vec_u16(interleave_lo_u8(packed0, zero)); 378 Vec16xU16 p3 = vec_u16(interleave_hi_u8(packed1, zero)); 379 Vec16xU16 p2 = vec_u16(interleave_lo_u8(packed1, zero)); 380 381 p0 = div255_u16((p0 | alpha_mask) * swizzle_u16<AI, AI, AI, AI>(p0)); 382 p1 = div255_u16((p1 | alpha_mask) * swizzle_u16<AI, AI, AI, AI>(p1)); 383 p2 = div255_u16((p2 | alpha_mask) * swizzle_u16<AI, AI, AI, AI>(p2)); 384 p3 = div255_u16((p3 | alpha_mask) * swizzle_u16<AI, AI, AI, AI>(p3)); 385 386 storeu(dst_data + 0, vec_u8(packs_128_i16_u8(p0, p1)) | fill_mask); 387 storeu(dst_data + 32, vec_u8(packs_128_i16_u8(p2, p3)) | fill_mask); 388 389 dst_data += 64; 390 src_data += 64; 391 i -= 16; 392 } 393 394 if (i) { 395 Vec32xU8 packed0 = loadu_256_mask32<Vec32xU8>(src_data + 0, load_store_mask_lo); 396 Vec32xU8 packed1 = loadu_256_mask32<Vec32xU8>(src_data + 32, load_store_mask_hi); 397 398 if (UseShufB) { 399 packed0 = swizzlev_u8(packed0, predicate); 400 packed1 = swizzlev_u8(packed1, predicate); 401 } 402 403 Vec16xU16 p1 = vec_u16(interleave_hi_u8(packed0, zero)); 404 Vec16xU16 p0 = vec_u16(interleave_lo_u8(packed0, zero)); 405 Vec16xU16 p3 = vec_u16(interleave_hi_u8(packed1, zero)); 406 Vec16xU16 p2 = vec_u16(interleave_lo_u8(packed1, zero)); 407 408 p0 = div255_u16((p0 | alpha_mask) * swizzle_u16<AI, AI, AI, AI>(p0)); 409 p1 = div255_u16((p1 | alpha_mask) * swizzle_u16<AI, AI, AI, AI>(p1)); 410 p2 = div255_u16((p2 | alpha_mask) * swizzle_u16<AI, AI, AI, AI>(p2)); 411 p3 = div255_u16((p3 | alpha_mask) * swizzle_u16<AI, AI, AI, AI>(p3)); 412 413 storeu_256_mask32(dst_data + 0, vec_u8(packs_128_i16_u8(p0, p1)) | fill_mask, load_store_mask_lo); 414 storeu_256_mask32(dst_data + 32, vec_u8(packs_128_i16_u8(p2, p3)) | fill_mask, load_store_mask_hi); 415 416 dst_data += i * 4; 417 src_data += i * 4; 418 } 419 420 dst_data = bl_pixel_converter_fill_gap(dst_data, gap); 421 dst_data += dst_stride; 422 src_data += src_stride; 423 } 424 425 return BL_SUCCESS; 426 } 427 428 BLResult bl_convert_premultiply_8888_leading_alpha_avx2( 429 const BLPixelConverterCore* self, 430 uint8_t* dst_data, intptr_t dst_stride, 431 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 432 433 return bl_convert_premultiply_8888_template_avx2<24, false>(self, dst_data, dst_stride, src_data, src_stride, w, h, options); 434 } 435 436 BLResult bl_convert_premultiply_8888_trailing_alpha_avx2( 437 const BLPixelConverterCore* self, 438 uint8_t* dst_data, intptr_t dst_stride, 439 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 440 441 return bl_convert_premultiply_8888_template_avx2<0, false>(self, dst_data, dst_stride, src_data, src_stride, w, h, options); 442 } 443 444 BLResult bl_convert_premultiply_8888_leading_alpha_shufb_avx2( 445 const BLPixelConverterCore* self, 446 uint8_t* dst_data, intptr_t dst_stride, 447 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 448 449 return bl_convert_premultiply_8888_template_avx2<24, true>(self, dst_data, dst_stride, src_data, src_stride, w, h, options); 450 } 451 452 BLResult bl_convert_premultiply_8888_trailing_alpha_shufb_avx2( 453 const BLPixelConverterCore* self, 454 uint8_t* dst_data, intptr_t dst_stride, 455 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 456 457 return bl_convert_premultiply_8888_template_avx2<0, true>(self, dst_data, dst_stride, src_data, src_stride, w, h, options); 458 } 459 460 // PixelConverter - Unpremultiply (PMULLD) (AVX2) 461 // ============================================== 462 463 template<uint32_t A_Shift> 464 static BL_INLINE BLResult bl_convert_unpremultiply_8888_pmulld_template_avx2( 465 const BLPixelConverterCore* self, 466 uint8_t* dst_data, intptr_t dst_stride, 467 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 468 469 using namespace SIMD; 470 471 bl_unused(self); 472 473 if (!options) 474 options = &bl_pixel_converter_default_options; 475 476 const size_t gap = options->gap; 477 dst_stride -= uintptr_t(w) * 4u + gap; 478 src_stride -= uintptr_t(w) * 4u; 479 480 const uint32_t* rcp_table = bl::common_table.unpremultiply_rcp; 481 482 Vec8xU32 half = make256_u32(0x8000u); 483 Vec32xU8 alpha_mask = make256_u32<Vec32xU8>(0xFFu << A_Shift); 484 Vec8xU32 component_mask = make256_u32<Vec8xU32>(0xFFu); 485 Vec32xU8 load_store_mask = loada_64_i8_i32<Vec32xU8>(bl::common_table.loadstore16_lo8_msk8() + (w & 7)); 486 487 // Alpha byte-index that can be used by instructions that perform shuffling. 488 constexpr uint32_t AI = A_Shift / 8u; 489 constexpr uint32_t RI = (AI + 1) % 4; 490 constexpr uint32_t GI = (AI + 2) % 4; 491 constexpr uint32_t BI = (AI + 3) % 4; 492 493 for (uint32_t y = h; y != 0; y--) { 494 uint32_t i = w; 495 496 BL_NOUNROLL 497 while (i >= 8) { 498 Vec32xU8 pix = loadu<Vec32xU8>(src_data); 499 500 Vec4xU32 rcp_lo = loada_32<Vec4xU32>(rcp_table + src_data[0 * 4 + AI]); 501 Vec4xU32 rcp_hi = loada_32<Vec4xU32>(rcp_table + src_data[4 * 4 + AI]); 502 503 rcp_lo = insert_m32<1>(rcp_lo, rcp_table + src_data[1 * 4 + AI]); 504 rcp_hi = insert_m32<1>(rcp_hi, rcp_table + src_data[5 * 4 + AI]); 505 506 rcp_lo = insert_m32<2>(rcp_lo, rcp_table + src_data[2 * 4 + AI]); 507 rcp_hi = insert_m32<2>(rcp_hi, rcp_table + src_data[6 * 4 + AI]); 508 509 rcp_lo = insert_m32<3>(rcp_lo, rcp_table + src_data[3 * 4 + AI]); 510 rcp_hi = insert_m32<3>(rcp_hi, rcp_table + src_data[7 * 4 + AI]); 511 512 Vec8xU32 rcp = interleave_i128<Vec8xU32>(rcp_lo, rcp_hi); 513 Vec8xU32 pr = vec_u32(srli_u32<RI * 8>(pix)); 514 Vec8xU32 pg = vec_u32(srli_u32<GI * 8>(pix)); 515 Vec8xU32 pb = vec_u32(srli_u32<BI * 8>(pix)); 516 517 if (RI != 3) pr = pr & component_mask; 518 if (GI != 3) pg = pg & component_mask; 519 if (BI != 3) pb = pb & component_mask; 520 521 pix = pix & alpha_mask; 522 pr = slli_i32<RI * 8>(srli_u32<16>(pr * rcp + half)); 523 pg = slli_i32<GI * 8>(srli_u32<16>(pg * rcp + half)); 524 pb = slli_i32<BI * 8>(srli_u32<16>(pb * rcp + half)); 525 pix = pix | vec_u8(pr) | vec_u8(pg) | vec_u8(pb); 526 storeu(dst_data, pix); 527 528 dst_data += 32; 529 src_data += 32; 530 i -= 8; 531 } 532 533 if (i) { 534 Vec32xU8 pix = loadu_256_mask32<Vec32xU8>(src_data, load_store_mask); 535 536 Vec4xU32 rcp_lo = loada_32<Vec4xU32>(rcp_table + src_data[0 * 4 + AI]); 537 Vec4xU32 rcp_hi = loada_32<Vec4xU32>(rcp_table + src_data[4 * 4 + AI]); 538 539 rcp_lo = insert_m32<1>(rcp_lo, rcp_table + src_data[1 * 4 + AI]); 540 rcp_hi = insert_m32<1>(rcp_hi, rcp_table + src_data[5 * 4 + AI]); 541 542 rcp_lo = insert_m32<2>(rcp_lo, rcp_table + src_data[2 * 4 + AI]); 543 rcp_hi = insert_m32<2>(rcp_hi, rcp_table + src_data[6 * 4 + AI]); 544 545 rcp_lo = insert_m32<3>(rcp_lo, rcp_table + src_data[3 * 4 + AI]); 546 rcp_hi = insert_m32<3>(rcp_hi, rcp_table + src_data[7 * 4 + AI]); 547 548 Vec8xU32 rcp = interleave_i128<Vec8xU32>(rcp_lo, rcp_hi); 549 Vec8xU32 pr = vec_u32(srli_u32<RI * 8>(pix)); 550 Vec8xU32 pg = vec_u32(srli_u32<GI * 8>(pix)); 551 Vec8xU32 pb = vec_u32(srli_u32<BI * 8>(pix)); 552 553 if (RI != 3) pr = pr & component_mask; 554 if (GI != 3) pg = pg & component_mask; 555 if (BI != 3) pb = pb & component_mask; 556 557 pix = pix & alpha_mask; 558 pr = slli_i32<RI * 8>(srli_u32<16>(pr * rcp + half)); 559 pg = slli_i32<GI * 8>(srli_u32<16>(pg * rcp + half)); 560 pb = slli_i32<BI * 8>(srli_u32<16>(pb * rcp + half)); 561 pix = pix | vec_u8(pr) | vec_u8(pg) | vec_u8(pb); 562 storeu_256_mask32(dst_data, pix, load_store_mask); 563 564 dst_data += i * 4; 565 src_data += i * 4; 566 } 567 568 dst_data = bl_pixel_converter_fill_gap(dst_data, gap); 569 dst_data += dst_stride; 570 src_data += src_stride; 571 } 572 573 return BL_SUCCESS; 574 } 575 576 BLResult bl_convert_unpremultiply_8888_leading_alpha_pmulld_avx2( 577 const BLPixelConverterCore* self, 578 uint8_t* dst_data, intptr_t dst_stride, 579 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 580 581 return bl_convert_unpremultiply_8888_pmulld_template_avx2<24>(self, dst_data, dst_stride, src_data, src_stride, w, h, options); 582 } 583 584 BLResult bl_convert_unpremultiply_8888_trailing_alpha_pmulld_avx2( 585 const BLPixelConverterCore* self, 586 uint8_t* dst_data, intptr_t dst_stride, 587 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 588 589 return bl_convert_unpremultiply_8888_pmulld_template_avx2<0>(self, dst_data, dst_stride, src_data, src_stride, w, h, options); 590 } 591 592 // PixelConverter - Unpremultiply (FLOAT) (AVX2) 593 // ============================================= 594 595 template<uint32_t A_Shift> 596 static BL_INLINE BLResult bl_convert_unpremultiply_8888_float_template_avx2( 597 const BLPixelConverterCore* self, 598 uint8_t* dst_data, intptr_t dst_stride, 599 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 600 601 using namespace SIMD; 602 603 bl_unused(self); 604 605 if (!options) 606 options = &bl_pixel_converter_default_options; 607 608 const size_t gap = options->gap; 609 dst_stride -= uintptr_t(w) * 4u + gap; 610 src_stride -= uintptr_t(w) * 4u; 611 612 Vec32xU8 alpha_mask = make256_u32<Vec32xU8>(0xFFu << A_Shift); 613 Vec8xU32 component_mask = make256_u32(0xFFu); 614 Vec32xU8 load_store_mask = loada_64_i8_i32<Vec32xU8>(bl::common_table.loadstore16_lo8_msk8() + (w & 7u)); 615 616 Vec8xF32 f32_255 = make256_f32(255.0001f); 617 Vec8xF32 f32_lessThanOne = make256_f32(0.1f); 618 619 // Alpha byte-index that can be used by instructions that perform shuffling. 620 constexpr uint32_t AI = A_Shift / 8u; 621 constexpr uint32_t RI = (AI + 1) % 4; 622 constexpr uint32_t GI = (AI + 2) % 4; 623 constexpr uint32_t BI = (AI + 3) % 4; 624 625 for (uint32_t y = h; y != 0; y--) { 626 uint32_t i = w; 627 628 BL_NOUNROLL 629 while (i >= 8) { 630 Vec32xU8 pix = loadu<Vec32xU8>(src_data); 631 Vec8xU32 pa = vec_u32(srli_u32<AI * 8>(pix)); 632 633 if (AI != 3) pa = pa & component_mask; 634 635 Vec8xF32 fa = cvt_i32_f32(pa); 636 fa = f32_255 / max(fa, f32_lessThanOne); 637 638 Vec8xU32 pr = vec_u32(srli_u32<RI * 8>(pix)); 639 Vec8xU32 pg = vec_u32(srli_u32<GI * 8>(pix)); 640 Vec8xU32 pb = vec_u32(srli_u32<BI * 8>(pix)); 641 642 if (RI != 3) pr = pr & component_mask; 643 if (GI != 3) pg = pg & component_mask; 644 if (BI != 3) pb = pb & component_mask; 645 646 pr = vec_u32(cvt_f32_i32(cvt_i32_f32(pr) * fa)); 647 pg = vec_u32(cvt_f32_i32(cvt_i32_f32(pg) * fa)); 648 pb = vec_u32(cvt_f32_i32(cvt_i32_f32(pb) * fa)); 649 pix = pix & alpha_mask; 650 651 pr = slli_i32<RI * 8>(pr); 652 pg = slli_i32<GI * 8>(pg); 653 pb = slli_i32<BI * 8>(pb); 654 pix = pix | vec_u8(pr) | vec_u8(pg) | vec_u8(pb); 655 656 storeu(dst_data, pix); 657 658 dst_data += 32; 659 src_data += 32; 660 i -= 8; 661 } 662 663 if (i) { 664 Vec32xU8 pix = loadu_256_mask32<Vec32xU8>(src_data, load_store_mask); 665 Vec8xU32 pa = vec_u32(srli_u32<AI * 8>(pix)); 666 667 if (AI != 3) pa = pa & component_mask; 668 669 Vec8xF32 fa = cvt_i32_f32(pa); 670 fa = f32_255 / max(fa, f32_lessThanOne); 671 672 Vec8xU32 pr = vec_u32(srli_u32<RI * 8>(pix)); 673 Vec8xU32 pg = vec_u32(srli_u32<GI * 8>(pix)); 674 Vec8xU32 pb = vec_u32(srli_u32<BI * 8>(pix)); 675 676 if (RI != 3) pr = pr & component_mask; 677 if (GI != 3) pg = pg & component_mask; 678 if (BI != 3) pb = pb & component_mask; 679 680 pr = vec_u32(cvt_f32_i32(cvt_i32_f32(pr) * fa)); 681 pg = vec_u32(cvt_f32_i32(cvt_i32_f32(pg) * fa)); 682 pb = vec_u32(cvt_f32_i32(cvt_i32_f32(pb) * fa)); 683 pix = pix & alpha_mask; 684 685 pr = slli_i32<RI * 8>(pr); 686 pg = slli_i32<GI * 8>(pg); 687 pb = slli_i32<BI * 8>(pb); 688 pix = pix | vec_u8(pr) | vec_u8(pg) | vec_u8(pb); 689 690 storeu_256_mask32(dst_data, pix, load_store_mask); 691 692 dst_data += i * 4; 693 src_data += i * 4; 694 } 695 696 dst_data = bl_pixel_converter_fill_gap(dst_data, gap); 697 dst_data += dst_stride; 698 src_data += src_stride; 699 } 700 701 return BL_SUCCESS; 702 } 703 704 BLResult bl_convert_unpremultiply_8888_leading_alpha_float_avx2( 705 const BLPixelConverterCore* self, 706 uint8_t* dst_data, intptr_t dst_stride, 707 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 708 709 return bl_convert_unpremultiply_8888_float_template_avx2<24>(self, dst_data, dst_stride, src_data, src_stride, w, h, options); 710 } 711 712 BLResult bl_convert_unpremultiply_8888_trailing_alpha_float_avx2( 713 const BLPixelConverterCore* self, 714 uint8_t* dst_data, intptr_t dst_stride, 715 const uint8_t* src_data, intptr_t src_stride, uint32_t w, uint32_t h, const BLPixelConverterOptions* options) noexcept { 716 717 return bl_convert_unpremultiply_8888_float_template_avx2<0>(self, dst_data, dst_stride, src_data, src_stride, w, h, options); 718 } 719 720 #endif