jpegops_sse2.cpp (15247B)
1 // This file is part of Blend2D project <https://blend2d.com> 2 // 3 // See blend2d.h or LICENSE.md for license and copyright information 4 // SPDX-License-Identifier: Zlib 5 6 // The JPEG codec is based on stb_image <https://github.com/nothings/stb> 7 // released into PUBLIC DOMAIN. Blend2D's JPEG codec can be distributed 8 // under Blend2D's ZLIB license or under STB's PUBLIC DOMAIN as well. 9 10 #include "../api-build_p.h" 11 #if defined(BL_TARGET_OPT_SSE2) 12 13 #include "../rgba_p.h" 14 #include "../codec/jpegops_p.h" 15 #include "../simd/simd_p.h" 16 #include "../support/intops_p.h" 17 #include "../support/memops_p.h" 18 19 namespace bl::Jpeg { 20 21 // bl::Jpeg::Opts - IDCT - SSE2 22 // ============================ 23 24 struct alignas(16) OptConstSSE2 { 25 // IDCT. 26 int16_t idct_rot0a[8], idct_rot0b[8]; 27 int16_t idct_rot1a[8], idct_rot1b[8]; 28 int16_t idct_rot2a[8], idct_rot2b[8]; 29 int16_t idct_rot3a[8], idct_rot3b[8]; 30 31 int32_t idct_col_bias[4]; 32 int32_t idct_row_bias[4]; 33 34 // YCbCr. 35 int32_t ycbcr_allones[4]; 36 int16_t ycbcr_tosigned[8]; 37 int32_t ycbcr_round[4]; 38 int16_t ycbcr_yycr_mul[8]; 39 int16_t ycbcr_yycb_mul[8]; 40 int16_t ycbcr_cbcr_mul[8]; 41 }; 42 43 #define DATA_4X(...) { __VA_ARGS__, __VA_ARGS__, __VA_ARGS__, __VA_ARGS__ } 44 static const OptConstSSE2 optConstSSE2 = { 45 // IDCT. 46 DATA_4X(BL_JPEG_IDCT_P_0_541196100 , 47 BL_JPEG_IDCT_P_0_541196100 + BL_JPEG_IDCT_M_1_847759065), 48 DATA_4X(BL_JPEG_IDCT_P_0_541196100 + BL_JPEG_IDCT_P_0_765366865 , 49 BL_JPEG_IDCT_P_0_541196100 ), 50 DATA_4X(BL_JPEG_IDCT_P_1_175875602 + BL_JPEG_IDCT_M_0_899976223 , 51 BL_JPEG_IDCT_P_1_175875602 ), 52 DATA_4X(BL_JPEG_IDCT_P_1_175875602 , 53 BL_JPEG_IDCT_P_1_175875602 + BL_JPEG_IDCT_M_2_562915447), 54 DATA_4X(BL_JPEG_IDCT_M_1_961570560 + BL_JPEG_IDCT_P_0_298631336 , 55 BL_JPEG_IDCT_M_1_961570560 ), 56 DATA_4X(BL_JPEG_IDCT_M_1_961570560 , 57 BL_JPEG_IDCT_M_1_961570560 + BL_JPEG_IDCT_P_3_072711026), 58 DATA_4X(BL_JPEG_IDCT_M_0_390180644 + BL_JPEG_IDCT_P_2_053119869 , 59 BL_JPEG_IDCT_M_0_390180644 ), 60 DATA_4X(BL_JPEG_IDCT_M_0_390180644 , 61 BL_JPEG_IDCT_M_0_390180644 + BL_JPEG_IDCT_P_1_501321110), 62 63 DATA_4X(BL_JPEG_IDCT_COL_BIAS), 64 DATA_4X(BL_JPEG_IDCT_ROW_BIAS), 65 66 // YCbCr. 67 DATA_4X(-1), 68 DATA_4X(-128, -128), 69 DATA_4X(1 << (BL_JPEG_YCBCR_PREC - 1)), 70 DATA_4X( BL_JPEG_YCBCR_FIXED(1.00000), BL_JPEG_YCBCR_FIXED(1.40200)), 71 DATA_4X( BL_JPEG_YCBCR_FIXED(1.00000), BL_JPEG_YCBCR_FIXED(1.77200)), 72 DATA_4X(-BL_JPEG_YCBCR_FIXED(0.34414), -BL_JPEG_YCBCR_FIXED(0.71414)) 73 }; 74 #undef DATA_4X 75 76 #define BL_JPEG_IDCT_INTERLEAVE8_XMM(a, b) { auto t = a; a = interleave_lo_u8(a, b); b = interleave_hi_u8(t, b); } 77 #define BL_JPEG_IDCT_INTERLEAVE16_XMM(a, b) { auto t = a; a = interleave_lo_u16(a, b); b = interleave_hi_u16(t, b); } 78 79 // out(0) = c0[even]*x + c0[odd]*y (in 16-bit, out 32-bit). 80 // out(1) = c1[even]*x + c1[odd]*y (in 16-bit, out 32-bit). 81 #define BL_JPEG_IDCT_ROTATE_XMM(dst0, dst1, x, y, c0, c1) \ 82 VecPair<Vec4xI32> dst0; \ 83 VecPair<Vec4xI32> dst1; \ 84 \ 85 { \ 86 VecPair<Vec4xI32> tmp; \ 87 \ 88 tmp[0] = vec_i32(interleave_lo_u16(x, y)); \ 89 tmp[1] = vec_i32(interleave_hi_u16(x, y)); \ 90 dst0[0] = maddw_i16_i32(tmp[0], vec_const<Vec4xI32>(constants.c0)); \ 91 dst0[1] = maddw_i16_i32(tmp[1], vec_const<Vec4xI32>(constants.c0)); \ 92 dst1[0] = maddw_i16_i32(tmp[0], vec_const<Vec4xI32>(constants.c1)); \ 93 dst1[1] = maddw_i16_i32(tmp[1], vec_const<Vec4xI32>(constants.c1)); \ 94 } 95 96 // out = in << 12 (in 16-bit, out 32-bit) 97 #define BL_JPEG_IDCT_WIDEN_XMM(dst, in) \ 98 VecPair<Vec4xI32> dst; \ 99 dst[0] = srai_i32<4>(vec_i32(interleave_lo_u16(make_zero<Vec8xI16>(), in))); \ 100 dst[1] = srai_i32<4>(vec_i32(interleave_hi_u16(make_zero<Vec8xI16>(), in))); 101 102 // Wide add (32-bit). 103 #define BL_JPEG_IDCT_WADD_XMM(dst, a, b) \ 104 VecPair<Vec4xI32> dst{add_i32(a[0], b[0]), add_i32(a[1], b[1])}; 105 106 // Wide sub (32-bit). 107 #define BL_JPEG_IDCT_WSUB_XMM(dst, a, b) \ 108 VecPair<Vec4xI32> dst{sub_i32(a[0], b[0]), sub_i32(a[1], b[1])}; 109 110 // Butterfly a/b, add bias, then shift by `norm` and pack to 16-bit. 111 #define BL_JPEG_IDCT_BFLY_XMM(dst0, dst1, a, b, bias, norm) \ 112 { \ 113 VecPair<Vec4xI32> a_biased{add_i32(a[0], bias), add_i32(a[1], bias)}; \ 114 BL_JPEG_IDCT_WADD_XMM(sum, a_biased, b) \ 115 BL_JPEG_IDCT_WSUB_XMM(diff, a_biased, b) \ 116 \ 117 dst0 = vec_i16(packs_128_i32_i16(srai_i32<norm>(sum[0]), srai_i32<norm>(sum[1]))); \ 118 dst1 = vec_i16(packs_128_i32_i16(srai_i32<norm>(diff[0]), srai_i32<norm>(diff[1]))); \ 119 } 120 121 #define BL_JPEG_IDCT_IDCT_PASS_XMM(bias, norm) { \ 122 /* Even part. */ \ 123 BL_JPEG_IDCT_ROTATE_XMM(t2e, t3e, row2, row6, idct_rot0a, idct_rot0b) \ 124 \ 125 Vec8xI16 sum04 = add_i16(row0, row4); \ 126 Vec8xI16 dif04 = sub_i16(row0, row4); \ 127 \ 128 BL_JPEG_IDCT_WIDEN_XMM(t0e, sum04) \ 129 BL_JPEG_IDCT_WIDEN_XMM(t1e, dif04) \ 130 \ 131 BL_JPEG_IDCT_WADD_XMM(x0, t0e, t3e) \ 132 BL_JPEG_IDCT_WSUB_XMM(x3, t0e, t3e) \ 133 BL_JPEG_IDCT_WADD_XMM(x1, t1e, t2e) \ 134 BL_JPEG_IDCT_WSUB_XMM(x2, t1e, t2e) \ 135 \ 136 /* Odd part */ \ 137 BL_JPEG_IDCT_ROTATE_XMM(y0o, y2o, row7, row3, idct_rot2a, idct_rot2b) \ 138 BL_JPEG_IDCT_ROTATE_XMM(y1o, y3o, row5, row1, idct_rot3a, idct_rot3b) \ 139 Vec8xI16 sum17 = add_i16(row1, row7); \ 140 Vec8xI16 sum35 = add_i16(row3, row5); \ 141 BL_JPEG_IDCT_ROTATE_XMM(y4o,y5o, sum17, sum35, idct_rot1a, idct_rot1b) \ 142 \ 143 BL_JPEG_IDCT_WADD_XMM(x4, y0o, y4o) \ 144 BL_JPEG_IDCT_WADD_XMM(x5, y1o, y5o) \ 145 BL_JPEG_IDCT_WADD_XMM(x6, y2o, y5o) \ 146 BL_JPEG_IDCT_WADD_XMM(x7, y3o, y4o) \ 147 \ 148 BL_JPEG_IDCT_BFLY_XMM(row0, row7, x0, x7, bias, norm) \ 149 BL_JPEG_IDCT_BFLY_XMM(row1, row6, x1, x6, bias, norm) \ 150 BL_JPEG_IDCT_BFLY_XMM(row2, row5, x2, x5, bias, norm) \ 151 BL_JPEG_IDCT_BFLY_XMM(row3, row4, x3, x4, bias, norm) \ 152 } 153 154 void BL_CDECL idct8_sse2(uint8_t* dst, intptr_t dst_stride, const int16_t* src, const uint16_t* q_table) noexcept { 155 using namespace SIMD; 156 157 const OptConstSSE2& constants = optConstSSE2; 158 159 // Load and dequantize (`src` is aligned to 16 bytes, `q_table` doesn't have to be). 160 Vec8xI16 row0 = loadu<Vec8xI16>(q_table + 0) * loada<Vec8xI16>(src + 0); 161 Vec8xI16 row1 = loadu<Vec8xI16>(q_table + 8) * loada<Vec8xI16>(src + 8); 162 Vec8xI16 row2 = loadu<Vec8xI16>(q_table + 16) * loada<Vec8xI16>(src + 16); 163 Vec8xI16 row3 = loadu<Vec8xI16>(q_table + 24) * loada<Vec8xI16>(src + 24); 164 Vec8xI16 row4 = loadu<Vec8xI16>(q_table + 32) * loada<Vec8xI16>(src + 32); 165 Vec8xI16 row5 = loadu<Vec8xI16>(q_table + 40) * loada<Vec8xI16>(src + 40); 166 Vec8xI16 row6 = loadu<Vec8xI16>(q_table + 48) * loada<Vec8xI16>(src + 48); 167 Vec8xI16 row7 = loadu<Vec8xI16>(q_table + 56) * loada<Vec8xI16>(src + 56); 168 169 // IDCT columns. 170 BL_JPEG_IDCT_IDCT_PASS_XMM(vec_const<Vec4xI32>(constants.idct_col_bias), BL_JPEG_IDCT_COL_NORM) 171 172 // Transpose. 173 BL_JPEG_IDCT_INTERLEAVE16_XMM(row0, row4) // [a0a4|b0b4|c0c4|d0d4] | [e0e4|f0f4|g0g4|h0h4] 174 BL_JPEG_IDCT_INTERLEAVE16_XMM(row2, row6) // [a2a6|b2b6|c2c6|d2d6] | [e2e6|f2f6|g2g6|h2h6] 175 BL_JPEG_IDCT_INTERLEAVE16_XMM(row1, row5) // [a1a5|b1b5|c2c5|d1d5] | [e1e5|f1f5|g1g5|h1h5] 176 BL_JPEG_IDCT_INTERLEAVE16_XMM(row3, row7) // [a3a7|b3b7|c3c7|d3d7] | [e3e7|f3f7|g3g7|h3h7] 177 178 BL_JPEG_IDCT_INTERLEAVE16_XMM(row0, row2) // [a0a2|a4a6|b0b2|b4b6] | [c0c2|c4c6|d0d2|d4d6] 179 BL_JPEG_IDCT_INTERLEAVE16_XMM(row1, row3) // [a1a3|a5a7|b1b3|b5b7] | [c1c3|c5c7|d1d3|d5d7] 180 BL_JPEG_IDCT_INTERLEAVE16_XMM(row4, row6) // [e0e2|e4e6|f0f2|f4f6] | [g0g2|g4g6|h0h2|h4h6] 181 BL_JPEG_IDCT_INTERLEAVE16_XMM(row5, row7) // [e1e3|e5e7|f1f3|f5f7] | [g1g3|g5g7|h1h3|h5h7] 182 183 BL_JPEG_IDCT_INTERLEAVE16_XMM(row0, row1) // [a0a1|a2a3|a4a5|a6a7] | [b0b1|b2b3|b4b5|b6b7] 184 BL_JPEG_IDCT_INTERLEAVE16_XMM(row2, row3) // [c0c1|c2c3|c4c5|c6c7] | [d0d1|d2d3|d4d5|d6d7] 185 BL_JPEG_IDCT_INTERLEAVE16_XMM(row4, row5) // [e0e1|e2e3|e4e5|e6e7] | [f0f1|f2f3|f4f5|f6f7] 186 BL_JPEG_IDCT_INTERLEAVE16_XMM(row6, row7) // [g0g1|g2g3|g4g5|g6g7] | [h0h1|h2h3|h4h5|h6h7] 187 188 // IDCT rows. 189 BL_JPEG_IDCT_IDCT_PASS_XMM(vec_const<Vec4xI32>(constants.idct_row_bias), BL_JPEG_IDCT_ROW_NORM) 190 191 // Pack to 8-bit unsigned integers with saturation. 192 row0 = packs_128_i16_u8(row0, row1); // [a0a1a2a3|a4a5a6a7|b0b1b2b3|b4b5b6b7] 193 row2 = packs_128_i16_u8(row2, row3); // [c0c1c2c3|c4c5c6c7|d0d1d2d3|d4d5d6d7] 194 row4 = packs_128_i16_u8(row4, row5); // [e0e1e2e3|e4e5e6e7|f0f1f2f3|f4f5f6f7] 195 row6 = packs_128_i16_u8(row6, row7); // [g0g1g2g3|g4g5g6g7|h0h1h2h3|h4h5h6h7] 196 197 // Transpose. 198 BL_JPEG_IDCT_INTERLEAVE8_XMM(row0, row4) // [a0e0a1e1|a2e2a3e3|a4e4a5e5|a6e6a7e7] | [b0f0b1f1|b2f2b3f3|b4f4b5f5|b6f6b7f7] 199 BL_JPEG_IDCT_INTERLEAVE8_XMM(row2, row6) // [c0g0c1g1|c2g2c3g3|c4g4c5g5|c6g6c7g7] | [d0h0d1h1|d2h2d3h3|d4h4d5h5|d6h6d7h7] 200 BL_JPEG_IDCT_INTERLEAVE8_XMM(row0, row2) // [a0c0e0g0|a1c1e1g1|a2c2e2g2|a3c3e3g3] | [a4c4e4g4|a5c5e5g5|a6c6e6g6|a7c7e7g7] 201 BL_JPEG_IDCT_INTERLEAVE8_XMM(row4, row6) // [b0d0f0h0|b1d1f1h1|b2d2f2h2|b3d3f3h3| | [b4d4f4h4|b5d5f5h5|b6d6f6h6|b7d7f7h7] 202 BL_JPEG_IDCT_INTERLEAVE8_XMM(row0, row4) // [a0b0c0d0|e0f0g0h0|a1b1c1d1|e1f1g1h1] | [a2b2c2d2|e2f2g2h2|a3b3c3d3|e3f3g3h3] 203 BL_JPEG_IDCT_INTERLEAVE8_XMM(row2, row6) // [a4b4c4d4|e4f4g4h4|a5b5c5d5|e5f5g5h5] | [a6b6c6d6|e6f6g6h6|a7b7c7d7|e7f7g7h7] 204 205 // Store. 206 uint8_t* dst0 = dst; 207 uint8_t* dst1 = dst + dst_stride; 208 intptr_t dstStride2 = dst_stride * 2; 209 210 storeu_64(dst0, row0); dst0 += dstStride2; 211 storeh_64(dst1, row0); dst1 += dstStride2; 212 213 storeu_64(dst0, row4); dst0 += dstStride2; 214 storeh_64(dst1, row4); dst1 += dstStride2; 215 216 storeu_64(dst0, row2); dst0 += dstStride2; 217 storeh_64(dst1, row2); dst1 += dstStride2; 218 219 storeu_64(dst0, row6); 220 storeh_64(dst1, row6); 221 } 222 223 // bl::Jpeg::Opts - RGB32 From YCbCr8 - SSE2 224 // ========================================= 225 226 void BL_CDECL rgb32_from_ycbcr8_sse2(uint8_t* dst, const uint8_t* pY, const uint8_t* pCb, const uint8_t* pCr, uint32_t count) noexcept { 227 using namespace SIMD; 228 uint32_t i = count; 229 230 const OptConstSSE2& constants = optConstSSE2; 231 232 while (i >= 8) { 233 Vec8xI16 yy = unpack_lo64_u8_u16(loadu_64<Vec8xI16>(pY)); 234 Vec8xI16 cb = unpack_lo64_u8_u16(loadu_64<Vec8xI16>(pCb)); 235 Vec8xI16 cr = unpack_lo64_u8_u16(loadu_64<Vec8xI16>(pCr)); 236 237 cb = add_i16(cb, vec_const<Vec8xI16>(constants.ycbcr_tosigned)); 238 cr = add_i16(cr, vec_const<Vec8xI16>(constants.ycbcr_tosigned)); 239 240 Vec4xI32 r_l = vec_i32(maddw_i16_i32(interleave_lo_u16(yy, cr), vec_const<Vec8xI16>(constants.ycbcr_yycr_mul))); 241 Vec4xI32 r_h = vec_i32(maddw_i16_i32(interleave_hi_u16(yy, cr), vec_const<Vec8xI16>(constants.ycbcr_yycr_mul))); 242 243 Vec4xI32 b_l = vec_i32(maddw_i16_i32(interleave_lo_u16(yy, cb), vec_const<Vec8xI16>(constants.ycbcr_yycb_mul))); 244 Vec4xI32 b_h = vec_i32(maddw_i16_i32(interleave_hi_u16(yy, cb), vec_const<Vec8xI16>(constants.ycbcr_yycb_mul))); 245 246 Vec4xI32 g_l = vec_i32(maddw_i16_i32(interleave_lo_u16(cb, cr), vec_const<Vec8xI16>(constants.ycbcr_cbcr_mul))); 247 Vec4xI32 g_h = vec_i32(maddw_i16_i32(interleave_hi_u16(cb, cr), vec_const<Vec8xI16>(constants.ycbcr_cbcr_mul))); 248 249 g_l = add_i32(g_l, slli_i32<BL_JPEG_YCBCR_PREC>(vec_i32(unpack_lo64_u16_u32(yy)))); 250 g_h = add_i32(g_h, slli_i32<BL_JPEG_YCBCR_PREC>(vec_i32(unpack_hi64_u16_u32(yy)))); 251 252 r_l = add_i32(r_l, vec_const<Vec4xI32>(constants.ycbcr_round)); 253 r_h = add_i32(r_h, vec_const<Vec4xI32>(constants.ycbcr_round)); 254 g_l = add_i32(g_l, vec_const<Vec4xI32>(constants.ycbcr_round)); 255 g_h = add_i32(g_h, vec_const<Vec4xI32>(constants.ycbcr_round)); 256 b_l = add_i32(b_l, vec_const<Vec4xI32>(constants.ycbcr_round)); 257 b_h = add_i32(b_h, vec_const<Vec4xI32>(constants.ycbcr_round)); 258 259 r_l = srai_i32<BL_JPEG_YCBCR_PREC>(r_l); 260 r_h = srai_i32<BL_JPEG_YCBCR_PREC>(r_h); 261 g_l = srai_i32<BL_JPEG_YCBCR_PREC>(g_l); 262 g_h = srai_i32<BL_JPEG_YCBCR_PREC>(g_h); 263 b_l = srai_i32<BL_JPEG_YCBCR_PREC>(b_l); 264 b_h = srai_i32<BL_JPEG_YCBCR_PREC>(b_h); 265 266 Vec16xU8 r = vec_u8(packz_128_u32_u8(r_l, r_h)); 267 Vec16xU8 g = vec_u8(packz_128_u32_u8(g_l, g_h)); 268 Vec16xU8 b = vec_u8(packz_128_u32_u8(b_l, b_h)); 269 270 Vec16xU8 ra = interleave_lo_u8(r, vec_const<Vec16xU8>(constants.ycbcr_allones)); 271 Vec16xU8 bg = interleave_lo_u8(b, g); 272 273 Vec16xU8 bgra0 = interleave_lo_u16(bg, ra); 274 Vec16xU8 bgra1 = interleave_hi_u16(bg, ra); 275 276 storeu(dst + 0, bgra0); 277 storeu(dst + 16, bgra1); 278 279 dst += 32; 280 pY += 8; 281 pCb += 8; 282 pCr += 8; 283 i -= 8; 284 } 285 286 while (i) { 287 int yy = (int(pY[0]) << BL_JPEG_YCBCR_PREC) + (1 << (BL_JPEG_YCBCR_PREC - 1)); 288 int cr = int(pCr[0]) - 128; 289 int cb = int(pCb[0]) - 128; 290 291 int r = yy + cr * BL_JPEG_YCBCR_FIXED(1.40200); 292 int g = yy - cr * BL_JPEG_YCBCR_FIXED(0.71414) - cb * BL_JPEG_YCBCR_FIXED(0.34414); 293 int b = yy + cb * BL_JPEG_YCBCR_FIXED(1.77200); 294 295 uint32_t rgba32 = RgbaInternal::packRgba32(IntOps::clamp_to_byte(r >> BL_JPEG_YCBCR_PREC), 296 IntOps::clamp_to_byte(g >> BL_JPEG_YCBCR_PREC), 297 IntOps::clamp_to_byte(b >> BL_JPEG_YCBCR_PREC)); 298 MemOps::writeU32a(dst, rgba32); 299 300 dst += 4; 301 pY += 1; 302 pCb += 1; 303 pCr += 1; 304 i -= 1; 305 } 306 } 307 308 } // {bl::Jpeg} 309 310 #endif // BL_TARGET_OPT_SSE2