filters_sse2.c (12650B)
1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Use of this source code is governed by a BSD-style license 4 // that can be found in the COPYING file in the root of the source 5 // tree. An additional intellectual property rights grant can be found 6 // in the file PATENTS. All contributing project authors may 7 // be found in the AUTHORS file in the root of the source tree. 8 // ----------------------------------------------------------------------------- 9 // 10 // SSE2 variant of alpha filters 11 // 12 // Author: Skal (pascal.massimino@gmail.com) 13 14 #include "src/dsp/dsp.h" 15 16 #if defined(WEBP_USE_SSE2) 17 18 #include <assert.h> 19 #include <emmintrin.h> 20 #include <stdlib.h> 21 #include <string.h> 22 23 #include "src/dsp/cpu.h" 24 #include "src/webp/types.h" 25 26 //------------------------------------------------------------------------------ 27 // Helpful macro. 28 29 #define DCHECK(in, out) \ 30 do { \ 31 assert((in) != NULL); \ 32 assert((out) != NULL); \ 33 assert((in) != (out)); \ 34 assert(width > 0); \ 35 assert(height > 0); \ 36 assert(stride >= width); \ 37 } while (0) 38 39 static void PredictLineTop_SSE2(const uint8_t* WEBP_RESTRICT src, 40 const uint8_t* WEBP_RESTRICT pred, 41 uint8_t* WEBP_RESTRICT dst, int length) { 42 int i; 43 const int max_pos = length & ~31; 44 assert(length >= 0); 45 for (i = 0; i < max_pos; i += 32) { 46 const __m128i A0 = _mm_loadu_si128((const __m128i*)&src[i + 0]); 47 const __m128i A1 = _mm_loadu_si128((const __m128i*)&src[i + 16]); 48 const __m128i B0 = _mm_loadu_si128((const __m128i*)&pred[i + 0]); 49 const __m128i B1 = _mm_loadu_si128((const __m128i*)&pred[i + 16]); 50 const __m128i C0 = _mm_sub_epi8(A0, B0); 51 const __m128i C1 = _mm_sub_epi8(A1, B1); 52 _mm_storeu_si128((__m128i*)&dst[i + 0], C0); 53 _mm_storeu_si128((__m128i*)&dst[i + 16], C1); 54 } 55 for (; i < length; ++i) dst[i] = src[i] - pred[i]; 56 } 57 58 // Special case for left-based prediction (when preds==dst-1 or preds==src-1). 59 static void PredictLineLeft_SSE2(const uint8_t* WEBP_RESTRICT src, 60 uint8_t* WEBP_RESTRICT dst, int length) { 61 int i; 62 const int max_pos = length & ~31; 63 assert(length >= 0); 64 for (i = 0; i < max_pos; i += 32) { 65 const __m128i A0 = _mm_loadu_si128((const __m128i*)(src + i + 0 )); 66 const __m128i B0 = _mm_loadu_si128((const __m128i*)(src + i + 0 - 1)); 67 const __m128i A1 = _mm_loadu_si128((const __m128i*)(src + i + 16 )); 68 const __m128i B1 = _mm_loadu_si128((const __m128i*)(src + i + 16 - 1)); 69 const __m128i C0 = _mm_sub_epi8(A0, B0); 70 const __m128i C1 = _mm_sub_epi8(A1, B1); 71 _mm_storeu_si128((__m128i*)(dst + i + 0), C0); 72 _mm_storeu_si128((__m128i*)(dst + i + 16), C1); 73 } 74 for (; i < length; ++i) dst[i] = src[i] - src[i - 1]; 75 } 76 77 //------------------------------------------------------------------------------ 78 // Horizontal filter. 79 80 static WEBP_INLINE void DoHorizontalFilter_SSE2( 81 const uint8_t* WEBP_RESTRICT in, int width, int height, int stride, 82 uint8_t* WEBP_RESTRICT out) { 83 int row; 84 DCHECK(in, out); 85 86 // Leftmost pixel is the same as input for topmost scanline. 87 out[0] = in[0]; 88 PredictLineLeft_SSE2(in + 1, out + 1, width - 1); 89 in += stride; 90 out += stride; 91 92 // Filter line-by-line. 93 for (row = 1; row < height; ++row) { 94 // Leftmost pixel is predicted from above. 95 out[0] = in[0] - in[-stride]; 96 PredictLineLeft_SSE2(in + 1, out + 1, width - 1); 97 in += stride; 98 out += stride; 99 } 100 } 101 102 //------------------------------------------------------------------------------ 103 // Vertical filter. 104 105 static WEBP_INLINE void DoVerticalFilter_SSE2(const uint8_t* WEBP_RESTRICT in, 106 int width, int height, int stride, 107 uint8_t* WEBP_RESTRICT out) { 108 int row; 109 DCHECK(in, out); 110 111 // Very first top-left pixel is copied. 112 out[0] = in[0]; 113 // Rest of top scan-line is left-predicted. 114 PredictLineLeft_SSE2(in + 1, out + 1, width - 1); 115 in += stride; 116 out += stride; 117 118 // Filter line-by-line. 119 for (row = 1; row < height; ++row) { 120 PredictLineTop_SSE2(in, in - stride, out, width); 121 in += stride; 122 out += stride; 123 } 124 } 125 126 //------------------------------------------------------------------------------ 127 // Gradient filter. 128 129 static WEBP_INLINE int GradientPredictor_SSE2(uint8_t a, uint8_t b, uint8_t c) { 130 const int g = a + b - c; 131 return ((g & ~0xff) == 0) ? g : (g < 0) ? 0 : 255; // clip to 8bit 132 } 133 134 static void GradientPredictDirect_SSE2(const uint8_t* const row, 135 const uint8_t* const top, 136 uint8_t* WEBP_RESTRICT const out, 137 int length) { 138 const int max_pos = length & ~7; 139 int i; 140 const __m128i zero = _mm_setzero_si128(); 141 for (i = 0; i < max_pos; i += 8) { 142 const __m128i A0 = _mm_loadl_epi64((const __m128i*)&row[i - 1]); 143 const __m128i B0 = _mm_loadl_epi64((const __m128i*)&top[i]); 144 const __m128i C0 = _mm_loadl_epi64((const __m128i*)&top[i - 1]); 145 const __m128i D = _mm_loadl_epi64((const __m128i*)&row[i]); 146 const __m128i A1 = _mm_unpacklo_epi8(A0, zero); 147 const __m128i B1 = _mm_unpacklo_epi8(B0, zero); 148 const __m128i C1 = _mm_unpacklo_epi8(C0, zero); 149 const __m128i E = _mm_add_epi16(A1, B1); 150 const __m128i F = _mm_sub_epi16(E, C1); 151 const __m128i G = _mm_packus_epi16(F, zero); 152 const __m128i H = _mm_sub_epi8(D, G); 153 _mm_storel_epi64((__m128i*)(out + i), H); 154 } 155 for (; i < length; ++i) { 156 const int delta = GradientPredictor_SSE2(row[i - 1], top[i], top[i - 1]); 157 out[i] = (uint8_t)(row[i] - delta); 158 } 159 } 160 161 static WEBP_INLINE void DoGradientFilter_SSE2(const uint8_t* WEBP_RESTRICT in, 162 int width, int height, int stride, 163 uint8_t* WEBP_RESTRICT out) { 164 int row; 165 DCHECK(in, out); 166 167 // left prediction for top scan-line 168 out[0] = in[0]; 169 PredictLineLeft_SSE2(in + 1, out + 1, width - 1); 170 in += stride; 171 out += stride; 172 173 // Filter line-by-line. 174 for (row = 1; row < height; ++row) { 175 out[0] = (uint8_t)(in[0] - in[-stride]); 176 GradientPredictDirect_SSE2(in + 1, in + 1 - stride, out + 1, width - 1); 177 in += stride; 178 out += stride; 179 } 180 } 181 182 #undef DCHECK 183 184 //------------------------------------------------------------------------------ 185 186 static void HorizontalFilter_SSE2(const uint8_t* WEBP_RESTRICT data, 187 int width, int height, int stride, 188 uint8_t* WEBP_RESTRICT filtered_data) { 189 DoHorizontalFilter_SSE2(data, width, height, stride, filtered_data); 190 } 191 192 static void VerticalFilter_SSE2(const uint8_t* WEBP_RESTRICT data, 193 int width, int height, int stride, 194 uint8_t* WEBP_RESTRICT filtered_data) { 195 DoVerticalFilter_SSE2(data, width, height, stride, filtered_data); 196 } 197 198 static void GradientFilter_SSE2(const uint8_t* WEBP_RESTRICT data, 199 int width, int height, int stride, 200 uint8_t* WEBP_RESTRICT filtered_data) { 201 DoGradientFilter_SSE2(data, width, height, stride, filtered_data); 202 } 203 204 //------------------------------------------------------------------------------ 205 // Inverse transforms 206 207 static void HorizontalUnfilter_SSE2(const uint8_t* prev, const uint8_t* in, 208 uint8_t* out, int width) { 209 int i; 210 __m128i last; 211 out[0] = (uint8_t)(in[0] + (prev == NULL ? 0 : prev[0])); 212 if (width <= 1) return; 213 last = _mm_set_epi32(0, 0, 0, out[0]); 214 for (i = 1; i + 8 <= width; i += 8) { 215 const __m128i A0 = _mm_loadl_epi64((const __m128i*)(in + i)); 216 const __m128i A1 = _mm_add_epi8(A0, last); 217 const __m128i A2 = _mm_slli_si128(A1, 1); 218 const __m128i A3 = _mm_add_epi8(A1, A2); 219 const __m128i A4 = _mm_slli_si128(A3, 2); 220 const __m128i A5 = _mm_add_epi8(A3, A4); 221 const __m128i A6 = _mm_slli_si128(A5, 4); 222 const __m128i A7 = _mm_add_epi8(A5, A6); 223 _mm_storel_epi64((__m128i*)(out + i), A7); 224 last = _mm_srli_epi64(A7, 56); 225 } 226 for (; i < width; ++i) out[i] = (uint8_t)(in[i] + out[i - 1]); 227 } 228 229 static void VerticalUnfilter_SSE2(const uint8_t* prev, const uint8_t* in, 230 uint8_t* out, int width) { 231 if (prev == NULL) { 232 HorizontalUnfilter_SSE2(NULL, in, out, width); 233 } else { 234 int i; 235 const int max_pos = width & ~31; 236 assert(width >= 0); 237 for (i = 0; i < max_pos; i += 32) { 238 const __m128i A0 = _mm_loadu_si128((const __m128i*)&in[i + 0]); 239 const __m128i A1 = _mm_loadu_si128((const __m128i*)&in[i + 16]); 240 const __m128i B0 = _mm_loadu_si128((const __m128i*)&prev[i + 0]); 241 const __m128i B1 = _mm_loadu_si128((const __m128i*)&prev[i + 16]); 242 const __m128i C0 = _mm_add_epi8(A0, B0); 243 const __m128i C1 = _mm_add_epi8(A1, B1); 244 _mm_storeu_si128((__m128i*)&out[i + 0], C0); 245 _mm_storeu_si128((__m128i*)&out[i + 16], C1); 246 } 247 for (; i < width; ++i) out[i] = (uint8_t)(in[i] + prev[i]); 248 } 249 } 250 251 static void GradientPredictInverse_SSE2(const uint8_t* const in, 252 const uint8_t* const top, 253 uint8_t* const row, int length) { 254 if (length > 0) { 255 int i; 256 const int max_pos = length & ~7; 257 const __m128i zero = _mm_setzero_si128(); 258 __m128i A = _mm_set_epi32(0, 0, 0, row[-1]); // left sample 259 for (i = 0; i < max_pos; i += 8) { 260 const __m128i tmp0 = _mm_loadl_epi64((const __m128i*)&top[i]); 261 const __m128i tmp1 = _mm_loadl_epi64((const __m128i*)&top[i - 1]); 262 const __m128i B = _mm_unpacklo_epi8(tmp0, zero); 263 const __m128i C = _mm_unpacklo_epi8(tmp1, zero); 264 const __m128i D = _mm_loadl_epi64((const __m128i*)&in[i]); // base input 265 const __m128i E = _mm_sub_epi16(B, C); // unclipped gradient basis B - C 266 __m128i out = zero; // accumulator for output 267 __m128i mask_hi = _mm_set_epi32(0, 0, 0, 0xff); 268 int k = 8; 269 while (1) { 270 const __m128i tmp3 = _mm_add_epi16(A, E); // delta = A + B - C 271 const __m128i tmp4 = _mm_packus_epi16(tmp3, zero); // saturate delta 272 const __m128i tmp5 = _mm_add_epi8(tmp4, D); // add to in[] 273 A = _mm_and_si128(tmp5, mask_hi); // 1-complement clip 274 out = _mm_or_si128(out, A); // accumulate output 275 if (--k == 0) break; 276 A = _mm_slli_si128(A, 1); // rotate left sample 277 mask_hi = _mm_slli_si128(mask_hi, 1); // rotate mask 278 A = _mm_unpacklo_epi8(A, zero); // convert 8b->16b 279 } 280 A = _mm_srli_si128(A, 7); // prepare left sample for next iteration 281 _mm_storel_epi64((__m128i*)&row[i], out); 282 } 283 for (; i < length; ++i) { 284 const int delta = GradientPredictor_SSE2(row[i - 1], top[i], top[i - 1]); 285 row[i] = (uint8_t)(in[i] + delta); 286 } 287 } 288 } 289 290 static void GradientUnfilter_SSE2(const uint8_t* prev, const uint8_t* in, 291 uint8_t* out, int width) { 292 if (prev == NULL) { 293 HorizontalUnfilter_SSE2(NULL, in, out, width); 294 } else { 295 out[0] = (uint8_t)(in[0] + prev[0]); // predict from above 296 GradientPredictInverse_SSE2(in + 1, prev + 1, out + 1, width - 1); 297 } 298 } 299 300 //------------------------------------------------------------------------------ 301 // Entry point 302 303 extern void VP8FiltersInitSSE2(void); 304 305 WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitSSE2(void) { 306 WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter_SSE2; 307 #if defined(CHROMIUM) 308 // TODO(crbug.com/654974) 309 (void)VerticalUnfilter_SSE2; 310 #else 311 WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter_SSE2; 312 #endif 313 WebPUnfilters[WEBP_FILTER_GRADIENT] = GradientUnfilter_SSE2; 314 315 WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter_SSE2; 316 WebPFilters[WEBP_FILTER_VERTICAL] = VerticalFilter_SSE2; 317 WebPFilters[WEBP_FILTER_GRADIENT] = GradientFilter_SSE2; 318 } 319 320 #else // !WEBP_USE_SSE2 321 322 WEBP_DSP_INIT_STUB(VP8FiltersInitSSE2) 323 324 #endif // WEBP_USE_SSE2