idct8x8: Fix undefined negative shifts
[libav.git] / libavcodec / simple_idct_template.c
CommitLineData
e7a972e1
MR
1/*
2 * Simple IDCT
3 *
4 * Copyright (c) 2001 Michael Niedermayer <michaelni@gmx.at>
5 *
6 * This file is part of Libav.
7 *
8 * Libav is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * Libav is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with Libav; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23/**
24 * @file
25 * simpleidct in C.
26 */
27
28/*
29 based upon some outcommented c code from mpeg2dec (idct_mmx.c
30 written by Aaron Holtzman <aholtzma@ess.engr.uvic.ca>)
31 */
32
aba70bb5
DB
33#include "simple_idct.h"
34
e7a972e1
MR
35#include "bit_depth_template.c"
36
37#undef W1
38#undef W2
39#undef W3
40#undef W4
41#undef W5
42#undef W6
43#undef W7
44#undef ROW_SHIFT
45#undef COL_SHIFT
46#undef DC_SHIFT
47#undef MUL
48#undef MAC
49
50#if BIT_DEPTH == 8
51
52#define W1 22725 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
53#define W2 21407 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
54#define W3 19266 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
55#define W4 16383 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
56#define W5 12873 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
57#define W6 8867 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
58#define W7 4520 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
59
60#define ROW_SHIFT 11
61#define COL_SHIFT 20
62#define DC_SHIFT 3
63
64#define MUL(a, b) MUL16(a, b)
65#define MAC(a, b, c) MAC16(a, b, c)
66
67#elif BIT_DEPTH == 10
68
69#define W1 90901
70#define W2 85627
71#define W3 77062
72#define W4 65535
73#define W5 51491
74#define W6 35468
75#define W7 18081
76
77#define ROW_SHIFT 15
78#define COL_SHIFT 20
79#define DC_SHIFT 1
80
81#define MUL(a, b) ((a) * (b))
82#define MAC(a, b, c) ((a) += (b) * (c))
83
84#else
85
86#error "Unsupported bitdepth"
87
88#endif
89
88bd7fdc 90static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift)
e7a972e1 91{
2cc4f3b2 92 int a0, a1, a2, a3, b0, b1, b2, b3;
e7a972e1
MR
93
94#if HAVE_FAST_64BIT
95#define ROW0_MASK (0xffffLL << 48 * HAVE_BIGENDIAN)
2cc4f3b2 96 if (((((uint64_t *)row)[0] & ~ROW0_MASK) | ((uint64_t *)row)[1]) == 0) {
f78cd0c2
RB
97 uint64_t temp;
98 if (DC_SHIFT - extra_shift > 0) {
1389b4c1 99 temp = (row[0] * (1 << (DC_SHIFT - extra_shift))) & 0xffff;
f78cd0c2
RB
100 } else {
101 temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff;
102 }
1389b4c1
KBA
103 temp += temp * (1 << 16);
104 temp += temp * ((uint64_t) 1 << 32);
2cc4f3b2
MR
105 ((uint64_t *)row)[0] = temp;
106 ((uint64_t *)row)[1] = temp;
107 return;
108 }
e7a972e1 109#else
2cc4f3b2
MR
110 if (!(((uint32_t*)row)[1] |
111 ((uint32_t*)row)[2] |
112 ((uint32_t*)row)[3] |
113 row[1])) {
f78cd0c2
RB
114 uint32_t temp;
115 if (DC_SHIFT - extra_shift > 0) {
1389b4c1 116 temp = (row[0] * (1 << (DC_SHIFT - extra_shift))) & 0xffff;
f78cd0c2
RB
117 } else {
118 temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff;
119 }
1389b4c1 120 temp += temp * (1 << 16);
2cc4f3b2
MR
121 ((uint32_t*)row)[0]=((uint32_t*)row)[1] =
122 ((uint32_t*)row)[2]=((uint32_t*)row)[3] = temp;
123 return;
124 }
e7a972e1
MR
125#endif
126
2cc4f3b2
MR
127 a0 = (W4 * row[0]) + (1 << (ROW_SHIFT - 1));
128 a1 = a0;
129 a2 = a0;
130 a3 = a0;
131
132 a0 += W2 * row[2];
133 a1 += W6 * row[2];
134 a2 -= W6 * row[2];
135 a3 -= W2 * row[2];
136
137 b0 = MUL(W1, row[1]);
138 MAC(b0, W3, row[3]);
139 b1 = MUL(W3, row[1]);
140 MAC(b1, -W7, row[3]);
141 b2 = MUL(W5, row[1]);
142 MAC(b2, -W1, row[3]);
143 b3 = MUL(W7, row[1]);
144 MAC(b3, -W5, row[3]);
145
146 if (AV_RN64A(row + 4)) {
147 a0 += W4*row[4] + W6*row[6];
148 a1 += - W4*row[4] - W2*row[6];
149 a2 += - W4*row[4] + W2*row[6];
150 a3 += W4*row[4] - W6*row[6];
151
152 MAC(b0, W5, row[5]);
153 MAC(b0, W7, row[7]);
154
155 MAC(b1, -W1, row[5]);
156 MAC(b1, -W5, row[7]);
157
158 MAC(b2, W7, row[5]);
159 MAC(b2, W3, row[7]);
160
161 MAC(b3, W3, row[5]);
162 MAC(b3, -W1, row[7]);
163 }
164
f78cd0c2
RB
165 row[0] = (a0 + b0) >> (ROW_SHIFT + extra_shift);
166 row[7] = (a0 - b0) >> (ROW_SHIFT + extra_shift);
167 row[1] = (a1 + b1) >> (ROW_SHIFT + extra_shift);
168 row[6] = (a1 - b1) >> (ROW_SHIFT + extra_shift);
169 row[2] = (a2 + b2) >> (ROW_SHIFT + extra_shift);
170 row[5] = (a2 - b2) >> (ROW_SHIFT + extra_shift);
171 row[3] = (a3 + b3) >> (ROW_SHIFT + extra_shift);
172 row[4] = (a3 - b3) >> (ROW_SHIFT + extra_shift);
e7a972e1
MR
173}
174
a402f109
MR
175#define IDCT_COLS do { \
176 a0 = W4 * (col[8*0] + ((1<<(COL_SHIFT-1))/W4)); \
177 a1 = a0; \
178 a2 = a0; \
179 a3 = a0; \
180 \
181 a0 += W2*col[8*2]; \
182 a1 += W6*col[8*2]; \
183 a2 += -W6*col[8*2]; \
184 a3 += -W2*col[8*2]; \
185 \
186 b0 = MUL(W1, col[8*1]); \
187 b1 = MUL(W3, col[8*1]); \
188 b2 = MUL(W5, col[8*1]); \
189 b3 = MUL(W7, col[8*1]); \
190 \
191 MAC(b0, W3, col[8*3]); \
192 MAC(b1, -W7, col[8*3]); \
193 MAC(b2, -W1, col[8*3]); \
194 MAC(b3, -W5, col[8*3]); \
195 \
196 if (col[8*4]) { \
197 a0 += W4*col[8*4]; \
198 a1 += -W4*col[8*4]; \
199 a2 += -W4*col[8*4]; \
200 a3 += W4*col[8*4]; \
201 } \
202 \
203 if (col[8*5]) { \
204 MAC(b0, W5, col[8*5]); \
205 MAC(b1, -W1, col[8*5]); \
206 MAC(b2, W7, col[8*5]); \
207 MAC(b3, W3, col[8*5]); \
208 } \
209 \
210 if (col[8*6]) { \
211 a0 += W6*col[8*6]; \
212 a1 += -W2*col[8*6]; \
213 a2 += W2*col[8*6]; \
214 a3 += -W6*col[8*6]; \
215 } \
216 \
217 if (col[8*7]) { \
218 MAC(b0, W7, col[8*7]); \
219 MAC(b1, -W5, col[8*7]); \
220 MAC(b2, W3, col[8*7]); \
221 MAC(b3, -W1, col[8*7]); \
222 } \
223 } while (0)
224
e7a972e1 225static inline void FUNC(idctSparseColPut)(pixel *dest, int line_size,
88bd7fdc 226 int16_t *col)
e7a972e1 227{
2cc4f3b2 228 int a0, a1, a2, a3, b0, b1, b2, b3;
2cc4f3b2
MR
229
230 IDCT_COLS;
231
c23acbae 232 dest[0] = av_clip_pixel((a0 + b0) >> COL_SHIFT);
2cc4f3b2 233 dest += line_size;
c23acbae 234 dest[0] = av_clip_pixel((a1 + b1) >> COL_SHIFT);
2cc4f3b2 235 dest += line_size;
c23acbae 236 dest[0] = av_clip_pixel((a2 + b2) >> COL_SHIFT);
2cc4f3b2 237 dest += line_size;
c23acbae 238 dest[0] = av_clip_pixel((a3 + b3) >> COL_SHIFT);
2cc4f3b2 239 dest += line_size;
c23acbae 240 dest[0] = av_clip_pixel((a3 - b3) >> COL_SHIFT);
2cc4f3b2 241 dest += line_size;
c23acbae 242 dest[0] = av_clip_pixel((a2 - b2) >> COL_SHIFT);
2cc4f3b2 243 dest += line_size;
c23acbae 244 dest[0] = av_clip_pixel((a1 - b1) >> COL_SHIFT);
2cc4f3b2 245 dest += line_size;
c23acbae 246 dest[0] = av_clip_pixel((a0 - b0) >> COL_SHIFT);
e7a972e1
MR
247}
248
249static inline void FUNC(idctSparseColAdd)(pixel *dest, int line_size,
88bd7fdc 250 int16_t *col)
e7a972e1 251{
2cc4f3b2 252 int a0, a1, a2, a3, b0, b1, b2, b3;
2cc4f3b2
MR
253
254 IDCT_COLS;
255
c23acbae 256 dest[0] = av_clip_pixel(dest[0] + ((a0 + b0) >> COL_SHIFT));
2cc4f3b2 257 dest += line_size;
c23acbae 258 dest[0] = av_clip_pixel(dest[0] + ((a1 + b1) >> COL_SHIFT));
2cc4f3b2 259 dest += line_size;
c23acbae 260 dest[0] = av_clip_pixel(dest[0] + ((a2 + b2) >> COL_SHIFT));
2cc4f3b2 261 dest += line_size;
c23acbae 262 dest[0] = av_clip_pixel(dest[0] + ((a3 + b3) >> COL_SHIFT));
2cc4f3b2 263 dest += line_size;
c23acbae 264 dest[0] = av_clip_pixel(dest[0] + ((a3 - b3) >> COL_SHIFT));
2cc4f3b2 265 dest += line_size;
c23acbae 266 dest[0] = av_clip_pixel(dest[0] + ((a2 - b2) >> COL_SHIFT));
2cc4f3b2 267 dest += line_size;
c23acbae 268 dest[0] = av_clip_pixel(dest[0] + ((a1 - b1) >> COL_SHIFT));
2cc4f3b2 269 dest += line_size;
c23acbae 270 dest[0] = av_clip_pixel(dest[0] + ((a0 - b0) >> COL_SHIFT));
e7a972e1
MR
271}
272
88bd7fdc 273static inline void FUNC(idctSparseCol)(int16_t *col)
e7a972e1 274{
2cc4f3b2
MR
275 int a0, a1, a2, a3, b0, b1, b2, b3;
276
277 IDCT_COLS;
278
279 col[0 ] = ((a0 + b0) >> COL_SHIFT);
280 col[8 ] = ((a1 + b1) >> COL_SHIFT);
281 col[16] = ((a2 + b2) >> COL_SHIFT);
282 col[24] = ((a3 + b3) >> COL_SHIFT);
283 col[32] = ((a3 - b3) >> COL_SHIFT);
284 col[40] = ((a2 - b2) >> COL_SHIFT);
285 col[48] = ((a1 - b1) >> COL_SHIFT);
286 col[56] = ((a0 - b0) >> COL_SHIFT);
e7a972e1
MR
287}
288
88bd7fdc 289void FUNC(ff_simple_idct_put)(uint8_t *dest_, int line_size, int16_t *block)
e7a972e1
MR
290{
291 pixel *dest = (pixel *)dest_;
292 int i;
e7bcc5ba
MR
293
294 line_size /= sizeof(pixel);
295
2cc4f3b2 296 for (i = 0; i < 8; i++)
f78cd0c2 297 FUNC(idctRowCondDC)(block + i*8, 0);
e7a972e1 298
2cc4f3b2 299 for (i = 0; i < 8; i++)
e7a972e1
MR
300 FUNC(idctSparseColPut)(dest + i, line_size, block + i);
301}
302
88bd7fdc 303void FUNC(ff_simple_idct_add)(uint8_t *dest_, int line_size, int16_t *block)
e7a972e1
MR
304{
305 pixel *dest = (pixel *)dest_;
306 int i;
e7bcc5ba
MR
307
308 line_size /= sizeof(pixel);
309
2cc4f3b2 310 for (i = 0; i < 8; i++)
f78cd0c2 311 FUNC(idctRowCondDC)(block + i*8, 0);
e7a972e1 312
2cc4f3b2 313 for (i = 0; i < 8; i++)
e7a972e1
MR
314 FUNC(idctSparseColAdd)(dest + i, line_size, block + i);
315}
316
88bd7fdc 317void FUNC(ff_simple_idct)(int16_t *block)
e7a972e1
MR
318{
319 int i;
2cc4f3b2
MR
320
321 for (i = 0; i < 8; i++)
f78cd0c2 322 FUNC(idctRowCondDC)(block + i*8, 0);
e7a972e1 323
2cc4f3b2 324 for (i = 0; i < 8; i++)
e7a972e1
MR
325 FUNC(idctSparseCol)(block + i);
326}