idct8x8: Fix undefined negative shifts
[libav.git] / libavcodec / jfdctint_template.c
CommitLineData
0a72533e 1/*
0a72533e
MR
2 * This file is part of the Independent JPEG Group's software.
3 *
4 * The authors make NO WARRANTY or representation, either express or implied,
5 * with respect to this software, its quality, accuracy, merchantability, or
6 * fitness for a particular purpose. This software is provided "AS IS", and
7 * you, its user, assume the entire risk as to its quality and accuracy.
8 *
9 * This software is copyright (C) 1991-1996, Thomas G. Lane.
10 * All Rights Reserved except as specified below.
11 *
12 * Permission is hereby granted to use, copy, modify, and distribute this
13 * software (or portions thereof) for any purpose, without fee, subject to
14 * these conditions:
15 * (1) If any part of the source code for this software is distributed, then
16 * this README file must be included, with this copyright and no-warranty
17 * notice unaltered; and any additions, deletions, or changes to the original
18 * files must be clearly indicated in accompanying documentation.
19 * (2) If only executable code is distributed, then the accompanying
20 * documentation must state that "this software is based in part on the work
21 * of the Independent JPEG Group".
22 * (3) Permission for use of this software is granted only if the user accepts
23 * full responsibility for any undesirable consequences; the authors accept
24 * NO LIABILITY for damages of any kind.
25 *
26 * These conditions apply to any software derived from or based on the IJG
27 * code, not just to the unmodified library. If you use our work, you ought
28 * to acknowledge us.
29 *
30 * Permission is NOT granted for the use of any IJG author's name or company
31 * name in advertising or publicity relating to this software or products
32 * derived from it. This software may be referred to only as "the Independent
33 * JPEG Group's software".
34 *
35 * We specifically permit and encourage the use of this software as the basis
36 * of commercial products, provided that all warranty or liability claims are
37 * assumed by the product vendor.
38 *
39 * This file contains a slow-but-accurate integer implementation of the
40 * forward DCT (Discrete Cosine Transform).
41 *
42 * A 2-D DCT can be done by 1-D DCT on each row followed by 1-D DCT
43 * on each column. Direct algorithms are also available, but they are
44 * much more complex and seem not to be any faster when reduced to code.
45 *
46 * This implementation is based on an algorithm described in
47 * C. Loeffler, A. Ligtenberg and G. Moschytz, "Practical Fast 1-D DCT
48 * Algorithms with 11 Multiplications", Proc. Int'l. Conf. on Acoustics,
49 * Speech, and Signal Processing 1989 (ICASSP '89), pp. 988-991.
50 * The primary algorithm described there uses 11 multiplies and 29 adds.
51 * We use their alternate method with 12 multiplies and 32 adds.
52 * The advantage of this method is that no data path contains more than one
53 * multiplication; this allows a very simple and accurate implementation in
54 * scaled fixed-point arithmetic, with a minimal number of shifts.
55 */
56
57/**
58 * @file
59 * Independent JPEG Group's slow & accurate dct.
60 */
61
62#include "libavutil/common.h"
5d3d39c7 63#include "dct.h"
0a72533e
MR
64
65#include "bit_depth_template.c"
66
67#define DCTSIZE 8
68#define BITS_IN_JSAMPLE BIT_DEPTH
69#define GLOBAL(x) x
70#define RIGHT_SHIFT(x, n) ((x) >> (n))
71#define MULTIPLY16C16(var,const) ((var)*(const))
72
73#if 1 //def USE_ACCURATE_ROUNDING
74#define DESCALE(x,n) RIGHT_SHIFT((x) + (1 << ((n) - 1)), n)
75#else
76#define DESCALE(x,n) RIGHT_SHIFT(x, n)
77#endif
78
79
80/*
81 * This module is specialized to the case DCTSIZE = 8.
82 */
83
84#if DCTSIZE != 8
85#error "Sorry, this code only copes with 8x8 DCTs."
86#endif
87
88
89/*
90 * The poop on this scaling stuff is as follows:
91 *
92 * Each 1-D DCT step produces outputs which are a factor of sqrt(N)
93 * larger than the true DCT outputs. The final outputs are therefore
94 * a factor of N larger than desired; since N=8 this can be cured by
95 * a simple right shift at the end of the algorithm. The advantage of
96 * this arrangement is that we save two multiplications per 1-D DCT,
97 * because the y0 and y4 outputs need not be divided by sqrt(N).
98 * In the IJG code, this factor of 8 is removed by the quantization step
99 * (in jcdctmgr.c), NOT in this module.
100 *
101 * We have to do addition and subtraction of the integer inputs, which
102 * is no problem, and multiplication by fractional constants, which is
103 * a problem to do in integer arithmetic. We multiply all the constants
104 * by CONST_SCALE and convert them to integer constants (thus retaining
105 * CONST_BITS bits of precision in the constants). After doing a
106 * multiplication we have to divide the product by CONST_SCALE, with proper
107 * rounding, to produce the correct output. This division can be done
108 * cheaply as a right shift of CONST_BITS bits. We postpone shifting
109 * as long as possible so that partial sums can be added together with
110 * full fractional precision.
111 *
112 * The outputs of the first pass are scaled up by PASS1_BITS bits so that
113 * they are represented to better-than-integral precision. These outputs
114 * require BITS_IN_JSAMPLE + PASS1_BITS + 3 bits; this fits in a 16-bit word
115 * with the recommended scaling. (For 12-bit sample data, the intermediate
116 * array is int32_t anyway.)
117 *
118 * To avoid overflow of the 32-bit intermediate results in pass 2, we must
119 * have BITS_IN_JSAMPLE + CONST_BITS + PASS1_BITS <= 26. Error analysis
120 * shows that the values given below are the most effective.
121 */
122
123#undef CONST_BITS
124#undef PASS1_BITS
125#undef OUT_SHIFT
126
127#if BITS_IN_JSAMPLE == 8
128#define CONST_BITS 13
129#define PASS1_BITS 4 /* set this to 2 if 16x16 multiplies are faster */
130#define OUT_SHIFT PASS1_BITS
131#else
132#define CONST_BITS 13
133#define PASS1_BITS 1 /* lose a little precision to avoid overflow */
134#define OUT_SHIFT (PASS1_BITS + 1)
135#endif
136
137/* Some C compilers fail to reduce "FIX(constant)" at compile time, thus
138 * causing a lot of useless floating-point operations at run time.
139 * To get around this we use the following pre-calculated constants.
140 * If you change CONST_BITS you may want to add appropriate values.
141 * (With a reasonable C compiler, you can just rely on the FIX() macro...)
142 */
143
144#if CONST_BITS == 13
145#define FIX_0_298631336 ((int32_t) 2446) /* FIX(0.298631336) */
146#define FIX_0_390180644 ((int32_t) 3196) /* FIX(0.390180644) */
147#define FIX_0_541196100 ((int32_t) 4433) /* FIX(0.541196100) */
148#define FIX_0_765366865 ((int32_t) 6270) /* FIX(0.765366865) */
149#define FIX_0_899976223 ((int32_t) 7373) /* FIX(0.899976223) */
150#define FIX_1_175875602 ((int32_t) 9633) /* FIX(1.175875602) */
151#define FIX_1_501321110 ((int32_t) 12299) /* FIX(1.501321110) */
152#define FIX_1_847759065 ((int32_t) 15137) /* FIX(1.847759065) */
153#define FIX_1_961570560 ((int32_t) 16069) /* FIX(1.961570560) */
154#define FIX_2_053119869 ((int32_t) 16819) /* FIX(2.053119869) */
155#define FIX_2_562915447 ((int32_t) 20995) /* FIX(2.562915447) */
156#define FIX_3_072711026 ((int32_t) 25172) /* FIX(3.072711026) */
157#else
158#define FIX_0_298631336 FIX(0.298631336)
159#define FIX_0_390180644 FIX(0.390180644)
160#define FIX_0_541196100 FIX(0.541196100)
161#define FIX_0_765366865 FIX(0.765366865)
162#define FIX_0_899976223 FIX(0.899976223)
163#define FIX_1_175875602 FIX(1.175875602)
164#define FIX_1_501321110 FIX(1.501321110)
165#define FIX_1_847759065 FIX(1.847759065)
166#define FIX_1_961570560 FIX(1.961570560)
167#define FIX_2_053119869 FIX(2.053119869)
168#define FIX_2_562915447 FIX(2.562915447)
169#define FIX_3_072711026 FIX(3.072711026)
170#endif
171
172
173/* Multiply an int32_t variable by an int32_t constant to yield an int32_t result.
174 * For 8-bit samples with the recommended scaling, all the variable
175 * and constant values involved are no more than 16 bits wide, so a
176 * 16x16->32 bit multiply can be used instead of a full 32x32 multiply.
177 * For 12-bit samples, a full 32-bit multiplication will be needed.
178 */
179
180#if BITS_IN_JSAMPLE == 8 && CONST_BITS<=13 && PASS1_BITS<=2
181#define MULTIPLY(var,const) MULTIPLY16C16(var,const)
182#else
183#define MULTIPLY(var,const) ((var) * (const))
184#endif
185
186
88bd7fdc 187static av_always_inline void FUNC(row_fdct)(int16_t *data)
0a72533e
MR
188{
189 int tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
190 int tmp10, tmp11, tmp12, tmp13;
191 int z1, z2, z3, z4, z5;
88bd7fdc 192 int16_t *dataptr;
0a72533e
MR
193 int ctr;
194
195 /* Pass 1: process rows. */
196 /* Note results are scaled up by sqrt(8) compared to a true DCT; */
197 /* furthermore, we scale the results by 2**PASS1_BITS. */
198
199 dataptr = data;
200 for (ctr = DCTSIZE-1; ctr >= 0; ctr--) {
201 tmp0 = dataptr[0] + dataptr[7];
202 tmp7 = dataptr[0] - dataptr[7];
203 tmp1 = dataptr[1] + dataptr[6];
204 tmp6 = dataptr[1] - dataptr[6];
205 tmp2 = dataptr[2] + dataptr[5];
206 tmp5 = dataptr[2] - dataptr[5];
207 tmp3 = dataptr[3] + dataptr[4];
208 tmp4 = dataptr[3] - dataptr[4];
209
210 /* Even part per LL&M figure 1 --- note that published figure is faulty;
211 * rotator "sqrt(2)*c1" should be "sqrt(2)*c6".
212 */
213
214 tmp10 = tmp0 + tmp3;
215 tmp13 = tmp0 - tmp3;
216 tmp11 = tmp1 + tmp2;
217 tmp12 = tmp1 - tmp2;
218
1389b4c1
KBA
219 dataptr[0] = (int16_t) ((tmp10 + tmp11) * (1 << PASS1_BITS));
220 dataptr[4] = (int16_t) ((tmp10 - tmp11) * (1 << PASS1_BITS));
0a72533e
MR
221
222 z1 = MULTIPLY(tmp12 + tmp13, FIX_0_541196100);
88bd7fdc 223 dataptr[2] = (int16_t) DESCALE(z1 + MULTIPLY(tmp13, FIX_0_765366865),
0a72533e 224 CONST_BITS-PASS1_BITS);
88bd7fdc 225 dataptr[6] = (int16_t) DESCALE(z1 + MULTIPLY(tmp12, - FIX_1_847759065),
0a72533e
MR
226 CONST_BITS-PASS1_BITS);
227
228 /* Odd part per figure 8 --- note paper omits factor of sqrt(2).
229 * cK represents cos(K*pi/16).
230 * i0..i3 in the paper are tmp4..tmp7 here.
231 */
232
233 z1 = tmp4 + tmp7;
234 z2 = tmp5 + tmp6;
235 z3 = tmp4 + tmp6;
236 z4 = tmp5 + tmp7;
237 z5 = MULTIPLY(z3 + z4, FIX_1_175875602); /* sqrt(2) * c3 */
238
239 tmp4 = MULTIPLY(tmp4, FIX_0_298631336); /* sqrt(2) * (-c1+c3+c5-c7) */
240 tmp5 = MULTIPLY(tmp5, FIX_2_053119869); /* sqrt(2) * ( c1+c3-c5+c7) */
241 tmp6 = MULTIPLY(tmp6, FIX_3_072711026); /* sqrt(2) * ( c1+c3+c5-c7) */
242 tmp7 = MULTIPLY(tmp7, FIX_1_501321110); /* sqrt(2) * ( c1+c3-c5-c7) */
243 z1 = MULTIPLY(z1, - FIX_0_899976223); /* sqrt(2) * (c7-c3) */
244 z2 = MULTIPLY(z2, - FIX_2_562915447); /* sqrt(2) * (-c1-c3) */
245 z3 = MULTIPLY(z3, - FIX_1_961570560); /* sqrt(2) * (-c3-c5) */
246 z4 = MULTIPLY(z4, - FIX_0_390180644); /* sqrt(2) * (c5-c3) */
247
248 z3 += z5;
249 z4 += z5;
250
88bd7fdc
DB
251 dataptr[7] = (int16_t) DESCALE(tmp4 + z1 + z3, CONST_BITS-PASS1_BITS);
252 dataptr[5] = (int16_t) DESCALE(tmp5 + z2 + z4, CONST_BITS-PASS1_BITS);
253 dataptr[3] = (int16_t) DESCALE(tmp6 + z2 + z3, CONST_BITS-PASS1_BITS);
254 dataptr[1] = (int16_t) DESCALE(tmp7 + z1 + z4, CONST_BITS-PASS1_BITS);
0a72533e
MR
255
256 dataptr += DCTSIZE; /* advance pointer to next row */
257 }
258}
259
260/*
261 * Perform the forward DCT on one block of samples.
262 */
263
264GLOBAL(void)
88bd7fdc 265FUNC(ff_jpeg_fdct_islow)(int16_t *data)
0a72533e
MR
266{
267 int tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
268 int tmp10, tmp11, tmp12, tmp13;
269 int z1, z2, z3, z4, z5;
88bd7fdc 270 int16_t *dataptr;
0a72533e
MR
271 int ctr;
272
273 FUNC(row_fdct)(data);
274
275 /* Pass 2: process columns.
276 * We remove the PASS1_BITS scaling, but leave the results scaled up
277 * by an overall factor of 8.
278 */
279
280 dataptr = data;
281 for (ctr = DCTSIZE-1; ctr >= 0; ctr--) {
282 tmp0 = dataptr[DCTSIZE*0] + dataptr[DCTSIZE*7];
283 tmp7 = dataptr[DCTSIZE*0] - dataptr[DCTSIZE*7];
284 tmp1 = dataptr[DCTSIZE*1] + dataptr[DCTSIZE*6];
285 tmp6 = dataptr[DCTSIZE*1] - dataptr[DCTSIZE*6];
286 tmp2 = dataptr[DCTSIZE*2] + dataptr[DCTSIZE*5];
287 tmp5 = dataptr[DCTSIZE*2] - dataptr[DCTSIZE*5];
288 tmp3 = dataptr[DCTSIZE*3] + dataptr[DCTSIZE*4];
289 tmp4 = dataptr[DCTSIZE*3] - dataptr[DCTSIZE*4];
290
291 /* Even part per LL&M figure 1 --- note that published figure is faulty;
292 * rotator "sqrt(2)*c1" should be "sqrt(2)*c6".
293 */
294
295 tmp10 = tmp0 + tmp3;
296 tmp13 = tmp0 - tmp3;
297 tmp11 = tmp1 + tmp2;
298 tmp12 = tmp1 - tmp2;
299
300 dataptr[DCTSIZE*0] = DESCALE(tmp10 + tmp11, OUT_SHIFT);
301 dataptr[DCTSIZE*4] = DESCALE(tmp10 - tmp11, OUT_SHIFT);
302
303 z1 = MULTIPLY(tmp12 + tmp13, FIX_0_541196100);
304 dataptr[DCTSIZE*2] = DESCALE(z1 + MULTIPLY(tmp13, FIX_0_765366865),
305 CONST_BITS + OUT_SHIFT);
306 dataptr[DCTSIZE*6] = DESCALE(z1 + MULTIPLY(tmp12, - FIX_1_847759065),
307 CONST_BITS + OUT_SHIFT);
308
309 /* Odd part per figure 8 --- note paper omits factor of sqrt(2).
310 * cK represents cos(K*pi/16).
311 * i0..i3 in the paper are tmp4..tmp7 here.
312 */
313
314 z1 = tmp4 + tmp7;
315 z2 = tmp5 + tmp6;
316 z3 = tmp4 + tmp6;
317 z4 = tmp5 + tmp7;
318 z5 = MULTIPLY(z3 + z4, FIX_1_175875602); /* sqrt(2) * c3 */
319
320 tmp4 = MULTIPLY(tmp4, FIX_0_298631336); /* sqrt(2) * (-c1+c3+c5-c7) */
321 tmp5 = MULTIPLY(tmp5, FIX_2_053119869); /* sqrt(2) * ( c1+c3-c5+c7) */
322 tmp6 = MULTIPLY(tmp6, FIX_3_072711026); /* sqrt(2) * ( c1+c3+c5-c7) */
323 tmp7 = MULTIPLY(tmp7, FIX_1_501321110); /* sqrt(2) * ( c1+c3-c5-c7) */
324 z1 = MULTIPLY(z1, - FIX_0_899976223); /* sqrt(2) * (c7-c3) */
325 z2 = MULTIPLY(z2, - FIX_2_562915447); /* sqrt(2) * (-c1-c3) */
326 z3 = MULTIPLY(z3, - FIX_1_961570560); /* sqrt(2) * (-c3-c5) */
327 z4 = MULTIPLY(z4, - FIX_0_390180644); /* sqrt(2) * (c5-c3) */
328
329 z3 += z5;
330 z4 += z5;
331
332 dataptr[DCTSIZE*7] = DESCALE(tmp4 + z1 + z3, CONST_BITS + OUT_SHIFT);
333 dataptr[DCTSIZE*5] = DESCALE(tmp5 + z2 + z4, CONST_BITS + OUT_SHIFT);
334 dataptr[DCTSIZE*3] = DESCALE(tmp6 + z2 + z3, CONST_BITS + OUT_SHIFT);
335 dataptr[DCTSIZE*1] = DESCALE(tmp7 + z1 + z4, CONST_BITS + OUT_SHIFT);
336
337 dataptr++; /* advance pointer to next column */
338 }
339}
340
341/*
342 * The secret of DCT2-4-8 is really simple -- you do the usual 1-DCT
da9cea77 343 * on the rows and then, instead of doing even and odd, part on the columns
0a72533e
MR
344 * you do even part two times.
345 */
346GLOBAL(void)
88bd7fdc 347FUNC(ff_fdct248_islow)(int16_t *data)
0a72533e
MR
348{
349 int tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
350 int tmp10, tmp11, tmp12, tmp13;
351 int z1;
88bd7fdc 352 int16_t *dataptr;
0a72533e
MR
353 int ctr;
354
355 FUNC(row_fdct)(data);
356
357 /* Pass 2: process columns.
358 * We remove the PASS1_BITS scaling, but leave the results scaled up
359 * by an overall factor of 8.
360 */
361
362 dataptr = data;
363 for (ctr = DCTSIZE-1; ctr >= 0; ctr--) {
364 tmp0 = dataptr[DCTSIZE*0] + dataptr[DCTSIZE*1];
365 tmp1 = dataptr[DCTSIZE*2] + dataptr[DCTSIZE*3];
366 tmp2 = dataptr[DCTSIZE*4] + dataptr[DCTSIZE*5];
367 tmp3 = dataptr[DCTSIZE*6] + dataptr[DCTSIZE*7];
368 tmp4 = dataptr[DCTSIZE*0] - dataptr[DCTSIZE*1];
369 tmp5 = dataptr[DCTSIZE*2] - dataptr[DCTSIZE*3];
370 tmp6 = dataptr[DCTSIZE*4] - dataptr[DCTSIZE*5];
371 tmp7 = dataptr[DCTSIZE*6] - dataptr[DCTSIZE*7];
372
373 tmp10 = tmp0 + tmp3;
374 tmp11 = tmp1 + tmp2;
375 tmp12 = tmp1 - tmp2;
376 tmp13 = tmp0 - tmp3;
377
378 dataptr[DCTSIZE*0] = DESCALE(tmp10 + tmp11, OUT_SHIFT);
379 dataptr[DCTSIZE*4] = DESCALE(tmp10 - tmp11, OUT_SHIFT);
380
381 z1 = MULTIPLY(tmp12 + tmp13, FIX_0_541196100);
382 dataptr[DCTSIZE*2] = DESCALE(z1 + MULTIPLY(tmp13, FIX_0_765366865),
383 CONST_BITS+OUT_SHIFT);
384 dataptr[DCTSIZE*6] = DESCALE(z1 + MULTIPLY(tmp12, - FIX_1_847759065),
385 CONST_BITS+OUT_SHIFT);
386
387 tmp10 = tmp4 + tmp7;
388 tmp11 = tmp5 + tmp6;
389 tmp12 = tmp5 - tmp6;
390 tmp13 = tmp4 - tmp7;
391
392 dataptr[DCTSIZE*1] = DESCALE(tmp10 + tmp11, OUT_SHIFT);
393 dataptr[DCTSIZE*5] = DESCALE(tmp10 - tmp11, OUT_SHIFT);
394
395 z1 = MULTIPLY(tmp12 + tmp13, FIX_0_541196100);
396 dataptr[DCTSIZE*3] = DESCALE(z1 + MULTIPLY(tmp13, FIX_0_765366865),
397 CONST_BITS + OUT_SHIFT);
398 dataptr[DCTSIZE*7] = DESCALE(z1 + MULTIPLY(tmp12, - FIX_1_847759065),
399 CONST_BITS + OUT_SHIFT);
400
401 dataptr++; /* advance pointer to next column */
402 }
403}