mpegvideo_enc: drop support for reusing the input motion vectors.
[libav.git] / libavcodec / vp56.c
CommitLineData
6168781f 1/*
5ce117c3
AJ
2 * Copyright (C) 2006 Aurelien Jacobs <aurel@gnuage.org>
3 *
2912e87a 4 * This file is part of Libav.
b78e7197 5 *
2912e87a 6 * Libav is free software; you can redistribute it and/or
5ce117c3
AJ
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
2912e87a 11 * Libav is distributed in the hope that it will be useful,
5ce117c3
AJ
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
2912e87a 17 * License along with Libav; if not, write to the Free Software
e5a389a1 18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
5ce117c3
AJ
19 */
20
6168781f
DB
21/**
22 * @file
23 * VP5 and VP6 compatible video decoder (common features)
24 */
25
5ce117c3 26#include "avcodec.h"
91fc2cf1 27#include "bytestream.h"
594d4d5d 28#include "internal.h"
79dad2a9 29#include "h264chroma.h"
5ce117c3
AJ
30#include "vp56.h"
31#include "vp56data.h"
32
33
d9504970 34void ff_vp56_init_dequant(VP56Context *s, int quantizer)
5ce117c3
AJ
35{
36 s->quantizer = quantizer;
37 s->dequant_dc = vp56_dc_dequant[quantizer] << 2;
38 s->dequant_ac = vp56_ac_dequant[quantizer] << 2;
800274f8 39 memset(s->qscale_table, quantizer, s->mb_width);
5ce117c3
AJ
40}
41
3d52bca6
AJ
42static int vp56_get_vectors_predictors(VP56Context *s, int row, int col,
43 VP56Frame ref_frame)
5ce117c3
AJ
44{
45 int nb_pred = 0;
3d52bca6 46 VP56mv vect[2] = {{0,0}, {0,0}};
5ce117c3 47 int pos, offset;
3d52bca6 48 VP56mv mvp;
5ce117c3
AJ
49
50 for (pos=0; pos<12; pos++) {
51 mvp.x = col + vp56_candidate_predictor_pos[pos][0];
52 mvp.y = row + vp56_candidate_predictor_pos[pos][1];
53 if (mvp.x < 0 || mvp.x >= s->mb_width ||
54 mvp.y < 0 || mvp.y >= s->mb_height)
55 continue;
56 offset = mvp.x + s->mb_width*mvp.y;
57
58 if (vp56_reference_frame[s->macroblocks[offset].type] != ref_frame)
59 continue;
d120e402
AJ
60 if ((s->macroblocks[offset].mv.x == vect[0].x &&
61 s->macroblocks[offset].mv.y == vect[0].y) ||
5ce117c3
AJ
62 (s->macroblocks[offset].mv.x == 0 &&
63 s->macroblocks[offset].mv.y == 0))
64 continue;
65
d120e402 66 vect[nb_pred++] = s->macroblocks[offset].mv;
5ce117c3
AJ
67 if (nb_pred > 1) {
68 nb_pred = -1;
69 break;
70 }
71 s->vector_candidate_pos = pos;
72 }
73
d120e402
AJ
74 s->vector_candidate[0] = vect[0];
75 s->vector_candidate[1] = vect[1];
5ce117c3
AJ
76
77 return nb_pred+1;
78}
79
3d52bca6 80static void vp56_parse_mb_type_models(VP56Context *s)
5ce117c3 81{
3d52bca6 82 VP56RangeCoder *c = &s->c;
d887151d 83 VP56Model *model = s->modelp;
5ce117c3
AJ
84 int i, ctx, type;
85
86 for (ctx=0; ctx<3; ctx++) {
87 if (vp56_rac_get_prob(c, 174)) {
88 int idx = vp56_rac_gets(c, 4);
247df384
AJ
89 memcpy(model->mb_types_stats[ctx],
90 vp56_pre_def_mb_type_stats[idx][ctx],
91 sizeof(model->mb_types_stats[ctx]));
5ce117c3
AJ
92 }
93 if (vp56_rac_get_prob(c, 254)) {
94 for (type=0; type<10; type++) {
95 for(i=0; i<2; i++) {
96 if (vp56_rac_get_prob(c, 205)) {
97 int delta, sign = vp56_rac_get(c);
98
99 delta = vp56_rac_get_tree(c, vp56_pmbtm_tree,
100 vp56_mb_type_model_model);
101 if (!delta)
102 delta = 4 * vp56_rac_gets(c, 7);
247df384 103 model->mb_types_stats[ctx][type][i] += (delta ^ -sign) + sign;
5ce117c3
AJ
104 }
105 }
106 }
107 }
108 }
109
110 /* compute MB type probability tables based on previous MB type */
111 for (ctx=0; ctx<3; ctx++) {
112 int p[10];
113
114 for (type=0; type<10; type++)
247df384 115 p[type] = 100 * model->mb_types_stats[ctx][type][1];
5ce117c3
AJ
116
117 for (type=0; type<10; type++) {
118 int p02, p34, p0234, p17, p56, p89, p5689, p156789;
119
120 /* conservative MB type probability */
247df384 121 model->mb_type[ctx][type][0] = 255 - (255 * model->mb_types_stats[ctx][type][0]) / (1 + model->mb_types_stats[ctx][type][0] + model->mb_types_stats[ctx][type][1]);
5ce117c3
AJ
122
123 p[type] = 0; /* same MB type => weight is null */
124
125 /* binary tree parsing probabilities */
126 p02 = p[0] + p[2];
127 p34 = p[3] + p[4];
128 p0234 = p02 + p34;
129 p17 = p[1] + p[7];
130 p56 = p[5] + p[6];
131 p89 = p[8] + p[9];
132 p5689 = p56 + p89;
133 p156789 = p17 + p5689;
134
247df384
AJ
135 model->mb_type[ctx][type][1] = 1 + 255 * p0234/(1+p0234+p156789);
136 model->mb_type[ctx][type][2] = 1 + 255 * p02 / (1+p0234);
137 model->mb_type[ctx][type][3] = 1 + 255 * p17 / (1+p156789);
138 model->mb_type[ctx][type][4] = 1 + 255 * p[0] / (1+p02);
139 model->mb_type[ctx][type][5] = 1 + 255 * p[3] / (1+p34);
140 model->mb_type[ctx][type][6] = 1 + 255 * p[1] / (1+p17);
141 model->mb_type[ctx][type][7] = 1 + 255 * p56 / (1+p5689);
142 model->mb_type[ctx][type][8] = 1 + 255 * p[5] / (1+p56);
143 model->mb_type[ctx][type][9] = 1 + 255 * p[8] / (1+p89);
5ce117c3
AJ
144
145 /* restore initial value */
247df384 146 p[type] = 100 * model->mb_types_stats[ctx][type][1];
5ce117c3
AJ
147 }
148 }
149}
150
3d52bca6 151static VP56mb vp56_parse_mb_type(VP56Context *s,
76025d91 152 VP56mb prev_type, int ctx)
5ce117c3 153{
247df384 154 uint8_t *mb_type_model = s->modelp->mb_type[ctx][prev_type];
3d52bca6 155 VP56RangeCoder *c = &s->c;
5ce117c3
AJ
156
157 if (vp56_rac_get_prob(c, mb_type_model[0]))
158 return prev_type;
159 else
160 return vp56_rac_get_tree(c, vp56_pmbt_tree, mb_type_model);
161}
162
3d52bca6 163static void vp56_decode_4mv(VP56Context *s, int row, int col)
5ce117c3 164{
3d52bca6 165 VP56mv mv = {0,0};
5ce117c3
AJ
166 int type[4];
167 int b;
168
169 /* parse each block type */
170 for (b=0; b<4; b++) {
171 type[b] = vp56_rac_gets(&s->c, 2);
172 if (type[b])
173 type[b]++; /* only returns 0, 2, 3 or 4 (all INTER_PF) */
174 }
175
176 /* get vectors */
177 for (b=0; b<4; b++) {
178 switch (type[b]) {
179 case VP56_MB_INTER_NOVEC_PF:
3d52bca6 180 s->mv[b] = (VP56mv) {0,0};
5ce117c3
AJ
181 break;
182 case VP56_MB_INTER_DELTA_PF:
183 s->parse_vector_adjustment(s, &s->mv[b]);
184 break;
185 case VP56_MB_INTER_V1_PF:
186 s->mv[b] = s->vector_candidate[0];
187 break;
188 case VP56_MB_INTER_V2_PF:
189 s->mv[b] = s->vector_candidate[1];
190 break;
191 }
192 mv.x += s->mv[b].x;
193 mv.y += s->mv[b].y;
194 }
195
196 /* this is the one selected for the whole MB for prediction */
197 s->macroblocks[row * s->mb_width + col].mv = s->mv[3];
198
199 /* chroma vectors are average luma vectors */
36ef5369 200 if (s->avctx->codec->id == AV_CODEC_ID_VP5) {
5ce117c3
AJ
201 s->mv[4].x = s->mv[5].x = RSHIFT(mv.x,2);
202 s->mv[4].y = s->mv[5].y = RSHIFT(mv.y,2);
203 } else {
3d52bca6 204 s->mv[4] = s->mv[5] = (VP56mv) {mv.x/4, mv.y/4};
5ce117c3
AJ
205 }
206}
207
3d52bca6 208static VP56mb vp56_decode_mv(VP56Context *s, int row, int col)
5ce117c3 209{
3d52bca6 210 VP56mv *mv, vect = {0,0};
5ce117c3
AJ
211 int ctx, b;
212
213 ctx = vp56_get_vectors_predictors(s, row, col, VP56_FRAME_PREVIOUS);
214 s->mb_type = vp56_parse_mb_type(s, s->mb_type, ctx);
215 s->macroblocks[row * s->mb_width + col].type = s->mb_type;
216
217 switch (s->mb_type) {
218 case VP56_MB_INTER_V1_PF:
219 mv = &s->vector_candidate[0];
220 break;
221
222 case VP56_MB_INTER_V2_PF:
223 mv = &s->vector_candidate[1];
224 break;
225
226 case VP56_MB_INTER_V1_GF:
227 vp56_get_vectors_predictors(s, row, col, VP56_FRAME_GOLDEN);
228 mv = &s->vector_candidate[0];
229 break;
230
231 case VP56_MB_INTER_V2_GF:
232 vp56_get_vectors_predictors(s, row, col, VP56_FRAME_GOLDEN);
233 mv = &s->vector_candidate[1];
234 break;
235
236 case VP56_MB_INTER_DELTA_PF:
d120e402
AJ
237 s->parse_vector_adjustment(s, &vect);
238 mv = &vect;
5ce117c3
AJ
239 break;
240
241 case VP56_MB_INTER_DELTA_GF:
242 vp56_get_vectors_predictors(s, row, col, VP56_FRAME_GOLDEN);
d120e402
AJ
243 s->parse_vector_adjustment(s, &vect);
244 mv = &vect;
5ce117c3
AJ
245 break;
246
247 case VP56_MB_INTER_4V:
248 vp56_decode_4mv(s, row, col);
249 return s->mb_type;
250
251 default:
d120e402 252 mv = &vect;
5ce117c3
AJ
253 break;
254 }
255
256 s->macroblocks[row*s->mb_width + col].mv = *mv;
257
258 /* same vector for all blocks */
259 for (b=0; b<6; b++)
260 s->mv[b] = *mv;
261
262 return s->mb_type;
263}
264
3d52bca6 265static void vp56_add_predictors_dc(VP56Context *s, VP56Frame ref_frame)
5ce117c3
AJ
266{
267 int idx = s->scantable.permutated[0];
d7af6a9d 268 int b;
5ce117c3 269
d7af6a9d 270 for (b=0; b<6; b++) {
3d52bca6 271 VP56RefDc *ab = &s->above_blocks[s->above_block_idx[b]];
d1b357d7 272 VP56RefDc *lb = &s->left_block[ff_vp56_b6to4[b]];
5ce117c3
AJ
273 int count = 0;
274 int dc = 0;
7ecae905 275 int i;
5ce117c3
AJ
276
277 if (ref_frame == lb->ref_frame) {
278 dc += lb->dc_coeff;
279 count++;
280 }
281 if (ref_frame == ab->ref_frame) {
282 dc += ab->dc_coeff;
283 count++;
284 }
36ef5369 285 if (s->avctx->codec->id == AV_CODEC_ID_VP5)
7ecae905
AJ
286 for (i=0; i<2; i++)
287 if (count < 2 && ref_frame == ab[-1+2*i].ref_frame) {
288 dc += ab[-1+2*i].dc_coeff;
289 count++;
290 }
5ce117c3 291 if (count == 0)
d1b357d7 292 dc = s->prev_dc[ff_vp56_b2p[b]][ref_frame];
5ce117c3
AJ
293 else if (count == 2)
294 dc /= 2;
295
d7af6a9d 296 s->block_coeff[b][idx] += dc;
d1b357d7 297 s->prev_dc[ff_vp56_b2p[b]][ref_frame] = s->block_coeff[b][idx];
d7af6a9d 298 ab->dc_coeff = s->block_coeff[b][idx];
5ce117c3 299 ab->ref_frame = ref_frame;
d7af6a9d 300 lb->dc_coeff = s->block_coeff[b][idx];
5ce117c3 301 lb->ref_frame = ref_frame;
d7af6a9d 302 s->block_coeff[b][idx] *= s->dequant_dc;
5ce117c3
AJ
303 }
304}
305
3d52bca6 306static void vp56_deblock_filter(VP56Context *s, uint8_t *yuv,
5ce117c3
AJ
307 int stride, int dx, int dy)
308{
309 int t = vp56_filter_threshold[s->quantizer];
5e1ba34b
MR
310 if (dx) s->vp56dsp.edge_filter_hor(yuv + 10-dx , stride, t);
311 if (dy) s->vp56dsp.edge_filter_ver(yuv + stride*(10-dy), stride, t);
5ce117c3
AJ
312}
313
3d52bca6 314static void vp56_mc(VP56Context *s, int b, int plane, uint8_t *src,
5ce117c3
AJ
315 int stride, int x, int y)
316{
704a2881 317 uint8_t *dst=s->framep[VP56_FRAME_CURRENT]->data[plane]+s->block_offset[b];
5ce117c3
AJ
318 uint8_t *src_block;
319 int src_offset;
320 int overlap_offset = 0;
321 int mask = s->vp56_coord_div[b] - 1;
322 int deblock_filtering = s->deblock_filtering;
323 int dx;
324 int dy;
325
326 if (s->avctx->skip_loop_filter >= AVDISCARD_ALL ||
327 (s->avctx->skip_loop_filter >= AVDISCARD_NONKEY
704a2881 328 && !s->framep[VP56_FRAME_CURRENT]->key_frame))
5ce117c3
AJ
329 deblock_filtering = 0;
330
331 dx = s->mv[b].x / s->vp56_coord_div[b];
332 dy = s->mv[b].y / s->vp56_coord_div[b];
333
334 if (b >= 4) {
335 x /= 2;
336 y /= 2;
337 }
338 x += dx - 2;
339 y += dy - 2;
340
341 if (x<0 || x+12>=s->plane_width[plane] ||
342 y<0 || y+12>=s->plane_height[plane]) {
8c53d39e 343 s->vdsp.emulated_edge_mc(s->edge_emu_buffer,
5ce117c3
AJ
344 src + s->block_offset[b] + (dy-2)*stride + (dx-2),
345 stride, 12, 12, x, y,
346 s->plane_width[plane],
347 s->plane_height[plane]);
348 src_block = s->edge_emu_buffer;
349 src_offset = 2 + 2*stride;
350 } else if (deblock_filtering) {
a8678a3a
AJ
351 /* only need a 12x12 block, but there is no such dsp function, */
352 /* so copy a 16x12 block */
353 s->dsp.put_pixels_tab[0][0](s->edge_emu_buffer,
354 src + s->block_offset[b] + (dy-2)*stride + (dx-2),
355 stride, 12);
5ce117c3
AJ
356 src_block = s->edge_emu_buffer;
357 src_offset = 2 + 2*stride;
358 } else {
359 src_block = src;
360 src_offset = s->block_offset[b] + dy*stride + dx;
361 }
362
363 if (deblock_filtering)
364 vp56_deblock_filter(s, src_block, stride, dx&7, dy&7);
365
366 if (s->mv[b].x & mask)
367 overlap_offset += (s->mv[b].x > 0) ? 1 : -1;
368 if (s->mv[b].y & mask)
369 overlap_offset += (s->mv[b].y > 0) ? stride : -stride;
370
371 if (overlap_offset) {
372 if (s->filter)
373 s->filter(s, dst, src_block, src_offset, src_offset+overlap_offset,
374 stride, s->mv[b], mask, s->filter_selection, b<4);
375 else
4a73fbd9
RB
376 s->vp3dsp.put_no_rnd_pixels_l2(dst, src_block+src_offset,
377 src_block+src_offset+overlap_offset,
378 stride, 8);
5ce117c3
AJ
379 } else {
380 s->dsp.put_pixels_tab[1][0](dst, src_block+src_offset, stride, 8);
381 }
382}
383
3d52bca6 384static void vp56_decode_mb(VP56Context *s, int row, int col, int is_alpha)
5ce117c3
AJ
385{
386 AVFrame *frame_current, *frame_ref;
3d52bca6
AJ
387 VP56mb mb_type;
388 VP56Frame ref_frame;
442b145a 389 int b, ab, b_max, plane, off;
5ce117c3 390
704a2881 391 if (s->framep[VP56_FRAME_CURRENT]->key_frame)
5ce117c3
AJ
392 mb_type = VP56_MB_INTRA;
393 else
394 mb_type = vp56_decode_mv(s, row, col);
395 ref_frame = vp56_reference_frame[mb_type];
396
5ce117c3
AJ
397 s->parse_coeff(s);
398
399 vp56_add_predictors_dc(s, ref_frame);
400
704a2881
AJ
401 frame_current = s->framep[VP56_FRAME_CURRENT];
402 frame_ref = s->framep[ref_frame];
0ec6d6e9
LA
403 if (mb_type != VP56_MB_INTRA && !frame_ref->data[0])
404 return;
5ce117c3 405
91fc2cf1
AJ
406 ab = 6*is_alpha;
407 b_max = 6 - 2*is_alpha;
408
5ce117c3
AJ
409 switch (mb_type) {
410 case VP56_MB_INTRA:
91fc2cf1 411 for (b=0; b<b_max; b++) {
d1b357d7 412 plane = ff_vp56_b2p[b+ab];
28f9ab70 413 s->vp3dsp.idct_put(frame_current->data[plane] + s->block_offset[b],
442b145a 414 s->stride[plane], s->block_coeff[b]);
5ce117c3
AJ
415 }
416 break;
417
418 case VP56_MB_INTER_NOVEC_PF:
419 case VP56_MB_INTER_NOVEC_GF:
91fc2cf1 420 for (b=0; b<b_max; b++) {
d1b357d7 421 plane = ff_vp56_b2p[b+ab];
5ce117c3 422 off = s->block_offset[b];
442b145a
AJ
423 s->dsp.put_pixels_tab[1][0](frame_current->data[plane] + off,
424 frame_ref->data[plane] + off,
425 s->stride[plane], 8);
28f9ab70 426 s->vp3dsp.idct_add(frame_current->data[plane] + off,
442b145a 427 s->stride[plane], s->block_coeff[b]);
5ce117c3
AJ
428 }
429 break;
430
431 case VP56_MB_INTER_DELTA_PF:
432 case VP56_MB_INTER_V1_PF:
433 case VP56_MB_INTER_V2_PF:
434 case VP56_MB_INTER_DELTA_GF:
435 case VP56_MB_INTER_4V:
436 case VP56_MB_INTER_V1_GF:
437 case VP56_MB_INTER_V2_GF:
91fc2cf1 438 for (b=0; b<b_max; b++) {
5ce117c3
AJ
439 int x_off = b==1 || b==3 ? 8 : 0;
440 int y_off = b==2 || b==3 ? 8 : 0;
d1b357d7 441 plane = ff_vp56_b2p[b+ab];
442b145a 442 vp56_mc(s, b, plane, frame_ref->data[plane], s->stride[plane],
5ce117c3 443 16*col+x_off, 16*row+y_off);
28f9ab70 444 s->vp3dsp.idct_add(frame_current->data[plane] + s->block_offset[b],
442b145a 445 s->stride[plane], s->block_coeff[b]);
5ce117c3
AJ
446 }
447 break;
448 }
f859678f
RB
449
450 if (is_alpha) {
451 s->block_coeff[4][0] = 0;
452 s->block_coeff[5][0] = 0;
453 }
5ce117c3
AJ
454}
455
1457516f 456static int vp56_size_changed(AVCodecContext *avctx)
5ce117c3 457{
3d52bca6 458 VP56Context *s = avctx->priv_data;
704a2881 459 int stride = s->framep[VP56_FRAME_CURRENT]->linesize[0];
5ce117c3
AJ
460 int i;
461
91fc2cf1 462 s->plane_width[0] = s->plane_width[3] = avctx->coded_width;
1457516f 463 s->plane_width[1] = s->plane_width[2] = avctx->coded_width/2;
91fc2cf1 464 s->plane_height[0] = s->plane_height[3] = avctx->coded_height;
1457516f 465 s->plane_height[1] = s->plane_height[2] = avctx->coded_height/2;
5ce117c3 466
91fc2cf1 467 for (i=0; i<4; i++)
704a2881 468 s->stride[i] = s->flip * s->framep[VP56_FRAME_CURRENT]->linesize[i];
5ce117c3 469
1457516f
AJ
470 s->mb_width = (avctx->coded_width +15) / 16;
471 s->mb_height = (avctx->coded_height+15) / 16;
5ce117c3
AJ
472
473 if (s->mb_width > 1000 || s->mb_height > 1000) {
3d09d001 474 avcodec_set_dimensions(avctx, 0, 0);
5ce117c3
AJ
475 av_log(avctx, AV_LOG_ERROR, "picture too big\n");
476 return -1;
477 }
478
800274f8 479 s->qscale_table = av_realloc(s->qscale_table, s->mb_width);
5ce117c3
AJ
480 s->above_blocks = av_realloc(s->above_blocks,
481 (4*s->mb_width+6) * sizeof(*s->above_blocks));
482 s->macroblocks = av_realloc(s->macroblocks,
483 s->mb_width*s->mb_height*sizeof(*s->macroblocks));
a8678a3a
AJ
484 av_free(s->edge_emu_buffer_alloc);
485 s->edge_emu_buffer_alloc = av_malloc(16*stride);
5ce117c3
AJ
486 s->edge_emu_buffer = s->edge_emu_buffer_alloc;
487 if (s->flip < 0)
488 s->edge_emu_buffer += 15 * stride;
489
490 return 0;
491}
492
df9b9567 493int ff_vp56_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
b6356d97 494 AVPacket *avpkt)
5ce117c3 495{
012f9308 496 const uint8_t *buf = avpkt->data;
3d52bca6 497 VP56Context *s = avctx->priv_data;
704a2881 498 AVFrame *const p = s->framep[VP56_FRAME_CURRENT];
012f9308 499 int remaining_buf_size = avpkt->size;
ed761067 500 int is_alpha, av_uninit(alpha_offset);
91fc2cf1
AJ
501
502 if (s->has_alpha) {
68a4d349
LA
503 if (remaining_buf_size < 3)
504 return -1;
91fc2cf1 505 alpha_offset = bytestream_get_be24(&buf);
5210529e 506 remaining_buf_size -= 3;
68a4d349
LA
507 if (remaining_buf_size < alpha_offset)
508 return -1;
91fc2cf1
AJ
509 }
510
511 for (is_alpha=0; is_alpha < 1+s->has_alpha; is_alpha++) {
f62a2b61
AJ
512 int mb_row, mb_col, mb_row_flip, mb_offset = 0;
513 int block, y, uv, stride_y, stride_uv;
514 int golden_frame = 0;
515 int res;
5ce117c3 516
f62a2b61 517 s->modelp = &s->models[is_alpha];
247df384 518
5210529e 519 res = s->parse_header(s, buf, remaining_buf_size, &golden_frame);
f33b5ba6
LB
520 if (res < 0) {
521 int i;
522 for (i = 0; i < 4; i++) {
523 if (s->frames[i].data[0])
524 avctx->release_buffer(avctx, &s->frames[i]);
525 }
bb675d3a 526 return res;
f33b5ba6 527 }
5ce117c3 528
bb675d3a 529 if (res == VP56_SIZE_CHANGE) {
3d09d001
LA
530 int i;
531 for (i = 0; i < 4; i++) {
532 if (s->frames[i].data[0])
533 avctx->release_buffer(avctx, &s->frames[i]);
534 }
535 if (is_alpha) {
536 avcodec_set_dimensions(avctx, 0, 0);
537 return -1;
538 }
539 }
540
f62a2b61
AJ
541 if (!is_alpha) {
542 p->reference = 1;
594d4d5d 543 if (ff_get_buffer(avctx, p) < 0) {
f62a2b61
AJ
544 av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
545 return -1;
546 }
5ce117c3 547
bb675d3a 548 if (res == VP56_SIZE_CHANGE)
f62a2b61
AJ
549 if (vp56_size_changed(avctx)) {
550 avctx->release_buffer(avctx, p);
551 return -1;
552 }
5ce117c3 553 }
5ce117c3 554
f62a2b61 555 if (p->key_frame) {
975a1447 556 p->pict_type = AV_PICTURE_TYPE_I;
f62a2b61
AJ
557 s->default_models_init(s);
558 for (block=0; block<s->mb_height*s->mb_width; block++)
559 s->macroblocks[block].type = VP56_MB_INTRA;
560 } else {
975a1447 561 p->pict_type = AV_PICTURE_TYPE_P;
f62a2b61
AJ
562 vp56_parse_mb_type_models(s);
563 s->parse_vector_models(s);
564 s->mb_type = VP56_MB_INTER_NOVEC_PF;
565 }
5ce117c3 566
066fff75
LA
567 if (s->parse_coeff_models(s))
568 goto next;
5ce117c3 569
f62a2b61
AJ
570 memset(s->prev_dc, 0, sizeof(s->prev_dc));
571 s->prev_dc[1][VP56_FRAME_CURRENT] = 128;
572 s->prev_dc[2][VP56_FRAME_CURRENT] = 128;
5ce117c3 573
f62a2b61 574 for (block=0; block < 4*s->mb_width+6; block++) {
738a89b9 575 s->above_blocks[block].ref_frame = VP56_FRAME_NONE;
f62a2b61
AJ
576 s->above_blocks[block].dc_coeff = 0;
577 s->above_blocks[block].not_null_dc = 0;
578 }
738a89b9
CEH
579 s->above_blocks[2*s->mb_width + 2].ref_frame = VP56_FRAME_CURRENT;
580 s->above_blocks[3*s->mb_width + 4].ref_frame = VP56_FRAME_CURRENT;
5ce117c3 581
f62a2b61
AJ
582 stride_y = p->linesize[0];
583 stride_uv = p->linesize[1];
5ce117c3 584
5ce117c3 585 if (s->flip < 0)
f62a2b61
AJ
586 mb_offset = 7;
587
588 /* main macroblocks loop */
589 for (mb_row=0; mb_row<s->mb_height; mb_row++) {
590 if (s->flip < 0)
591 mb_row_flip = s->mb_height - mb_row - 1;
592 else
593 mb_row_flip = mb_row;
594
595 for (block=0; block<4; block++) {
738a89b9 596 s->left_block[block].ref_frame = VP56_FRAME_NONE;
f62a2b61
AJ
597 s->left_block[block].dc_coeff = 0;
598 s->left_block[block].not_null_dc = 0;
5ce117c3 599 }
d3f9edba 600 memset(s->coeff_ctx, 0, sizeof(s->coeff_ctx));
f62a2b61
AJ
601 memset(s->coeff_ctx_last, 24, sizeof(s->coeff_ctx_last));
602
603 s->above_block_idx[0] = 1;
604 s->above_block_idx[1] = 2;
605 s->above_block_idx[2] = 1;
606 s->above_block_idx[3] = 2;
607 s->above_block_idx[4] = 2*s->mb_width + 2 + 1;
608 s->above_block_idx[5] = 3*s->mb_width + 4 + 1;
609
610 s->block_offset[s->frbi] = (mb_row_flip*16 + mb_offset) * stride_y;
611 s->block_offset[s->srbi] = s->block_offset[s->frbi] + 8*stride_y;
612 s->block_offset[1] = s->block_offset[0] + 8;
613 s->block_offset[3] = s->block_offset[2] + 8;
614 s->block_offset[4] = (mb_row_flip*8 + mb_offset) * stride_uv;
615 s->block_offset[5] = s->block_offset[4];
616
617 for (mb_col=0; mb_col<s->mb_width; mb_col++) {
618 vp56_decode_mb(s, mb_row, mb_col, is_alpha);
619
620 for (y=0; y<4; y++) {
621 s->above_block_idx[y] += 2;
622 s->block_offset[y] += 16;
623 }
5ce117c3 624
f62a2b61
AJ
625 for (uv=4; uv<6; uv++) {
626 s->above_block_idx[uv] += 1;
627 s->block_offset[uv] += 8;
628 }
5ce117c3
AJ
629 }
630 }
5ce117c3 631
066fff75 632 next:
f62a2b61
AJ
633 if (p->key_frame || golden_frame) {
634 if (s->framep[VP56_FRAME_GOLDEN]->data[0] &&
635 s->framep[VP56_FRAME_GOLDEN] != s->framep[VP56_FRAME_GOLDEN2])
636 avctx->release_buffer(avctx, s->framep[VP56_FRAME_GOLDEN]);
637 s->framep[VP56_FRAME_GOLDEN] = p;
638 }
91fc2cf1 639
f62a2b61
AJ
640 if (s->has_alpha) {
641 FFSWAP(AVFrame *, s->framep[VP56_FRAME_GOLDEN],
642 s->framep[VP56_FRAME_GOLDEN2]);
643 buf += alpha_offset;
5210529e 644 remaining_buf_size -= alpha_offset;
f62a2b61 645 }
91fc2cf1
AJ
646 }
647
648 if (s->framep[VP56_FRAME_PREVIOUS] == s->framep[VP56_FRAME_GOLDEN] ||
649 s->framep[VP56_FRAME_PREVIOUS] == s->framep[VP56_FRAME_GOLDEN2]) {
650 if (s->framep[VP56_FRAME_UNUSED] != s->framep[VP56_FRAME_GOLDEN] &&
651 s->framep[VP56_FRAME_UNUSED] != s->framep[VP56_FRAME_GOLDEN2])
652 FFSWAP(AVFrame *, s->framep[VP56_FRAME_PREVIOUS],
653 s->framep[VP56_FRAME_UNUSED]);
654 else
655 FFSWAP(AVFrame *, s->framep[VP56_FRAME_PREVIOUS],
656 s->framep[VP56_FRAME_UNUSED2]);
657 } else if (s->framep[VP56_FRAME_PREVIOUS]->data[0])
658 avctx->release_buffer(avctx, s->framep[VP56_FRAME_PREVIOUS]);
704a2881
AJ
659 FFSWAP(AVFrame *, s->framep[VP56_FRAME_CURRENT],
660 s->framep[VP56_FRAME_PREVIOUS]);
5ce117c3 661
800274f8
RD
662 p->qstride = 0;
663 p->qscale_table = s->qscale_table;
664 p->qscale_type = FF_QSCALE_TYPE_VP56;
eba0fcad 665 *(AVFrame*)data = *p;
df9b9567 666 *got_frame = 1;
5ce117c3 667
012f9308 668 return avpkt->size;
5ce117c3
AJ
669}
670
d9504970 671av_cold void ff_vp56_init(AVCodecContext *avctx, int flip, int has_alpha)
5ce117c3 672{
3d52bca6 673 VP56Context *s = avctx->priv_data;
5ce117c3
AJ
674 int i;
675
676 s->avctx = avctx;
716d413c 677 avctx->pix_fmt = has_alpha ? AV_PIX_FMT_YUVA420P : AV_PIX_FMT_YUV420P;
5ce117c3 678
9cf0841e 679 ff_dsputil_init(&s->dsp, avctx);
79dad2a9 680 ff_h264chroma_init(&s->h264chroma, 8);
8c53d39e 681 ff_videodsp_init(&s->vdsp, 8);
28f9ab70 682 ff_vp3dsp_init(&s->vp3dsp, avctx->flags);
5e1ba34b 683 ff_vp56dsp_init(&s->vp56dsp, avctx->codec->id);
28f9ab70 684 ff_init_scantable_permutation(s->dsp.idct_permutation, s->vp3dsp.idct_perm);
5ce117c3
AJ
685 ff_init_scantable(s->dsp.idct_permutation, &s->scantable,ff_zigzag_direct);
686
91fc2cf1 687 for (i=0; i<4; i++)
704a2881
AJ
688 s->framep[i] = &s->frames[i];
689 s->framep[VP56_FRAME_UNUSED] = s->framep[VP56_FRAME_GOLDEN];
91fc2cf1 690 s->framep[VP56_FRAME_UNUSED2] = s->framep[VP56_FRAME_GOLDEN2];
5ce117c3
AJ
691 s->edge_emu_buffer_alloc = NULL;
692
693 s->above_blocks = NULL;
694 s->macroblocks = NULL;
695 s->quantizer = -1;
696 s->deblock_filtering = 1;
697
698 s->filter = NULL;
699
91fc2cf1 700 s->has_alpha = has_alpha;
5ce117c3
AJ
701 if (flip) {
702 s->flip = -1;
703 s->frbi = 2;
704 s->srbi = 0;
705 } else {
706 s->flip = 1;
707 s->frbi = 0;
708 s->srbi = 2;
709 }
710}
711
d9504970 712av_cold int ff_vp56_free(AVCodecContext *avctx)
5ce117c3 713{
3d52bca6 714 VP56Context *s = avctx->priv_data;
5ce117c3 715
800274f8 716 av_freep(&s->qscale_table);
6242b1c4
RD
717 av_freep(&s->above_blocks);
718 av_freep(&s->macroblocks);
719 av_freep(&s->edge_emu_buffer_alloc);
1457516f 720 if (s->framep[VP56_FRAME_GOLDEN]->data[0])
704a2881 721 avctx->release_buffer(avctx, s->framep[VP56_FRAME_GOLDEN]);
91fc2cf1
AJ
722 if (s->framep[VP56_FRAME_GOLDEN2]->data[0])
723 avctx->release_buffer(avctx, s->framep[VP56_FRAME_GOLDEN2]);
704a2881
AJ
724 if (s->framep[VP56_FRAME_PREVIOUS]->data[0])
725 avctx->release_buffer(avctx, s->framep[VP56_FRAME_PREVIOUS]);
5ce117c3
AJ
726 return 0;
727}