vdpau: remove dysfunctional H.263 support
[libav.git] / libavcodec / h263dec.c
CommitLineData
de6d9b64 1/*
983e3246 2 * H.263 decoder
406792e7 3 * Copyright (c) 2001 Fabrice Bellard
8f2ab833 4 * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
de6d9b64 5 *
2912e87a 6 * This file is part of Libav.
b78e7197 7 *
2912e87a 8 * Libav is free software; you can redistribute it and/or
ff4ec49e
FB
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
b78e7197 11 * version 2.1 of the License, or (at your option) any later version.
de6d9b64 12 *
2912e87a 13 * Libav is distributed in the hope that it will be useful,
de6d9b64 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
ff4ec49e
FB
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
de6d9b64 17 *
ff4ec49e 18 * You should have received a copy of the GNU Lesser General Public
2912e87a 19 * License along with Libav; if not, write to the Free Software
5509bffa 20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
de6d9b64 21 */
115329f1 22
983e3246 23/**
ba87f080 24 * @file
983e3246
MN
25 * H.263 decoder.
26 */
115329f1 27
c6c98d08 28#include "libavutil/cpu.h"
de6d9b64 29#include "avcodec.h"
5f401b7b 30#include "error_resilience.h"
1ce3ec24 31#include "flv.h"
fc53b6af 32#include "h263.h"
c33d9814 33#include "h263_parser.h"
1ce3ec24 34#include "internal.h"
7b9ef8d7 35#include "mpeg_er.h"
1ce3ec24 36#include "mpeg4video.h"
df495dbd 37#include "mpeg4video_parser.h"
925b80d6 38#include "mpegutils.h"
1ce3ec24 39#include "mpegvideo.h"
15025553 40#include "msmpeg4.h"
368f5035 41#include "qpeldsp.h"
6a9c8594 42#include "thread.h"
2f15846a 43#include "wmv2.h"
de6d9b64 44
1f29e5d7
RDC
45static enum AVPixelFormat h263_get_format(AVCodecContext *avctx)
46{
47 if (avctx->codec->id == AV_CODEC_ID_MSS2)
48 return AV_PIX_FMT_YUV420P;
49
50 return avctx->pix_fmt = ff_get_format(avctx, avctx->codec->pix_fmts);
51}
52
98a6fff9 53av_cold int ff_h263_decode_init(AVCodecContext *avctx)
de6d9b64
FB
54{
55 MpegEncContext *s = avctx->priv_data;
07493148 56 int ret;
519c2b6d 57
1ce3ec24
LB
58 s->avctx = avctx;
59 s->out_format = FMT_H263;
60 s->width = avctx->coded_width;
61 s->height = avctx->coded_height;
62 s->workaround_bugs = avctx->workaround_bugs;
de6d9b64 63
4d2858de 64 // set defaults
835f798c 65 ff_mpv_decode_defaults(s);
1ce3ec24
LB
66 s->quant_precision = 5;
67 s->decode_mb = ff_h263_decode_mb;
68 s->low_delay = 1;
1ce3ec24 69 s->unrestricted_mv = 1;
4d2858de 70
de6d9b64 71 /* select sub codec */
1ce3ec24 72 switch (avctx->codec->id) {
36ef5369 73 case AV_CODEC_ID_H263:
1ce3ec24 74 s->unrestricted_mv = 0;
580a7465 75 avctx->chroma_sample_location = AVCHROMA_LOC_CENTER;
de6d9b64 76 break;
36ef5369 77 case AV_CODEC_ID_MPEG4:
de6d9b64 78 break;
36ef5369 79 case AV_CODEC_ID_MSMPEG4V1:
1ce3ec24
LB
80 s->h263_pred = 1;
81 s->msmpeg4_version = 1;
84afee34 82 break;
36ef5369 83 case AV_CODEC_ID_MSMPEG4V2:
1ce3ec24
LB
84 s->h263_pred = 1;
85 s->msmpeg4_version = 2;
84afee34 86 break;
36ef5369 87 case AV_CODEC_ID_MSMPEG4V3:
1ce3ec24
LB
88 s->h263_pred = 1;
89 s->msmpeg4_version = 3;
de6d9b64 90 break;
36ef5369 91 case AV_CODEC_ID_WMV1:
1ce3ec24
LB
92 s->h263_pred = 1;
93 s->msmpeg4_version = 4;
e1a9dbff 94 break;
36ef5369 95 case AV_CODEC_ID_WMV2:
1ce3ec24
LB
96 s->h263_pred = 1;
97 s->msmpeg4_version = 5;
8f8402e4 98 break;
36ef5369
AK
99 case AV_CODEC_ID_VC1:
100 case AV_CODEC_ID_WMV3:
101 case AV_CODEC_ID_VC1IMAGE:
102 case AV_CODEC_ID_WMV3IMAGE:
ee769c6a 103 case AV_CODEC_ID_MSS2:
1ce3ec24
LB
104 s->h263_pred = 1;
105 s->msmpeg4_version = 6;
580a7465 106 avctx->chroma_sample_location = AVCHROMA_LOC_LEFT;
0d33db8a 107 break;
36ef5369 108 case AV_CODEC_ID_H263I:
de6d9b64 109 break;
36ef5369 110 case AV_CODEC_ID_FLV1:
d4f5d74a
GM
111 s->h263_flv = 1;
112 break;
de6d9b64 113 default:
07493148
LB
114 av_log(avctx, AV_LOG_ERROR, "Unsupported codec %d\n",
115 avctx->codec->id);
116 return AVERROR(ENOSYS);
de6d9b64 117 }
1ce3ec24 118 s->codec_id = avctx->codec->id;
93a21abd 119
de6d9b64 120 /* for h263, we allocate the images after having read the header */
1ce3ec24 121 if (avctx->codec->id != AV_CODEC_ID_H263 &&
998c9f15 122 avctx->codec->id != AV_CODEC_ID_MPEG4) {
1f29e5d7 123 avctx->pix_fmt = h263_get_format(avctx);
998c9f15 124 ff_mpv_idct_init(s);
835f798c 125 if ((ret = ff_mpv_common_init(s)) < 0)
07493148 126 return ret;
998c9f15 127 }
de6d9b64 128
0338c396 129 ff_h263dsp_init(&s->h263dsp);
368f5035 130 ff_qpeldsp_init(&s->qdsp);
1ce3ec24 131 ff_h263_decode_init_vlc();
115329f1 132
de6d9b64
FB
133 return 0;
134}
135
98a6fff9 136av_cold int ff_h263_decode_end(AVCodecContext *avctx)
de6d9b64
FB
137{
138 MpegEncContext *s = avctx->priv_data;
139
835f798c 140 ff_mpv_common_end(s);
de6d9b64
FB
141 return 0;
142}
143
7866eeff 144/**
58c42af7 145 * Return the number of bytes consumed for building the current frame.
7866eeff 146 */
1ce3ec24
LB
147static int get_consumed_bytes(MpegEncContext *s, int buf_size)
148{
149 int pos = (get_bits_count(&s->gb) + 7) >> 3;
115329f1 150
1ce3ec24
LB
151 if (s->divx_packed || s->avctx->hwaccel) {
152 /* We would have to scan through the whole buf to handle the weird
153 * reordering ... */
115329f1 154 return buf_size;
7c6eb0a1 155 } else if (s->avctx->flags & AV_CODEC_FLAG_TRUNCATED) {
d7425f59 156 pos -= s->parse_context.last_index;
1ce3ec24
LB
157 // padding is not really read so this might be -1
158 if (pos < 0)
159 pos = 0;
d7425f59 160 return pos;
1ce3ec24
LB
161 } else {
162 // avoid infinite loops (maybe not needed...)
163 if (pos == 0)
164 pos = 1;
165 // oops ;)
166 if (pos + 10 > buf_size)
167 pos = buf_size;
7866eeff
MN
168
169 return pos;
170 }
171}
172
1ce3ec24
LB
173static int decode_slice(MpegEncContext *s)
174{
175 const int part_mask = s->partitioned_frame
176 ? (ER_AC_END | ER_AC_ERROR) : 0x7F;
2bcbd984 177 const int mb_size = 16;
07493148 178 int ret;
115329f1 179
1ce3ec24
LB
180 s->last_resync_gb = s->gb;
181 s->first_slice_line = 1;
182 s->resync_mb_x = s->mb_x;
183 s->resync_mb_y = s->mb_y;
4d2858de 184
554daa24 185 ff_set_qscale(s, s->qscale);
115329f1 186
1468d503 187 if (s->avctx->hwaccel) {
1ce3ec24
LB
188 const uint8_t *start = s->gb.buffer + get_bits_count(&s->gb) / 8;
189 const uint8_t *end = ff_h263_find_resync_marker(start + 1,
190 s->gb.buffer_end);
191 skip_bits_long(&s->gb, 8 * (end - start));
1468d503
GB
192 return s->avctx->hwaccel->decode_slice(s->avctx, start, end - start);
193 }
40e5d31b 194
1ce3ec24
LB
195 if (s->partitioned_frame) {
196 const int qscale = s->qscale;
4d2858de 197
1ce3ec24 198 if (CONFIG_MPEG4_DECODER && s->codec_id == AV_CODEC_ID_MPEG4)
1a890257 199 if ((ret = ff_mpeg4_decode_partitions(s->avctx->priv_data)) < 0)
07493148 200 return ret;
115329f1 201
d99ce8d7 202 /* restore variables which were modified */
1ce3ec24
LB
203 s->first_slice_line = 1;
204 s->mb_x = s->resync_mb_x;
205 s->mb_y = s->resync_mb_y;
554daa24 206 ff_set_qscale(s, qscale);
4d2858de
MN
207 }
208
1ce3ec24 209 for (; s->mb_y < s->mb_height; s->mb_y++) {
4d2858de 210 /* per-row end of slice checks */
1ce3ec24
LB
211 if (s->msmpeg4_version) {
212 if (s->resync_mb_y + s->slice_height == s->mb_y) {
213 ff_er_add_slice(&s->er, s->resync_mb_x, s->resync_mb_y,
214 s->mb_x - 1, s->mb_y, ER_MB_END);
46b4feec 215
4d2858de
MN
216 return 0;
217 }
218 }
115329f1 219
1ce3ec24
LB
220 if (s->msmpeg4_version == 1) {
221 s->last_dc[0] =
222 s->last_dc[1] =
223 s->last_dc[2] = 128;
4d2858de 224 }
115329f1 225
4d2858de 226 ff_init_block_index(s);
1ce3ec24 227 for (; s->mb_x < s->mb_width; s->mb_x++) {
4d2858de
MN
228 int ret;
229
230 ff_update_block_index(s);
231
1ce3ec24
LB
232 if (s->resync_mb_x == s->mb_x && s->resync_mb_y + 1 == s->mb_y)
233 s->first_slice_line = 0;
4d2858de
MN
234
235 /* DCT & quantize */
115329f1 236
1ce3ec24 237 s->mv_dir = MV_DIR_FORWARD;
4d2858de 238 s->mv_type = MV_TYPE_16X16;
4628443c
VG
239 ff_dlog(s, "%d %06X\n",
240 get_bits_count(&s->gb), show_bits(&s->gb, 24));
1ce3ec24 241 ret = s->decode_mb(s, s->block);
4d2858de 242
1ce3ec24 243 if (s->pict_type != AV_PICTURE_TYPE_B)
7d1c3fc1
MN
244 ff_h263_update_motion_val(s);
245
1ce3ec24
LB
246 if (ret < 0) {
247 const int xy = s->mb_x + s->mb_y * s->mb_stride;
248 if (ret == SLICE_END) {
835f798c 249 ff_mpv_decode_mb(s, s->block);
1ce3ec24 250 if (s->loop_filter)
332f9ac4 251 ff_h263_loop_filter(s);
aef95c8c 252
1ce3ec24
LB
253 ff_er_add_slice(&s->er, s->resync_mb_x, s->resync_mb_y,
254 s->mb_x, s->mb_y, ER_MB_END & part_mask);
a61f8265
MN
255
256 s->padding_bug_score--;
115329f1 257
1ce3ec24
LB
258 if (++s->mb_x >= s->mb_width) {
259 s->mb_x = 0;
260 ff_mpeg_draw_horiz_band(s, s->mb_y * mb_size, mb_size);
835f798c 261 ff_mpv_report_decode_progress(s);
4d2858de
MN
262 s->mb_y++;
263 }
115329f1 264 return 0;
1ce3ec24
LB
265 } else if (ret == SLICE_NOEND) {
266 av_log(s->avctx, AV_LOG_ERROR,
267 "Slice mismatch at MB: %d\n", xy);
268 ff_er_add_slice(&s->er, s->resync_mb_x, s->resync_mb_y,
269 s->mb_x + 1, s->mb_y,
270 ER_MB_END & part_mask);
07493148 271 return AVERROR_INVALIDDATA;
4d2858de 272 }
9b879566 273 av_log(s->avctx, AV_LOG_ERROR, "Error at MB: %d\n", xy);
1ce3ec24
LB
274 ff_er_add_slice(&s->er, s->resync_mb_x, s->resync_mb_y,
275 s->mb_x, s->mb_y, ER_MB_ERROR & part_mask);
115329f1 276
07493148 277 return AVERROR_INVALIDDATA;
4d2858de 278 }
aef95c8c 279
835f798c 280 ff_mpv_decode_mb(s, s->block);
1ce3ec24 281 if (s->loop_filter)
332f9ac4 282 ff_h263_loop_filter(s);
4d2858de 283 }
115329f1 284
1ce3ec24 285 ff_mpeg_draw_horiz_band(s, s->mb_y * mb_size, mb_size);
835f798c 286 ff_mpv_report_decode_progress(s);
115329f1 287
1ce3ec24 288 s->mb_x = 0;
4d2858de 289 }
115329f1 290
1ce3ec24 291 assert(s->mb_x == 0 && s->mb_y == s->mb_height);
4d2858de 292
1ce3ec24
LB
293 if (s->codec_id == AV_CODEC_ID_MPEG4 &&
294 (s->workaround_bugs & FF_BUG_AUTODETECT) &&
295 get_bits_left(&s->gb) >= 48 &&
296 show_bits(&s->gb, 24) == 0x4010 &&
297 !s->data_partitioning)
298 s->padding_bug_score += 32;
df59c9c0 299
a61f8265 300 /* try to detect the padding bug */
1ce3ec24
LB
301 if (s->codec_id == AV_CODEC_ID_MPEG4 &&
302 (s->workaround_bugs & FF_BUG_AUTODETECT) &&
303 get_bits_left(&s->gb) >= 0 &&
304 get_bits_left(&s->gb) < 48 &&
1ce3ec24
LB
305 !s->data_partitioning) {
306 const int bits_count = get_bits_count(&s->gb);
307 const int bits_left = s->gb.size_in_bits - bits_count;
308
309 if (bits_left == 0) {
310 s->padding_bug_score += 16;
311 } else if (bits_left != 1) {
312 int v = show_bits(&s->gb, 8);
313 v |= 0x7F >> (7 - (bits_count & 7));
314
315 if (v == 0x7F && bits_left <= 8)
a61f8265 316 s->padding_bug_score--;
1ce3ec24
LB
317 else if (v == 0x7F && ((get_bits_count(&s->gb) + 8) & 8) &&
318 bits_left <= 16)
319 s->padding_bug_score += 4;
a61f8265 320 else
115329f1
DB
321 s->padding_bug_score++;
322 }
a61f8265 323 }
115329f1 324
1ce3ec24 325 if (s->workaround_bugs & FF_BUG_AUTODETECT) {
9a03c232
JA
326 if (s->codec_id == AV_CODEC_ID_H263 ||
327 (s->padding_bug_score > -2 && !s->data_partitioning))
1ce3ec24 328 s->workaround_bugs |= FF_BUG_NO_PADDING;
4940ed6a
MN
329 else
330 s->workaround_bugs &= ~FF_BUG_NO_PADDING;
331 }
a61f8265 332
bb628dae 333 // handle formats which don't have unique end markers
1ce3ec24
LB
334 if (s->msmpeg4_version || (s->workaround_bugs & FF_BUG_NO_PADDING)) { // FIXME perhaps solve this more cleanly
335 int left = get_bits_left(&s->gb);
336 int max_extra = 7;
115329f1 337
4d2858de 338 /* no markers in M$ crap */
1ce3ec24
LB
339 if (s->msmpeg4_version && s->pict_type == AV_PICTURE_TYPE_I)
340 max_extra += 17;
341
342 /* buggy padding but the frame should still end approximately at
343 * the bitstream end */
344 if ((s->workaround_bugs & FF_BUG_NO_PADDING) &&
9c1db92a 345 (s->avctx->err_recognition & AV_EF_BUFFER))
1ce3ec24
LB
346 max_extra += 48;
347 else if ((s->workaround_bugs & FF_BUG_NO_PADDING))
348 max_extra += 256 * 256 * 256 * 64;
349
350 if (left > max_extra)
351 av_log(s->avctx, AV_LOG_ERROR,
352 "discarding %d junk bits at end, next would be %X\n",
353 left, show_bits(&s->gb, 24));
354 else if (left < 0)
9b879566 355 av_log(s->avctx, AV_LOG_ERROR, "overreading %d bits\n", -left);
1ce3ec24
LB
356 else
357 ff_er_add_slice(&s->er, s->resync_mb_x, s->resync_mb_y,
358 s->mb_x - 1, s->mb_y, ER_MB_END);
115329f1 359
4d2858de
MN
360 return 0;
361 }
a61f8265 362
1ce3ec24
LB
363 av_log(s->avctx, AV_LOG_ERROR,
364 "slice end not reached but screenspace end (%d left %06X, score= %d)\n",
365 get_bits_left(&s->gb), show_bits(&s->gb, 24), s->padding_bug_score);
115329f1 366
1ce3ec24
LB
367 ff_er_add_slice(&s->er, s->resync_mb_x, s->resync_mb_y, s->mb_x, s->mb_y,
368 ER_MB_END & part_mask);
46b4feec 369
07493148 370 return AVERROR_INVALIDDATA;
4d2858de
MN
371}
372
1ce3ec24
LB
373int ff_h263_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
374 AVPacket *avpkt)
de6d9b64 375{
7a00bbad 376 const uint8_t *buf = avpkt->data;
1ce3ec24
LB
377 int buf_size = avpkt->size;
378 MpegEncContext *s = avctx->priv_data;
1df1df0b 379 int ret;
115329f1
DB
380 AVFrame *pict = data;
381
e9174ba4 382 /* no supplementary picture */
de6d9b64 383 if (buf_size == 0) {
e9174ba4 384 /* special case for last picture */
1ce3ec24 385 if (s->low_delay == 0 && s->next_picture_ptr) {
f6774f90 386 if ((ret = av_frame_ref(pict, s->next_picture_ptr->f)) < 0)
759001c5 387 return ret;
1ce3ec24 388 s->next_picture_ptr = NULL;
e9174ba4 389
df9b9567 390 *got_frame = 1;
e9174ba4
MN
391 }
392
de6d9b64
FB
393 return 0;
394 }
b7a22d84 395
7c6eb0a1 396 if (s->avctx->flags & AV_CODEC_FLAG_TRUNCATED) {
d7425f59 397 int next;
115329f1 398
1ce3ec24
LB
399 if (CONFIG_MPEG4_DECODER && s->codec_id == AV_CODEC_ID_MPEG4) {
400 next = ff_mpeg4_find_frame_end(&s->parse_context, buf, buf_size);
401 } else if (CONFIG_H263_DECODER && s->codec_id == AV_CODEC_ID_H263) {
402 next = ff_h263_find_frame_end(&s->parse_context, buf, buf_size);
403 } else {
404 av_log(s->avctx, AV_LOG_ERROR,
405 "this codec does not support truncated bitstreams\n");
07493148 406 return AVERROR(ENOSYS);
d7425f59 407 }
115329f1 408
1ce3ec24
LB
409 if (ff_combine_frame(&s->parse_context, next, (const uint8_t **)&buf,
410 &buf_size) < 0)
d7425f59 411 return buf_size;
d7425f59 412 }
de6d9b64 413
1ce3ec24 414 if (s->bitstream_buffer_size && (s->divx_packed || buf_size < 20)) // divx 5.01+/xvid frame reorder
aaaf2dc0
MN
415 ret = init_get_bits8(&s->gb, s->bitstream_buffer,
416 s->bitstream_buffer_size);
1ce3ec24 417 else
aaaf2dc0 418 ret = init_get_bits8(&s->gb, buf, buf_size);
1ce3ec24 419 s->bitstream_buffer_size = 0;
115329f1 420
aaaf2dc0
MN
421 if (ret < 0)
422 return ret;
423
b869eea7 424 if (!s->context_initialized)
07493148 425 // we need the idct permutaton for reading a custom matrix
998c9f15 426 ff_mpv_idct_init(s);
115329f1 427
de6d9b64 428 /* let's go :-) */
1ce3ec24
LB
429 if (CONFIG_WMV2_DECODER && s->msmpeg4_version == 5) {
430 ret = ff_wmv2_decode_picture_header(s);
49fb20cb 431 } else if (CONFIG_MSMPEG4_DECODER && s->msmpeg4_version) {
04d38225 432 ret = ff_msmpeg4_decode_picture_header(s);
87337993 433 } else if (CONFIG_MPEG4_DECODER && avctx->codec_id == AV_CODEC_ID_MPEG4) {
1ce3ec24 434 if (s->avctx->extradata_size && s->picture_number == 0) {
b704e742 435 GetBitContext gb;
115329f1 436
aaaf2dc0
MN
437 ret = init_get_bits8(&gb, s->avctx->extradata,
438 s->avctx->extradata_size);
439 if (ret < 0)
440 return ret;
ee8af2dd 441 ff_mpeg4_decode_picture_header(avctx->priv_data, &gb);
b704e742 442 }
ee8af2dd 443 ret = ff_mpeg4_decode_picture_header(avctx->priv_data, &s->gb);
36ef5369 444 } else if (CONFIG_H263I_DECODER && s->codec_id == AV_CODEC_ID_H263I) {
d7525cba 445 ret = ff_intel_h263_decode_picture_header(s);
a0b0d753
MN
446 } else if (CONFIG_FLV_DECODER && s->h263_flv) {
447 ret = ff_flv_decode_picture_header(s);
de6d9b64 448 } else {
ddce8953 449 ret = ff_h263_decode_picture_header(s);
488fa19a 450 }
115329f1 451
1ce3ec24
LB
452 if (ret == FRAME_SKIPPED)
453 return get_consumed_bytes(s, buf_size);
ffba1dc0
MN
454
455 /* skip if the header was thrashed */
1ce3ec24 456 if (ret < 0) {
9b879566 457 av_log(s->avctx, AV_LOG_ERROR, "header damaged\n");
07493148 458 return ret;
ffba1dc0 459 }
115329f1 460
1f29e5d7
RDC
461 if (!s->context_initialized) {
462 avctx->pix_fmt = h263_get_format(avctx);
835f798c 463 if ((ret = ff_mpv_common_init(s)) < 0)
b869eea7 464 return ret;
1f29e5d7 465 }
b869eea7 466
f929ab05 467 if (!s->current_picture_ptr || s->current_picture_ptr->f->data[0]) {
4e17946f 468 int i = ff_find_unused_picture(s->avctx, s->picture, 0);
b869eea7
JS
469 if (i < 0)
470 return i;
471 s->current_picture_ptr = &s->picture[i];
472 }
473
1ce3ec24 474 avctx->has_b_frames = !s->low_delay;
115329f1 475
1ce3ec24 476#define SET_QPEL_FUNC(postfix1, postfix2) \
368f5035
DB
477 s->qdsp.put_ ## postfix1 = ff_put_ ## postfix2; \
478 s->qdsp.put_no_rnd_ ## postfix1 = ff_put_no_rnd_ ## postfix2; \
479 s->qdsp.avg_ ## postfix1 = ff_avg_ ## postfix2;
db794953 480
1ce3ec24
LB
481 if (s->workaround_bugs & FF_BUG_STD_QPEL) {
482 SET_QPEL_FUNC(qpel_pixels_tab[0][5], qpel16_mc11_old_c)
483 SET_QPEL_FUNC(qpel_pixels_tab[0][7], qpel16_mc31_old_c)
484 SET_QPEL_FUNC(qpel_pixels_tab[0][9], qpel16_mc12_old_c)
db794953
MN
485 SET_QPEL_FUNC(qpel_pixels_tab[0][11], qpel16_mc32_old_c)
486 SET_QPEL_FUNC(qpel_pixels_tab[0][13], qpel16_mc13_old_c)
487 SET_QPEL_FUNC(qpel_pixels_tab[0][15], qpel16_mc33_old_c)
488
1ce3ec24
LB
489 SET_QPEL_FUNC(qpel_pixels_tab[1][5], qpel8_mc11_old_c)
490 SET_QPEL_FUNC(qpel_pixels_tab[1][7], qpel8_mc31_old_c)
491 SET_QPEL_FUNC(qpel_pixels_tab[1][9], qpel8_mc12_old_c)
db794953
MN
492 SET_QPEL_FUNC(qpel_pixels_tab[1][11], qpel8_mc32_old_c)
493 SET_QPEL_FUNC(qpel_pixels_tab[1][13], qpel8_mc13_old_c)
494 SET_QPEL_FUNC(qpel_pixels_tab[1][15], qpel8_mc33_old_c)
495 }
4d2858de 496
1ce3ec24
LB
497 /* After H263 & mpeg4 header decode we have the height, width,
498 * and other parameters. So then we could init the picture.
499 * FIXME: By the way H263 decoder is evolving it should have
500 * an H263EncContext */
8701f4f8
JG
501 if (s->width != avctx->coded_width ||
502 s->height != avctx->coded_height ||
503 s->context_reinit) {
504 /* H.263 could change picture size any time */
505 s->context_reinit = 0;
506
3b086317
AK
507 ret = ff_set_dimensions(avctx, s->width, s->height);
508 if (ret < 0)
509 return ret;
8701f4f8 510
9e500efd
JR
511 ff_set_sar(avctx, avctx->sample_aspect_ratio);
512
835f798c 513 if ((ret = ff_mpv_common_frame_size_change(s)))
8701f4f8 514 return ret;
c80a8161
RDC
515
516 if (avctx->pix_fmt != h263_get_format(avctx)) {
517 av_log(avctx, AV_LOG_ERROR, "format change not supported\n");
518 avctx->pix_fmt = AV_PIX_FMT_NONE;
519 return AVERROR_UNKNOWN;
520 }
8701f4f8
JG
521 }
522
1ce3ec24
LB
523 if (s->codec_id == AV_CODEC_ID_H263 ||
524 s->codec_id == AV_CODEC_ID_H263P ||
525 s->codec_id == AV_CODEC_ID_H263I)
0c69164f 526 s->gob_index = H263_GOB_HEIGHT(s->height);
115329f1 527
8ed2ae09 528 // for skipping the frame
f6774f90 529 s->current_picture.f->pict_type = s->pict_type;
530 s->current_picture.f->key_frame = s->pict_type == AV_PICTURE_TYPE_I;
347ae22e 531
bb628dae 532 /* skip B-frames if we don't have reference frames */
f929ab05 533 if (!s->last_picture_ptr &&
ba0c8981
DB
534 (s->pict_type == AV_PICTURE_TYPE_B || s->droppable))
535 return get_consumed_bytes(s, buf_size);
1ce3ec24
LB
536 if ((avctx->skip_frame >= AVDISCARD_NONREF &&
537 s->pict_type == AV_PICTURE_TYPE_B) ||
538 (avctx->skip_frame >= AVDISCARD_NONKEY &&
539 s->pict_type != AV_PICTURE_TYPE_I) ||
540 avctx->skip_frame >= AVDISCARD_ALL)
8c3eba7c 541 return get_consumed_bytes(s, buf_size);
115329f1 542
1ce3ec24
LB
543 if (s->next_p_frame_damaged) {
544 if (s->pict_type == AV_PICTURE_TYPE_B)
7866eeff 545 return get_consumed_bytes(s, buf_size);
6733417a 546 else
1ce3ec24 547 s->next_p_frame_damaged = 0;
6733417a
MN
548 }
549
1ce3ec24 550 if ((!s->no_rounding) || s->pict_type == AV_PICTURE_TYPE_B) {
368f5035
DB
551 s->me.qpel_put = s->qdsp.put_qpel_pixels_tab;
552 s->me.qpel_avg = s->qdsp.avg_qpel_pixels_tab;
1ce3ec24 553 } else {
368f5035
DB
554 s->me.qpel_put = s->qdsp.put_no_rnd_qpel_pixels_tab;
555 s->me.qpel_avg = s->qdsp.avg_qpel_pixels_tab;
2833fc46
LM
556 }
557
835f798c 558 if ((ret = ff_mpv_frame_start(s, avctx)) < 0)
07493148 559 return ret;
de6d9b64 560
05fa79b8
JG
561 if (!s->divx_packed && !avctx->hwaccel)
562 ff_thread_finish_setup(avctx);
6a9c8594 563
07493148
LB
564 if (avctx->hwaccel) {
565 ret = avctx->hwaccel->start_frame(avctx, s->gb.buffer,
566 s->gb.buffer_end - s->gb.buffer);
567 if (ret < 0 )
568 return ret;
569 }
1468d503 570
54974c62 571 ff_mpeg_er_frame_start(s);
115329f1 572
1ce3ec24
LB
573 /* the second part of the wmv2 header contains the MB skip bits which
574 * are stored in current_picture->mb_type which is not available before
835f798c 575 * ff_mpv_frame_start() */
1ce3ec24 576 if (CONFIG_WMV2_DECODER && s->msmpeg4_version == 5) {
9abc7e0f 577 ret = ff_wmv2_decode_secondary_picture_header(s);
1ce3ec24
LB
578 if (ret < 0)
579 return ret;
580 if (ret == 1)
581 goto intrax8_decoded;
487bce53
MN
582 }
583
de6d9b64 584 /* decode each macroblock */
1ce3ec24
LB
585 s->mb_x = 0;
586 s->mb_y = 0;
115329f1 587
2a11952f 588 ret = decode_slice(s);
1ce3ec24
LB
589 while (s->mb_y < s->mb_height) {
590 if (s->msmpeg4_version) {
591 if (s->slice_height == 0 || s->mb_x != 0 ||
592 (s->mb_y % s->slice_height) != 0 || get_bits_left(&s->gb) < 0)
4d2858de 593 break;
1ce3ec24
LB
594 } else {
595 int prev_x = s->mb_x, prev_y = s->mb_y;
596 if (ff_h263_resync(s) < 0)
4d2858de 597 break;
6a9c8594 598 if (prev_y * s->mb_width + prev_x < s->mb_y * s->mb_width + s->mb_x)
54974c62 599 s->er.error_occurred = 1;
6733417a 600 }
115329f1 601
1ce3ec24 602 if (s->msmpeg4_version < 4 && s->h263_pred)
4d2858de 603 ff_mpeg4_clean_buffers(s);
649c00c9 604
1ce3ec24
LB
605 if (decode_slice(s) < 0)
606 ret = AVERROR_INVALIDDATA;
de6d9b64 607 }
8b32880c 608
1ce3ec24
LB
609 if (s->msmpeg4_version && s->msmpeg4_version < 4 &&
610 s->pict_type == AV_PICTURE_TYPE_I)
611 if (!CONFIG_MSMPEG4_DECODER ||
612 ff_msmpeg4_decode_ext_header(s, buf_size) < 0)
54974c62 613 s->er.error_status_table[s->mb_num - 1] = ER_MB_ERROR;
115329f1 614
1ce3ec24 615 assert(s->bitstream_buffer_size == 0);
fb204d4b 616
e2ceb176
AK
617 if (CONFIG_MPEG4_DECODER && avctx->codec_id == AV_CODEC_ID_MPEG4)
618 ff_mpeg4_frame_end(avctx, buf, buf_size);
6733417a 619
9abc7e0f 620intrax8_decoded:
54974c62 621 ff_er_frame_end(&s->er);
4d2858de 622
07493148
LB
623 if (avctx->hwaccel) {
624 ret = avctx->hwaccel->end_frame(avctx);
625 if (ret < 0)
626 return ret;
627 }
f986c8e2 628
835f798c 629 ff_mpv_frame_end(s);
db794953 630
05fa79b8
JG
631 if (!s->divx_packed && avctx->hwaccel)
632 ff_thread_finish_setup(avctx);
633
f6774f90 634 assert(s->current_picture.f->pict_type ==
635 s->current_picture_ptr->f->pict_type);
636 assert(s->current_picture.f->pict_type == s->pict_type);
975a1447 637 if (s->pict_type == AV_PICTURE_TYPE_B || s->low_delay) {
f6774f90 638 if ((ret = av_frame_ref(pict, s->current_picture_ptr->f)) < 0)
759001c5
AK
639 return ret;
640 ff_print_debug_info(s, s->current_picture_ptr);
4b1f5e50 641 } else if (s->last_picture_ptr) {
f6774f90 642 if ((ret = av_frame_ref(pict, s->last_picture_ptr->f)) < 0)
759001c5
AK
643 return ret;
644 ff_print_debug_info(s, s->last_picture_ptr);
5430b070
MN
645 }
646
1ce3ec24 647 if (s->last_picture_ptr || s->low_delay)
df9b9567 648 *got_frame = 1;
8fc69895 649
1ce3ec24
LB
650 if (ret && (avctx->err_recognition & AV_EF_EXPLODE))
651 return ret;
652 else
653 return get_consumed_bytes(s, buf_size);
de6d9b64
FB
654}
655
8d061989 656const enum AVPixelFormat ff_h263_hwaccel_pixfmt_list_420[] = {
33259118 657#if CONFIG_H263_VAAPI_HWACCEL || CONFIG_MPEG4_VAAPI_HWACCEL
8d061989
RB
658 AV_PIX_FMT_VAAPI_VLD,
659#endif
d35d0c72 660#if CONFIG_MPEG4_VDPAU_HWACCEL
8d061989
RB
661 AV_PIX_FMT_VDPAU,
662#endif
663 AV_PIX_FMT_YUV420P,
664 AV_PIX_FMT_NONE
665};
666
d36beb3f 667AVCodec ff_h263_decoder = {
ec6402b7 668 .name = "h263",
b2bed932 669 .long_name = NULL_IF_CONFIG_SMALL("H.263 / H.263-1996, H.263+ / H.263-1998 / H.263 version 2"),
ec6402b7 670 .type = AVMEDIA_TYPE_VIDEO,
36ef5369 671 .id = AV_CODEC_ID_H263,
ec6402b7
AK
672 .priv_data_size = sizeof(MpegEncContext),
673 .init = ff_h263_decode_init,
674 .close = ff_h263_decode_end,
675 .decode = ff_h263_decode_frame,
def97856
VG
676 .capabilities = AV_CODEC_CAP_DRAW_HORIZ_BAND | AV_CODEC_CAP_DR1 |
677 AV_CODEC_CAP_TRUNCATED | AV_CODEC_CAP_DELAY,
00c3b67b 678 .flush = ff_mpeg_flush,
8d061989 679 .pix_fmts = ff_h263_hwaccel_pixfmt_list_420,
de6d9b64 680};