indeo2: check decoding errors.
[libav.git] / libavcodec / indeo2.c
CommitLineData
856dbbff 1/*
02c1592f 2 * Intel Indeo 2 codec
856dbbff
MN
3 * Copyright (c) 2005 Konstantin Shishkov
4 *
2912e87a 5 * This file is part of Libav.
b78e7197 6 *
2912e87a 7 * Libav is free software; you can redistribute it and/or
856dbbff
MN
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
b78e7197 10 * version 2.1 of the License, or (at your option) any later version.
856dbbff 11 *
2912e87a 12 * Libav is distributed in the hope that it will be useful,
856dbbff
MN
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
2912e87a 18 * License along with Libav; if not, write to the Free Software
5509bffa 19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
856dbbff 20 */
115329f1 21
856dbbff 22/**
ba87f080 23 * @file
856dbbff
MN
24 * Intel Indeo 2 decoder.
25 */
d5c62122 26
aaf47bcd 27#define BITSTREAM_READER_LE
d5c62122 28#include "libavutil/attributes.h"
856dbbff 29#include "avcodec.h"
9106a698 30#include "get_bits.h"
856dbbff 31#include "indeo2data.h"
d5c62122 32#include "mathops.h"
856dbbff
MN
33
34typedef struct Ir2Context{
35 AVCodecContext *avctx;
36 AVFrame picture;
37 GetBitContext gb;
38 int decode_delta;
39} Ir2Context;
40
41#define CODE_VLC_BITS 14
42static VLC ir2_vlc;
43
44/* Indeo 2 codes are in range 0x01..0x7F and 0x81..0x90 */
45static inline int ir2_get_code(GetBitContext *gb)
46{
8b39f75b 47 return get_vlc2(gb, ir2_vlc.table, CODE_VLC_BITS, 1) + 1;
856dbbff 48}
856dbbff 49
f707a5eb 50static int ir2_decode_plane(Ir2Context *ctx, int width, int height, uint8_t *dst, int stride,
856dbbff
MN
51 const uint8_t *table)
52{
53 int i;
54 int j;
55 int out = 0;
56 int c;
57 int t;
115329f1 58
f707a5eb 59 if(width&1)
6ea2c9a4 60 return AVERROR_INVALIDDATA;
f707a5eb 61
856dbbff
MN
62 /* first line contain absolute values, other lines contain deltas */
63 while (out < width){
64 c = ir2_get_code(&ctx->gb);
8b39f75b
MN
65 if(c >= 0x80) { /* we have a run */
66 c -= 0x7F;
f707a5eb 67 if(out + c*2 > width)
6ea2c9a4 68 return AVERROR_INVALIDDATA;
856dbbff
MN
69 for (i = 0; i < c * 2; i++)
70 dst[out++] = 0x80;
71 } else { /* copy two values from table */
72 dst[out++] = table[c * 2];
73 dst[out++] = table[(c * 2) + 1];
74 }
75 }
76 dst += stride;
115329f1 77
856dbbff
MN
78 for (j = 1; j < height; j++){
79 out = 0;
80 while (out < width){
81 c = ir2_get_code(&ctx->gb);
8b39f75b
MN
82 if(c >= 0x80) { /* we have a skip */
83 c -= 0x7F;
f707a5eb 84 if(out + c*2 > width)
6ea2c9a4 85 return AVERROR_INVALIDDATA;
856dbbff
MN
86 for (i = 0; i < c * 2; i++) {
87 dst[out] = dst[out - stride];
88 out++;
89 }
90 } else { /* add two deltas from table */
91 t = dst[out - stride] + (table[c * 2] - 128);
f66e4f5f 92 t= av_clip_uint8(t);
856dbbff
MN
93 dst[out] = t;
94 out++;
95 t = dst[out - stride] + (table[(c * 2) + 1] - 128);
f66e4f5f 96 t= av_clip_uint8(t);
856dbbff
MN
97 dst[out] = t;
98 out++;
99 }
100 }
101 dst += stride;
102 }
f707a5eb 103 return 0;
856dbbff
MN
104}
105
f707a5eb 106static int ir2_decode_plane_inter(Ir2Context *ctx, int width, int height, uint8_t *dst, int stride,
856dbbff
MN
107 const uint8_t *table)
108{
109 int j;
110 int out = 0;
111 int c;
112 int t;
f707a5eb
MN
113
114 if(width&1)
6ea2c9a4 115 return AVERROR_INVALIDDATA;
f707a5eb 116
856dbbff
MN
117 for (j = 0; j < height; j++){
118 out = 0;
119 while (out < width){
120 c = ir2_get_code(&ctx->gb);
8b39f75b
MN
121 if(c >= 0x80) { /* we have a skip */
122 c -= 0x7F;
856dbbff
MN
123 out += c * 2;
124 } else { /* add two deltas from table */
32fe3ac0 125 t = dst[out] + (((table[c * 2] - 128)*3) >> 2);
f66e4f5f 126 t= av_clip_uint8(t);
856dbbff
MN
127 dst[out] = t;
128 out++;
32fe3ac0 129 t = dst[out] + (((table[(c * 2) + 1] - 128)*3) >> 2);
f66e4f5f 130 t= av_clip_uint8(t);
856dbbff
MN
131 dst[out] = t;
132 out++;
133 }
134 }
135 dst += stride;
136 }
f707a5eb 137 return 0;
856dbbff
MN
138}
139
115329f1 140static int ir2_decode_frame(AVCodecContext *avctx,
df9b9567 141 void *data, int *got_frame,
7a00bbad 142 AVPacket *avpkt)
856dbbff 143{
7a00bbad
TB
144 const uint8_t *buf = avpkt->data;
145 int buf_size = avpkt->size;
856dbbff
MN
146 Ir2Context * const s = avctx->priv_data;
147 AVFrame *picture = data;
562b6c74 148 AVFrame * const p = &s->picture;
6ea2c9a4 149 int start, ret;
856dbbff
MN
150
151 if(p->data[0])
152 avctx->release_buffer(avctx, p);
153
154 p->reference = 1;
155 p->buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE;
6ea2c9a4 156 if ((ret = avctx->reget_buffer(avctx, p)) < 0) {
856dbbff 157 av_log(s->avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
6ea2c9a4 158 return ret;
856dbbff
MN
159 }
160
b7ce4f1d
AC
161 start = 48; /* hardcoded for now */
162
163 if (start >= buf_size) {
164 av_log(s->avctx, AV_LOG_ERROR, "input buffer size too small (%d)\n", buf_size);
165 return AVERROR_INVALIDDATA;
166 }
167
856dbbff 168 s->decode_delta = buf[18];
115329f1 169
856dbbff 170 /* decide whether frame uses deltas or not */
aaf47bcd 171#ifndef BITSTREAM_READER_LE
856dbbff 172 for (i = 0; i < buf_size; i++)
d5c62122 173 buf[i] = ff_reverse[buf[i]];
ef56de32 174#endif
856dbbff 175
68ca330c 176 init_get_bits(&s->gb, buf + start, (buf_size - start) * 8);
856dbbff
MN
177
178 if (s->decode_delta) { /* intraframe */
7b1fbd47
AK
179 if ((ret = ir2_decode_plane(s, avctx->width, avctx->height,
180 s->picture.data[0], s->picture.linesize[0],
181 ir2_luma_table)) < 0)
182 return ret;
183
856dbbff 184 /* swapped U and V */
7b1fbd47
AK
185 if ((ret = ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2,
186 s->picture.data[2], s->picture.linesize[2],
187 ir2_luma_table)) < 0)
188 return ret;
189 if ((ret = ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2,
190 s->picture.data[1], s->picture.linesize[1],
191 ir2_luma_table)) < 0)
192 return ret;
856dbbff 193 } else { /* interframe */
7b1fbd47
AK
194 if ((ret = ir2_decode_plane_inter(s, avctx->width, avctx->height,
195 s->picture.data[0], s->picture.linesize[0],
196 ir2_luma_table)) < 0)
197 return ret;
856dbbff 198 /* swapped U and V */
7b1fbd47
AK
199 if ((ret = ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2,
200 s->picture.data[2], s->picture.linesize[2],
201 ir2_luma_table)) < 0)
202 return ret;
203 if ((ret = ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2,
204 s->picture.data[1], s->picture.linesize[1],
205 ir2_luma_table)) < 0)
206 return ret;
856dbbff
MN
207 }
208
324deaa2 209 *picture = s->picture;
df9b9567 210 *got_frame = 1;
856dbbff
MN
211
212 return buf_size;
213}
214
98a6fff9 215static av_cold int ir2_decode_init(AVCodecContext *avctx){
856dbbff 216 Ir2Context * const ic = avctx->priv_data;
bd4110f9 217 static VLC_TYPE vlc_tables[1 << CODE_VLC_BITS][2];
856dbbff
MN
218
219 ic->avctx = avctx;
220
716d413c 221 avctx->pix_fmt= AV_PIX_FMT_YUV410P;
115329f1 222
bd4110f9
KS
223 ir2_vlc.table = vlc_tables;
224 ir2_vlc.table_allocated = 1 << CODE_VLC_BITS;
aaf47bcd 225#ifdef BITSTREAM_READER_LE
856dbbff
MN
226 init_vlc(&ir2_vlc, CODE_VLC_BITS, IR2_CODES,
227 &ir2_codes[0][1], 4, 2,
bd4110f9 228 &ir2_codes[0][0], 4, 2, INIT_VLC_USE_NEW_STATIC | INIT_VLC_LE);
ef56de32 229#else
c11aac6c
CL
230 init_vlc(&ir2_vlc, CODE_VLC_BITS, IR2_CODES,
231 &ir2_codes[0][1], 4, 2,
bd4110f9 232 &ir2_codes[0][0], 4, 2, INIT_VLC_USE_NEW_STATIC);
ef56de32 233#endif
115329f1 234
856dbbff
MN
235 return 0;
236}
237
6d924b5a
KS
238static av_cold int ir2_decode_end(AVCodecContext *avctx){
239 Ir2Context * const ic = avctx->priv_data;
240 AVFrame *pic = &ic->picture;
241
242 if (pic->data[0])
243 avctx->release_buffer(avctx, pic);
6d924b5a
KS
244
245 return 0;
246}
247
d36beb3f 248AVCodec ff_indeo2_decoder = {
ec6402b7
AK
249 .name = "indeo2",
250 .type = AVMEDIA_TYPE_VIDEO,
36ef5369 251 .id = AV_CODEC_ID_INDEO2,
ec6402b7
AK
252 .priv_data_size = sizeof(Ir2Context),
253 .init = ir2_decode_init,
254 .close = ir2_decode_end,
255 .decode = ir2_decode_frame,
256 .capabilities = CODEC_CAP_DR1,
00c3b67b 257 .long_name = NULL_IF_CONFIG_SMALL("Intel Indeo 2"),
856dbbff 258};