kmvc: Use bytestream2 functions to prevent buffer overreads.
[libav.git] / libavcodec / kmvc.c
CommitLineData
fd7b1991
KS
1/*
2 * KMVC decoder
3 * Copyright (c) 2006 Konstantin Shishkov
4 *
2912e87a 5 * This file is part of Libav.
b78e7197 6 *
2912e87a 7 * Libav is free software; you can redistribute it and/or
fd7b1991
KS
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
b78e7197 10 * version 2.1 of the License, or (at your option) any later version.
fd7b1991 11 *
2912e87a 12 * Libav is distributed in the hope that it will be useful,
fd7b1991
KS
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
2912e87a 18 * License along with Libav; if not, write to the Free Software
fd7b1991 19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
fd7b1991
KS
20 */
21
22/**
ba87f080 23 * @file
fd7b1991
KS
24 * Karl Morton's Video Codec decoder
25 */
26
27#include <stdio.h>
28#include <stdlib.h>
29
fd7b1991 30#include "avcodec.h"
2c124cb6 31#include "bytestream.h"
fd7b1991
KS
32
33#define KMVC_KEYFRAME 0x80
34#define KMVC_PALETTE 0x40
35#define KMVC_METHOD 0x0F
36
37/*
38 * Decoder context
39 */
40typedef struct KmvcContext {
41 AVCodecContext *avctx;
42 AVFrame pic;
43
44 int setpal;
45 int palsize;
46 uint32_t pal[256];
47 uint8_t *cur, *prev;
48 uint8_t *frm0, *frm1;
da2e774f 49 GetByteContext g;
fd7b1991
KS
50} KmvcContext;
51
52typedef struct BitBuf {
53 int bits;
54 int bitbuf;
55} BitBuf;
56
57#define BLK(data, x, y) data[(x) + (y) * 320]
58
da2e774f 59#define kmvc_init_getbits(bb, g) bb.bits = 7; bb.bitbuf = bytestream2_get_byte(g);
fd7b1991 60
da2e774f 61#define kmvc_getbit(bb, g, res) {\
fd7b1991
KS
62 res = 0; \
63 if (bb.bitbuf & (1 << bb.bits)) res = 1; \
64 bb.bits--; \
65 if(bb.bits == -1) { \
da2e774f 66 bb.bitbuf = bytestream2_get_byte(g); \
fd7b1991
KS
67 bb.bits = 7; \
68 } \
69}
70
da2e774f 71static int kmvc_decode_intra_8x8(KmvcContext * ctx, int w, int h)
fd7b1991
KS
72{
73 BitBuf bb;
74 int res, val;
75 int i, j;
76 int bx, by;
77 int l0x, l1x, l0y, l1y;
78 int mx, my;
79
da2e774f 80 kmvc_init_getbits(bb, &ctx->g);
fd7b1991
KS
81
82 for (by = 0; by < h; by += 8)
83 for (bx = 0; bx < w; bx += 8) {
da2e774f
LI
84 if (!bytestream2_get_bytes_left(&ctx->g)) {
85 av_log(ctx->avctx, AV_LOG_ERROR, "Data overrun\n");
86 return AVERROR_INVALIDDATA;
87 }
88 kmvc_getbit(bb, &ctx->g, res);
fd7b1991 89 if (!res) { // fill whole 8x8 block
da2e774f 90 val = bytestream2_get_byte(&ctx->g);
fd7b1991
KS
91 for (i = 0; i < 64; i++)
92 BLK(ctx->cur, bx + (i & 0x7), by + (i >> 3)) = val;
93 } else { // handle four 4x4 subblocks
94 for (i = 0; i < 4; i++) {
95 l0x = bx + (i & 1) * 4;
96 l0y = by + (i & 2) * 2;
da2e774f 97 kmvc_getbit(bb, &ctx->g, res);
fd7b1991 98 if (!res) {
da2e774f 99 kmvc_getbit(bb, &ctx->g, res);
fd7b1991 100 if (!res) { // fill whole 4x4 block
da2e774f 101 val = bytestream2_get_byte(&ctx->g);
fd7b1991
KS
102 for (j = 0; j < 16; j++)
103 BLK(ctx->cur, l0x + (j & 3), l0y + (j >> 2)) = val;
104 } else { // copy block from already decoded place
da2e774f 105 val = bytestream2_get_byte(&ctx->g);
fd7b1991
KS
106 mx = val & 0xF;
107 my = val >> 4;
108 for (j = 0; j < 16; j++)
109 BLK(ctx->cur, l0x + (j & 3), l0y + (j >> 2)) =
110 BLK(ctx->cur, l0x + (j & 3) - mx, l0y + (j >> 2) - my);
111 }
112 } else { // descend to 2x2 sub-sub-blocks
113 for (j = 0; j < 4; j++) {
114 l1x = l0x + (j & 1) * 2;
115 l1y = l0y + (j & 2);
da2e774f 116 kmvc_getbit(bb, &ctx->g, res);
fd7b1991 117 if (!res) {
da2e774f 118 kmvc_getbit(bb, &ctx->g, res);
fd7b1991 119 if (!res) { // fill whole 2x2 block
da2e774f 120 val = bytestream2_get_byte(&ctx->g);
fd7b1991
KS
121 BLK(ctx->cur, l1x, l1y) = val;
122 BLK(ctx->cur, l1x + 1, l1y) = val;
123 BLK(ctx->cur, l1x, l1y + 1) = val;
124 BLK(ctx->cur, l1x + 1, l1y + 1) = val;
125 } else { // copy block from already decoded place
da2e774f 126 val = bytestream2_get_byte(&ctx->g);
fd7b1991
KS
127 mx = val & 0xF;
128 my = val >> 4;
129 BLK(ctx->cur, l1x, l1y) = BLK(ctx->cur, l1x - mx, l1y - my);
130 BLK(ctx->cur, l1x + 1, l1y) =
131 BLK(ctx->cur, l1x + 1 - mx, l1y - my);
132 BLK(ctx->cur, l1x, l1y + 1) =
133 BLK(ctx->cur, l1x - mx, l1y + 1 - my);
134 BLK(ctx->cur, l1x + 1, l1y + 1) =
135 BLK(ctx->cur, l1x + 1 - mx, l1y + 1 - my);
136 }
137 } else { // read values for block
da2e774f
LI
138 BLK(ctx->cur, l1x, l1y) = bytestream2_get_byte(&ctx->g);
139 BLK(ctx->cur, l1x + 1, l1y) = bytestream2_get_byte(&ctx->g);
140 BLK(ctx->cur, l1x, l1y + 1) = bytestream2_get_byte(&ctx->g);
141 BLK(ctx->cur, l1x + 1, l1y + 1) = bytestream2_get_byte(&ctx->g);
fd7b1991
KS
142 }
143 }
144 }
145 }
146 }
147 }
ad3161ec
GN
148
149 return 0;
fd7b1991
KS
150}
151
da2e774f 152static int kmvc_decode_inter_8x8(KmvcContext * ctx, int w, int h)
fd7b1991
KS
153{
154 BitBuf bb;
155 int res, val;
156 int i, j;
157 int bx, by;
158 int l0x, l1x, l0y, l1y;
159 int mx, my;
160
da2e774f 161 kmvc_init_getbits(bb, &ctx->g);
fd7b1991
KS
162
163 for (by = 0; by < h; by += 8)
164 for (bx = 0; bx < w; bx += 8) {
da2e774f 165 kmvc_getbit(bb, &ctx->g, res);
fd7b1991 166 if (!res) {
da2e774f 167 kmvc_getbit(bb, &ctx->g, res);
fd7b1991 168 if (!res) { // fill whole 8x8 block
da2e774f 169 if (!bytestream2_get_bytes_left(&ctx->g)) {
ad3161ec
GN
170 av_log(ctx->avctx, AV_LOG_ERROR, "Data overrun\n");
171 return AVERROR_INVALIDDATA;
172 }
da2e774f 173 val = bytestream2_get_byte(&ctx->g);
fd7b1991
KS
174 for (i = 0; i < 64; i++)
175 BLK(ctx->cur, bx + (i & 0x7), by + (i >> 3)) = val;
176 } else { // copy block from previous frame
177 for (i = 0; i < 64; i++)
178 BLK(ctx->cur, bx + (i & 0x7), by + (i >> 3)) =
179 BLK(ctx->prev, bx + (i & 0x7), by + (i >> 3));
180 }
181 } else { // handle four 4x4 subblocks
da2e774f
LI
182 if (!bytestream2_get_bytes_left(&ctx->g)) {
183 av_log(ctx->avctx, AV_LOG_ERROR, "Data overrun\n");
184 return AVERROR_INVALIDDATA;
185 }
fd7b1991
KS
186 for (i = 0; i < 4; i++) {
187 l0x = bx + (i & 1) * 4;
188 l0y = by + (i & 2) * 2;
da2e774f 189 kmvc_getbit(bb, &ctx->g, res);
fd7b1991 190 if (!res) {
da2e774f 191 kmvc_getbit(bb, &ctx->g, res);
fd7b1991 192 if (!res) { // fill whole 4x4 block
da2e774f 193 val = bytestream2_get_byte(&ctx->g);
fd7b1991
KS
194 for (j = 0; j < 16; j++)
195 BLK(ctx->cur, l0x + (j & 3), l0y + (j >> 2)) = val;
196 } else { // copy block
da2e774f 197 val = bytestream2_get_byte(&ctx->g);
fd7b1991
KS
198 mx = (val & 0xF) - 8;
199 my = (val >> 4) - 8;
200 for (j = 0; j < 16; j++)
201 BLK(ctx->cur, l0x + (j & 3), l0y + (j >> 2)) =
202 BLK(ctx->prev, l0x + (j & 3) + mx, l0y + (j >> 2) + my);
203 }
204 } else { // descend to 2x2 sub-sub-blocks
205 for (j = 0; j < 4; j++) {
206 l1x = l0x + (j & 1) * 2;
207 l1y = l0y + (j & 2);
da2e774f 208 kmvc_getbit(bb, &ctx->g, res);
fd7b1991 209 if (!res) {
da2e774f 210 kmvc_getbit(bb, &ctx->g, res);
fd7b1991 211 if (!res) { // fill whole 2x2 block
da2e774f 212 val = bytestream2_get_byte(&ctx->g);
fd7b1991
KS
213 BLK(ctx->cur, l1x, l1y) = val;
214 BLK(ctx->cur, l1x + 1, l1y) = val;
215 BLK(ctx->cur, l1x, l1y + 1) = val;
216 BLK(ctx->cur, l1x + 1, l1y + 1) = val;
217 } else { // copy block
da2e774f 218 val = bytestream2_get_byte(&ctx->g);
fd7b1991
KS
219 mx = (val & 0xF) - 8;
220 my = (val >> 4) - 8;
221 BLK(ctx->cur, l1x, l1y) = BLK(ctx->prev, l1x + mx, l1y + my);
222 BLK(ctx->cur, l1x + 1, l1y) =
223 BLK(ctx->prev, l1x + 1 + mx, l1y + my);
224 BLK(ctx->cur, l1x, l1y + 1) =
225 BLK(ctx->prev, l1x + mx, l1y + 1 + my);
226 BLK(ctx->cur, l1x + 1, l1y + 1) =
227 BLK(ctx->prev, l1x + 1 + mx, l1y + 1 + my);
228 }
229 } else { // read values for block
da2e774f
LI
230 BLK(ctx->cur, l1x, l1y) = bytestream2_get_byte(&ctx->g);
231 BLK(ctx->cur, l1x + 1, l1y) = bytestream2_get_byte(&ctx->g);
232 BLK(ctx->cur, l1x, l1y + 1) = bytestream2_get_byte(&ctx->g);
233 BLK(ctx->cur, l1x + 1, l1y + 1) = bytestream2_get_byte(&ctx->g);
fd7b1991
KS
234 }
235 }
236 }
237 }
238 }
239 }
ad3161ec
GN
240
241 return 0;
fd7b1991
KS
242}
243
7a00bbad 244static int decode_frame(AVCodecContext * avctx, void *data, int *data_size, AVPacket *avpkt)
fd7b1991 245{
e4141433 246 KmvcContext *const ctx = avctx->priv_data;
fd7b1991
KS
247 uint8_t *out, *src;
248 int i;
249 int header;
250 int blocksize;
2d8591c2 251 const uint8_t *pal = av_packet_get_side_data(avpkt, AV_PKT_DATA_PALETTE, NULL);
fd7b1991 252
da2e774f 253 bytestream2_init(&ctx->g, avpkt->data, avpkt->size);
fd7b1991
KS
254 if (ctx->pic.data[0])
255 avctx->release_buffer(avctx, &ctx->pic);
256
257 ctx->pic.reference = 1;
258 ctx->pic.buffer_hints = FF_BUFFER_HINTS_VALID;
259 if (avctx->get_buffer(avctx, &ctx->pic) < 0) {
260 av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
261 return -1;
262 }
263
da2e774f 264 header = bytestream2_get_byte(&ctx->g);
2d2b86c2
KS
265
266 /* blocksize 127 is really palette change event */
da2e774f
LI
267 if (bytestream2_peek_byte(&ctx->g) == 127) {
268 bytestream2_skip(&ctx->g, 3);
2d2b86c2 269 for (i = 0; i < 127; i++) {
da2e774f
LI
270 ctx->pal[i + (header & 0x81)] = bytestream2_get_be24(&ctx->g);
271 bytestream2_skip(&ctx->g, 1);
2d2b86c2 272 }
da2e774f 273 bytestream2_seek(&ctx->g, -127 * 4 - 3, SEEK_CUR);
2d2b86c2 274 }
fd7b1991
KS
275
276 if (header & KMVC_KEYFRAME) {
277 ctx->pic.key_frame = 1;
975a1447 278 ctx->pic.pict_type = AV_PICTURE_TYPE_I;
fd7b1991
KS
279 } else {
280 ctx->pic.key_frame = 0;
975a1447 281 ctx->pic.pict_type = AV_PICTURE_TYPE_P;
fd7b1991
KS
282 }
283
284 if (header & KMVC_PALETTE) {
285 ctx->pic.palette_has_changed = 1;
286 // palette starts from index 1 and has 127 entries
287 for (i = 1; i <= ctx->palsize; i++) {
da2e774f 288 ctx->pal[i] = bytestream2_get_be24(&ctx->g);
fd7b1991
KS
289 }
290 }
291
2d8591c2
KS
292 if (pal) {
293 ctx->pic.palette_has_changed = 1;
294 memcpy(ctx->pal, pal, AVPALETTE_SIZE);
295 }
296
fd7b1991
KS
297 if (ctx->setpal) {
298 ctx->setpal = 0;
299 ctx->pic.palette_has_changed = 1;
300 }
301
302 /* make the palette available on the way out */
303 memcpy(ctx->pic.data[1], ctx->pal, 1024);
304
da2e774f 305 blocksize = bytestream2_get_byte(&ctx->g);
fd7b1991 306
2d2b86c2 307 if (blocksize != 8 && blocksize != 127) {
fd7b1991
KS
308 av_log(avctx, AV_LOG_ERROR, "Block size = %i\n", blocksize);
309 return -1;
310 }
311 memset(ctx->cur, 0, 320 * 200);
312 switch (header & KMVC_METHOD) {
2d2b86c2
KS
313 case 0:
314 case 1: // used in palette changed event
315 memcpy(ctx->cur, ctx->prev, 320 * 200);
316 break;
fd7b1991 317 case 3:
da2e774f 318 kmvc_decode_intra_8x8(ctx, avctx->width, avctx->height);
fd7b1991
KS
319 break;
320 case 4:
da2e774f 321 kmvc_decode_inter_8x8(ctx, avctx->width, avctx->height);
fd7b1991
KS
322 break;
323 default:
324 av_log(avctx, AV_LOG_ERROR, "Unknown compression method %i\n", header & KMVC_METHOD);
325 return -1;
326 }
327
328 out = ctx->pic.data[0];
329 src = ctx->cur;
330 for (i = 0; i < avctx->height; i++) {
331 memcpy(out, src, avctx->width);
332 src += 320;
333 out += ctx->pic.linesize[0];
334 }
335
336 /* flip buffers */
337 if (ctx->cur == ctx->frm0) {
338 ctx->cur = ctx->frm1;
339 ctx->prev = ctx->frm0;
340 } else {
341 ctx->cur = ctx->frm0;
342 ctx->prev = ctx->frm1;
343 }
344
345 *data_size = sizeof(AVFrame);
346 *(AVFrame *) data = ctx->pic;
347
348 /* always report that the buffer was completely consumed */
da2e774f 349 return avpkt->size;
fd7b1991
KS
350}
351
352
353
354/*
355 * Init kmvc decoder
356 */
98a6fff9 357static av_cold int decode_init(AVCodecContext * avctx)
fd7b1991 358{
e4141433 359 KmvcContext *const c = avctx->priv_data;
fd7b1991
KS
360 int i;
361
362 c->avctx = avctx;
fd7b1991 363
fd7b1991
KS
364 if (avctx->width > 320 || avctx->height > 200) {
365 av_log(avctx, AV_LOG_ERROR, "KMVC supports frames <= 320x200\n");
366 return -1;
367 }
368
369 c->frm0 = av_mallocz(320 * 200);
370 c->frm1 = av_mallocz(320 * 200);
371 c->cur = c->frm0;
372 c->prev = c->frm1;
373
374 for (i = 0; i < 256; i++) {
375 c->pal[i] = i * 0x10101;
376 }
377
378 if (avctx->extradata_size < 12) {
379 av_log(NULL, 0, "Extradata missing, decoding may not work properly...\n");
380 c->palsize = 127;
381 } else {
fead30d4 382 c->palsize = AV_RL16(avctx->extradata + 10);
fd7b1991
KS
383 }
384
385 if (avctx->extradata_size == 1036) { // palette in extradata
386 uint8_t *src = avctx->extradata + 12;
2d2b86c2 387 for (i = 0; i < 256; i++) {
fead30d4 388 c->pal[i] = AV_RL32(src);
fd7b1991
KS
389 src += 4;
390 }
391 c->setpal = 1;
392 }
393
394 avctx->pix_fmt = PIX_FMT_PAL8;
395
396 return 0;
397}
398
399
400
401/*
402 * Uninit kmvc decoder
403 */
98a6fff9 404static av_cold int decode_end(AVCodecContext * avctx)
fd7b1991 405{
e4141433 406 KmvcContext *const c = avctx->priv_data;
fd7b1991 407
34a8dcd0
KS
408 av_freep(&c->frm0);
409 av_freep(&c->frm1);
fd7b1991
KS
410 if (c->pic.data[0])
411 avctx->release_buffer(avctx, &c->pic);
412
413 return 0;
414}
415
d36beb3f 416AVCodec ff_kmvc_decoder = {
ec6402b7
AK
417 .name = "kmvc",
418 .type = AVMEDIA_TYPE_VIDEO,
419 .id = CODEC_ID_KMVC,
420 .priv_data_size = sizeof(KmvcContext),
421 .init = decode_init,
422 .close = decode_end,
423 .decode = decode_frame,
424 .capabilities = CODEC_CAP_DR1,
fe4bf374 425 .long_name = NULL_IF_CONFIG_SMALL("Karl Morton's video codec"),
fd7b1991 426};