various security fixes and precautionary checks
[libav.git] / libavcodec / flicvideo.c
1 /*
2 * FLI/FLC Animation Video Decoder
3 * Copyright (C) 2003, 2004 the ffmpeg project
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 *
19 */
20
21 /**
22 * @file flic.c
23 * Autodesk Animator FLI/FLC Video Decoder
24 * by Mike Melanson (melanson@pcisys.net)
25 * for more information on the .fli/.flc file format and all of its many
26 * variations, visit:
27 * http://www.compuphase.com/flic.htm
28 *
29 * This decoder outputs PAL8 colorspace data. To use this decoder, be
30 * sure that your demuxer sends the FLI file header to the decoder via
31 * the extradata chunk in AVCodecContext. The chunk should be 128 bytes
32 * large. The only exception is for FLI files from the game "Magic Carpet",
33 * in which the header is only 12 bytes.
34 */
35
36 #include <stdio.h>
37 #include <stdlib.h>
38 #include <string.h>
39 #include <unistd.h>
40
41 #include "common.h"
42 #include "avcodec.h"
43 #include "bswap.h"
44
45 #define FLI_256_COLOR 4
46 #define FLI_DELTA 7
47 #define FLI_COLOR 11
48 #define FLI_LC 12
49 #define FLI_BLACK 13
50 #define FLI_BRUN 15
51 #define FLI_COPY 16
52 #define FLI_MINI 18
53
54 typedef struct FlicDecodeContext {
55 AVCodecContext *avctx;
56 AVFrame frame;
57
58 unsigned int palette[256];
59 int new_palette;
60 int fli_type; /* either 0xAF11 or 0xAF12, affects palette resolution */
61 } FlicDecodeContext;
62
63 static int flic_decode_init(AVCodecContext *avctx)
64 {
65 FlicDecodeContext *s = (FlicDecodeContext *)avctx->priv_data;
66 unsigned char *fli_header = (unsigned char *)avctx->extradata;
67
68 s->avctx = avctx;
69 avctx->pix_fmt = PIX_FMT_PAL8;
70 avctx->has_b_frames = 0;
71
72 if (s->avctx->extradata_size == 12) {
73 /* special case for magic carpet FLIs */
74 s->fli_type = 0xAF13;
75 } else if (s->avctx->extradata_size == 128) {
76 s->fli_type = LE_16(&fli_header[4]);
77 } else {
78 av_log(avctx, AV_LOG_ERROR, "Expected extradata of 12 or 128 bytes\n");
79 return -1;
80 }
81
82 s->frame.data[0] = NULL;
83 s->new_palette = 0;
84
85 return 0;
86 }
87
88 static int flic_decode_frame(AVCodecContext *avctx,
89 void *data, int *data_size,
90 uint8_t *buf, int buf_size)
91 {
92 FlicDecodeContext *s = (FlicDecodeContext *)avctx->priv_data;
93
94 int stream_ptr = 0;
95 int stream_ptr_after_color_chunk;
96 int pixel_ptr;
97 int palette_ptr;
98 unsigned char palette_idx1;
99 unsigned char palette_idx2;
100
101 unsigned int frame_size;
102 int num_chunks;
103
104 unsigned int chunk_size;
105 int chunk_type;
106
107 int i, j;
108
109 int color_packets;
110 int color_changes;
111 int color_shift;
112 unsigned char r, g, b;
113
114 int lines;
115 int compressed_lines;
116 int starting_line;
117 signed short line_packets;
118 int y_ptr;
119 signed char byte_run;
120 int pixel_skip;
121 int pixel_countdown;
122 unsigned char *pixels;
123
124 s->frame.reference = 1;
125 s->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE;
126 if (avctx->reget_buffer(avctx, &s->frame) < 0) {
127 av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
128 return -1;
129 }
130
131 pixels = s->frame.data[0];
132
133 frame_size = LE_32(&buf[stream_ptr]);
134 stream_ptr += 6; /* skip the magic number */
135 num_chunks = LE_16(&buf[stream_ptr]);
136 stream_ptr += 10; /* skip padding */
137
138 frame_size -= 16;
139
140 /* iterate through the chunks */
141 while ((frame_size > 0) && (num_chunks > 0)) {
142 chunk_size = LE_32(&buf[stream_ptr]);
143 stream_ptr += 4;
144 chunk_type = LE_16(&buf[stream_ptr]);
145 stream_ptr += 2;
146
147 switch (chunk_type) {
148 case FLI_256_COLOR:
149 case FLI_COLOR:
150 stream_ptr_after_color_chunk = stream_ptr + chunk_size - 6;
151 s->new_palette = 1;
152
153 /* check special case: If this file is from the Magic Carpet
154 * game and uses 6-bit colors even though it reports 256-color
155 * chunks in a 0xAF12-type file (fli_type is set to 0xAF13 during
156 * initialization) */
157 if ((chunk_type == FLI_256_COLOR) && (s->fli_type != 0xAF13))
158 color_shift = 0;
159 else
160 color_shift = 2;
161 /* set up the palette */
162 color_packets = LE_16(&buf[stream_ptr]);
163 stream_ptr += 2;
164 palette_ptr = 0;
165 for (i = 0; i < color_packets; i++) {
166 /* first byte is how many colors to skip */
167 palette_ptr += buf[stream_ptr++];
168
169 /* next byte indicates how many entries to change */
170 color_changes = buf[stream_ptr++];
171
172 /* if there are 0 color changes, there are actually 256 */
173 if (color_changes == 0)
174 color_changes = 256;
175
176 for (j = 0; j < color_changes; j++) {
177
178 /* wrap around, for good measure */
179 if ((unsigned)palette_ptr >= 256)
180 palette_ptr = 0;
181
182 r = buf[stream_ptr++] << color_shift;
183 g = buf[stream_ptr++] << color_shift;
184 b = buf[stream_ptr++] << color_shift;
185 s->palette[palette_ptr++] = (r << 16) | (g << 8) | b;
186 }
187 }
188
189 /* color chunks sometimes have weird 16-bit alignment issues;
190 * therefore, take the hardline approach and set the stream_ptr
191 * to the value calculated w.r.t. the size specified by the color
192 * chunk header */
193 stream_ptr = stream_ptr_after_color_chunk;
194
195 break;
196
197 case FLI_DELTA:
198 y_ptr = 0;
199 compressed_lines = LE_16(&buf[stream_ptr]);
200 stream_ptr += 2;
201 while (compressed_lines > 0) {
202 line_packets = LE_16(&buf[stream_ptr]);
203 stream_ptr += 2;
204 if (line_packets < 0) {
205 line_packets = -line_packets;
206 y_ptr += line_packets * s->frame.linesize[0];
207 } else {
208 compressed_lines--;
209 pixel_ptr = y_ptr;
210 pixel_countdown = s->avctx->width;
211 for (i = 0; i < line_packets; i++) {
212 /* account for the skip bytes */
213 pixel_skip = buf[stream_ptr++];
214 pixel_ptr += pixel_skip;
215 pixel_countdown -= pixel_skip;
216 byte_run = buf[stream_ptr++];
217 if (byte_run < 0) {
218 byte_run = -byte_run;
219 palette_idx1 = buf[stream_ptr++];
220 palette_idx2 = buf[stream_ptr++];
221 for (j = 0; j < byte_run; j++, pixel_countdown -= 2) {
222 pixels[pixel_ptr++] = palette_idx1;
223 pixels[pixel_ptr++] = palette_idx2;
224 }
225 } else {
226 for (j = 0; j < byte_run * 2; j++, pixel_countdown--) {
227 palette_idx1 = buf[stream_ptr++];
228 pixels[pixel_ptr++] = palette_idx1;
229 }
230 }
231 }
232
233 y_ptr += s->frame.linesize[0];
234 }
235 }
236 break;
237
238 case FLI_LC:
239 /* line compressed */
240 starting_line = LE_16(&buf[stream_ptr]);
241 stream_ptr += 2;
242 y_ptr = 0;
243 y_ptr += starting_line * s->frame.linesize[0];
244
245 compressed_lines = LE_16(&buf[stream_ptr]);
246 stream_ptr += 2;
247 while (compressed_lines > 0) {
248 pixel_ptr = y_ptr;
249 pixel_countdown = s->avctx->width;
250 line_packets = buf[stream_ptr++];
251 if (line_packets > 0) {
252 for (i = 0; i < line_packets; i++) {
253 /* account for the skip bytes */
254 pixel_skip = buf[stream_ptr++];
255 pixel_ptr += pixel_skip;
256 pixel_countdown -= pixel_skip;
257 byte_run = buf[stream_ptr++];
258 if (byte_run > 0) {
259 for (j = 0; j < byte_run; j++, pixel_countdown--) {
260 palette_idx1 = buf[stream_ptr++];
261 pixels[pixel_ptr++] = palette_idx1;
262 }
263 } else {
264 byte_run = -byte_run;
265 palette_idx1 = buf[stream_ptr++];
266 for (j = 0; j < byte_run; j++, pixel_countdown--) {
267 pixels[pixel_ptr++] = palette_idx1;
268 }
269 }
270 }
271 }
272
273 y_ptr += s->frame.linesize[0];
274 compressed_lines--;
275 }
276 break;
277
278 case FLI_BLACK:
279 /* set the whole frame to color 0 (which is usually black) */
280 memset(pixels, 0,
281 s->frame.linesize[0] * s->avctx->height);
282 break;
283
284 case FLI_BRUN:
285 /* Byte run compression: This chunk type only occurs in the first
286 * FLI frame and it will update the entire frame. */
287 y_ptr = 0;
288 for (lines = 0; lines < s->avctx->height; lines++) {
289 pixel_ptr = y_ptr;
290 /* disregard the line packets; instead, iterate through all
291 * pixels on a row */
292 stream_ptr++;
293 pixel_countdown = s->avctx->width;
294 while (pixel_countdown > 0) {
295 byte_run = buf[stream_ptr++];
296 if (byte_run > 0) {
297 palette_idx1 = buf[stream_ptr++];
298 for (j = 0; j < byte_run; j++) {
299 pixels[pixel_ptr++] = palette_idx1;
300 pixel_countdown--;
301 if (pixel_countdown < 0)
302 av_log(avctx, AV_LOG_ERROR, "pixel_countdown < 0 (%d)\n",
303 pixel_countdown);
304 }
305 } else { /* copy bytes if byte_run < 0 */
306 byte_run = -byte_run;
307 for (j = 0; j < byte_run; j++) {
308 palette_idx1 = buf[stream_ptr++];
309 pixels[pixel_ptr++] = palette_idx1;
310 pixel_countdown--;
311 if (pixel_countdown < 0)
312 av_log(avctx, AV_LOG_ERROR, "pixel_countdown < 0 (%d)\n",
313 pixel_countdown);
314 }
315 }
316 }
317
318 y_ptr += s->frame.linesize[0];
319 }
320 break;
321
322 case FLI_COPY:
323 /* copy the chunk (uncompressed frame) */
324 if (chunk_size - 6 > s->avctx->width * s->avctx->height) {
325 av_log(avctx, AV_LOG_ERROR, "In chunk FLI_COPY : source data (%d bytes) " \
326 "bigger than image, skipping chunk\n", chunk_size - 6);
327 stream_ptr += chunk_size - 6;
328 } else {
329 for (y_ptr = 0; y_ptr < s->frame.linesize[0] * s->avctx->height;
330 y_ptr += s->frame.linesize[0]) {
331 memcpy(&pixels[y_ptr], &buf[stream_ptr],
332 s->avctx->width);
333 stream_ptr += s->avctx->width;
334 }
335 }
336 break;
337
338 case FLI_MINI:
339 /* some sort of a thumbnail? disregard this chunk... */
340 stream_ptr += chunk_size - 6;
341 break;
342
343 default:
344 av_log(avctx, AV_LOG_ERROR, "Unrecognized chunk type: %d\n", chunk_type);
345 break;
346 }
347
348 frame_size -= chunk_size;
349 num_chunks--;
350 }
351
352 /* by the end of the chunk, the stream ptr should equal the frame
353 * size (minus 1, possibly); if it doesn't, issue a warning */
354 if ((stream_ptr != buf_size) && (stream_ptr != buf_size - 1))
355 av_log(avctx, AV_LOG_ERROR, "Processed FLI chunk where chunk size = %d " \
356 "and final chunk ptr = %d\n", buf_size, stream_ptr);
357
358 /* make the palette available on the way out */
359 // if (s->new_palette) {
360 if (1) {
361 memcpy(s->frame.data[1], s->palette, AVPALETTE_SIZE);
362 s->frame.palette_has_changed = 1;
363 s->new_palette = 0;
364 }
365
366 *data_size=sizeof(AVFrame);
367 *(AVFrame*)data = s->frame;
368
369 return buf_size;
370 }
371
372 static int flic_decode_end(AVCodecContext *avctx)
373 {
374 FlicDecodeContext *s = avctx->priv_data;
375
376 if (s->frame.data[0])
377 avctx->release_buffer(avctx, &s->frame);
378
379 return 0;
380 }
381
382 AVCodec flic_decoder = {
383 "flic",
384 CODEC_TYPE_VIDEO,
385 CODEC_ID_FLIC,
386 sizeof(FlicDecodeContext),
387 flic_decode_init,
388 NULL,
389 flic_decode_end,
390 flic_decode_frame,
391 CODEC_CAP_DR1,
392 NULL
393 };