3f173a596bb2054ee47ae805f3d98e446ad3d9c9
[libav.git] / libavcodec / qtrle.c
1 /*
2 * Quicktime Animation (RLE) Video Decoder
3 * Copyright (C) 2004 the ffmpeg project
4 *
5 * This file is part of Libav.
6 *
7 * Libav is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * Libav is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with Libav; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22 /**
23 * @file
24 * QT RLE Video Decoder by Mike Melanson (melanson@pcisys.net)
25 * For more information about the QT RLE format, visit:
26 * http://www.pcisys.net/~melanson/codecs/
27 *
28 * The QT RLE decoder has seven modes of operation:
29 * 1, 2, 4, 8, 16, 24, and 32 bits per pixel. For modes 1, 2, 4, and 8
30 * the decoder outputs PAL8 colorspace data. 16-bit data yields RGB555
31 * data. 24-bit data is RGB24 and 32-bit data is RGB32.
32 */
33
34 #include <stdio.h>
35 #include <stdlib.h>
36 #include <string.h>
37
38 #include "avcodec.h"
39 #include "bytestream.h"
40
41 typedef struct QtrleContext {
42 AVCodecContext *avctx;
43 AVFrame frame;
44
45 GetByteContext g;
46 uint32_t pal[256];
47 } QtrleContext;
48
49 #define CHECK_PIXEL_PTR(n) \
50 if ((pixel_ptr + n > pixel_limit) || (pixel_ptr + n < 0)) { \
51 av_log (s->avctx, AV_LOG_INFO, "Problem: pixel_ptr = %d, pixel_limit = %d\n", \
52 pixel_ptr + n, pixel_limit); \
53 return; \
54 } \
55
56 static void qtrle_decode_1bpp(QtrleContext *s, int row_ptr, int lines_to_change)
57 {
58 int rle_code;
59 int pixel_ptr = 0;
60 int row_inc = s->frame.linesize[0];
61 unsigned char pi0, pi1; /* 2 8-pixel values */
62 unsigned char *rgb = s->frame.data[0];
63 int pixel_limit = s->frame.linesize[0] * s->avctx->height;
64 int skip;
65
66 while (lines_to_change) {
67 skip = bytestream2_get_byte(&s->g);
68 rle_code = (signed char)bytestream2_get_byte(&s->g);
69 if (rle_code == 0)
70 break;
71 if(skip & 0x80) {
72 lines_to_change--;
73 row_ptr += row_inc;
74 pixel_ptr = row_ptr + 2 * (skip & 0x7f);
75 } else
76 pixel_ptr += 2 * skip;
77 CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
78
79 if (rle_code < 0) {
80 /* decode the run length code */
81 rle_code = -rle_code;
82 /* get the next 2 bytes from the stream, treat them as groups
83 * of 8 pixels, and output them rle_code times */
84
85 pi0 = bytestream2_get_byte(&s->g);
86 pi1 = bytestream2_get_byte(&s->g);
87 CHECK_PIXEL_PTR(rle_code * 2);
88
89 while (rle_code--) {
90 rgb[pixel_ptr++] = pi0;
91 rgb[pixel_ptr++] = pi1;
92 }
93 } else {
94 /* copy the same pixel directly to output 2 times */
95 rle_code *= 2;
96 CHECK_PIXEL_PTR(rle_code);
97
98 while (rle_code--)
99 rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
100 }
101 }
102 }
103
104 static inline void qtrle_decode_2n4bpp(QtrleContext *s, int row_ptr,
105 int lines_to_change, int bpp)
106 {
107 int rle_code, i;
108 int pixel_ptr;
109 int row_inc = s->frame.linesize[0];
110 unsigned char pi[16]; /* 16 palette indices */
111 unsigned char *rgb = s->frame.data[0];
112 int pixel_limit = s->frame.linesize[0] * s->avctx->height;
113 int num_pixels = (bpp == 4) ? 8 : 16;
114
115 while (lines_to_change--) {
116 pixel_ptr = row_ptr + (num_pixels * (bytestream2_get_byte(&s->g) - 1));
117
118 while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
119 if (rle_code == 0) {
120 /* there's another skip code in the stream */
121 pixel_ptr += (num_pixels * (bytestream2_get_byte(&s->g) - 1));
122 CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
123 } else if (rle_code < 0) {
124 /* decode the run length code */
125 rle_code = -rle_code;
126 /* get the next 4 bytes from the stream, treat them as palette
127 * indexes, and output them rle_code times */
128 for (i = num_pixels-1; i >= 0; i--) {
129 pi[num_pixels-1-i] = (bytestream2_peek_byte(&s->g) >> ((i*bpp) & 0x07)) & ((1<<bpp)-1);
130 bytestream2_skip(&s->g, ((i & ((num_pixels>>2)-1)) == 0));
131 }
132 CHECK_PIXEL_PTR(rle_code * num_pixels);
133 while (rle_code--) {
134 for (i = 0; i < num_pixels; i++)
135 rgb[pixel_ptr++] = pi[i];
136 }
137 } else {
138 /* copy the same pixel directly to output 4 times */
139 rle_code *= 4;
140 CHECK_PIXEL_PTR(rle_code*(num_pixels>>2));
141 while (rle_code--) {
142 if(bpp == 4) {
143 int x = bytestream2_get_byte(&s->g);
144 rgb[pixel_ptr++] = (x >> 4) & 0x0f;
145 rgb[pixel_ptr++] = x & 0x0f;
146 } else {
147 int x = bytestream2_get_byte(&s->g);
148 rgb[pixel_ptr++] = (x >> 6) & 0x03;
149 rgb[pixel_ptr++] = (x >> 4) & 0x03;
150 rgb[pixel_ptr++] = (x >> 2) & 0x03;
151 rgb[pixel_ptr++] = x & 0x03;
152 }
153 }
154 }
155 }
156 row_ptr += row_inc;
157 }
158 }
159
160 static void qtrle_decode_8bpp(QtrleContext *s, int row_ptr, int lines_to_change)
161 {
162 int rle_code;
163 int pixel_ptr;
164 int row_inc = s->frame.linesize[0];
165 unsigned char pi1, pi2, pi3, pi4; /* 4 palette indexes */
166 unsigned char *rgb = s->frame.data[0];
167 int pixel_limit = s->frame.linesize[0] * s->avctx->height;
168
169 while (lines_to_change--) {
170 pixel_ptr = row_ptr + (4 * (bytestream2_get_byte(&s->g) - 1));
171
172 while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
173 if (rle_code == 0) {
174 /* there's another skip code in the stream */
175 pixel_ptr += (4 * (bytestream2_get_byte(&s->g) - 1));
176 CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
177 } else if (rle_code < 0) {
178 /* decode the run length code */
179 rle_code = -rle_code;
180 /* get the next 4 bytes from the stream, treat them as palette
181 * indexes, and output them rle_code times */
182 pi1 = bytestream2_get_byte(&s->g);
183 pi2 = bytestream2_get_byte(&s->g);
184 pi3 = bytestream2_get_byte(&s->g);
185 pi4 = bytestream2_get_byte(&s->g);
186
187 CHECK_PIXEL_PTR(rle_code * 4);
188
189 while (rle_code--) {
190 rgb[pixel_ptr++] = pi1;
191 rgb[pixel_ptr++] = pi2;
192 rgb[pixel_ptr++] = pi3;
193 rgb[pixel_ptr++] = pi4;
194 }
195 } else {
196 /* copy the same pixel directly to output 4 times */
197 rle_code *= 4;
198 CHECK_PIXEL_PTR(rle_code);
199
200 while (rle_code--) {
201 rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
202 }
203 }
204 }
205 row_ptr += row_inc;
206 }
207 }
208
209 static void qtrle_decode_16bpp(QtrleContext *s, int row_ptr, int lines_to_change)
210 {
211 int rle_code;
212 int pixel_ptr;
213 int row_inc = s->frame.linesize[0];
214 unsigned short rgb16;
215 unsigned char *rgb = s->frame.data[0];
216 int pixel_limit = s->frame.linesize[0] * s->avctx->height;
217
218 while (lines_to_change--) {
219 pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 2;
220
221 while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
222 if (rle_code == 0) {
223 /* there's another skip code in the stream */
224 pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 2;
225 CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
226 } else if (rle_code < 0) {
227 /* decode the run length code */
228 rle_code = -rle_code;
229 rgb16 = bytestream2_get_be16(&s->g);
230
231 CHECK_PIXEL_PTR(rle_code * 2);
232
233 while (rle_code--) {
234 *(unsigned short *)(&rgb[pixel_ptr]) = rgb16;
235 pixel_ptr += 2;
236 }
237 } else {
238 CHECK_PIXEL_PTR(rle_code * 2);
239
240 /* copy pixels directly to output */
241 while (rle_code--) {
242 rgb16 = bytestream2_get_be16(&s->g);
243 *(unsigned short *)(&rgb[pixel_ptr]) = rgb16;
244 pixel_ptr += 2;
245 }
246 }
247 }
248 row_ptr += row_inc;
249 }
250 }
251
252 static void qtrle_decode_24bpp(QtrleContext *s, int row_ptr, int lines_to_change)
253 {
254 int rle_code;
255 int pixel_ptr;
256 int row_inc = s->frame.linesize[0];
257 unsigned char r, g, b;
258 unsigned char *rgb = s->frame.data[0];
259 int pixel_limit = s->frame.linesize[0] * s->avctx->height;
260
261 while (lines_to_change--) {
262 pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 3;
263
264 while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
265 if (rle_code == 0) {
266 /* there's another skip code in the stream */
267 pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 3;
268 CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
269 } else if (rle_code < 0) {
270 /* decode the run length code */
271 rle_code = -rle_code;
272 r = bytestream2_get_byte(&s->g);
273 g = bytestream2_get_byte(&s->g);
274 b = bytestream2_get_byte(&s->g);
275
276 CHECK_PIXEL_PTR(rle_code * 3);
277
278 while (rle_code--) {
279 rgb[pixel_ptr++] = r;
280 rgb[pixel_ptr++] = g;
281 rgb[pixel_ptr++] = b;
282 }
283 } else {
284 CHECK_PIXEL_PTR(rle_code * 3);
285
286 /* copy pixels directly to output */
287 while (rle_code--) {
288 rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
289 rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
290 rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
291 }
292 }
293 }
294 row_ptr += row_inc;
295 }
296 }
297
298 static void qtrle_decode_32bpp(QtrleContext *s, int row_ptr, int lines_to_change)
299 {
300 int rle_code;
301 int pixel_ptr;
302 int row_inc = s->frame.linesize[0];
303 unsigned int argb;
304 unsigned char *rgb = s->frame.data[0];
305 int pixel_limit = s->frame.linesize[0] * s->avctx->height;
306
307 while (lines_to_change--) {
308 pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 4;
309
310 while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
311 if (rle_code == 0) {
312 /* there's another skip code in the stream */
313 pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 4;
314 CHECK_PIXEL_PTR(0); /* make sure pixel_ptr is positive */
315 } else if (rle_code < 0) {
316 /* decode the run length code */
317 rle_code = -rle_code;
318 argb = bytestream2_get_be32(&s->g);
319
320 CHECK_PIXEL_PTR(rle_code * 4);
321
322 while (rle_code--) {
323 AV_WN32A(rgb + pixel_ptr, argb);
324 pixel_ptr += 4;
325 }
326 } else {
327 CHECK_PIXEL_PTR(rle_code * 4);
328
329 /* copy pixels directly to output */
330 while (rle_code--) {
331 argb = bytestream2_get_be32(&s->g);
332 AV_WN32A(rgb + pixel_ptr, argb);
333 pixel_ptr += 4;
334 }
335 }
336 }
337 row_ptr += row_inc;
338 }
339 }
340
341 static av_cold int qtrle_decode_init(AVCodecContext *avctx)
342 {
343 QtrleContext *s = avctx->priv_data;
344
345 s->avctx = avctx;
346 switch (avctx->bits_per_coded_sample) {
347 case 1:
348 case 33:
349 avctx->pix_fmt = AV_PIX_FMT_MONOWHITE;
350 break;
351
352 case 2:
353 case 4:
354 case 8:
355 case 34:
356 case 36:
357 case 40:
358 avctx->pix_fmt = AV_PIX_FMT_PAL8;
359 break;
360
361 case 16:
362 avctx->pix_fmt = AV_PIX_FMT_RGB555;
363 break;
364
365 case 24:
366 avctx->pix_fmt = AV_PIX_FMT_RGB24;
367 break;
368
369 case 32:
370 avctx->pix_fmt = AV_PIX_FMT_RGB32;
371 break;
372
373 default:
374 av_log (avctx, AV_LOG_ERROR, "Unsupported colorspace: %d bits/sample?\n",
375 avctx->bits_per_coded_sample);
376 return AVERROR_INVALIDDATA;
377 }
378
379 s->frame.data[0] = NULL;
380
381 return 0;
382 }
383
384 static int qtrle_decode_frame(AVCodecContext *avctx,
385 void *data, int *data_size,
386 AVPacket *avpkt)
387 {
388 QtrleContext *s = avctx->priv_data;
389 int header, start_line;
390 int height, row_ptr;
391 int has_palette = 0;
392
393 bytestream2_init(&s->g, avpkt->data, avpkt->size);
394 s->frame.reference = 1;
395 s->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE |
396 FF_BUFFER_HINTS_REUSABLE | FF_BUFFER_HINTS_READABLE;
397 if (avctx->reget_buffer(avctx, &s->frame)) {
398 av_log (s->avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
399 return -1;
400 }
401
402 /* check if this frame is even supposed to change */
403 if (avpkt->size < 8)
404 goto done;
405
406 /* start after the chunk size */
407 bytestream2_seek(&s->g, 4, SEEK_SET);
408
409 /* fetch the header */
410 header = bytestream2_get_be16(&s->g);
411
412 /* if a header is present, fetch additional decoding parameters */
413 if (header & 0x0008) {
414 if (avpkt->size < 14)
415 goto done;
416 start_line = bytestream2_get_be16(&s->g);
417 bytestream2_skip(&s->g, 2);
418 height = bytestream2_get_be16(&s->g);
419 bytestream2_skip(&s->g, 2);
420 } else {
421 start_line = 0;
422 height = s->avctx->height;
423 }
424 row_ptr = s->frame.linesize[0] * start_line;
425
426 switch (avctx->bits_per_coded_sample) {
427 case 1:
428 case 33:
429 qtrle_decode_1bpp(s, row_ptr, height);
430 break;
431
432 case 2:
433 case 34:
434 qtrle_decode_2n4bpp(s, row_ptr, height, 2);
435 has_palette = 1;
436 break;
437
438 case 4:
439 case 36:
440 qtrle_decode_2n4bpp(s, row_ptr, height, 4);
441 has_palette = 1;
442 break;
443
444 case 8:
445 case 40:
446 qtrle_decode_8bpp(s, row_ptr, height);
447 has_palette = 1;
448 break;
449
450 case 16:
451 qtrle_decode_16bpp(s, row_ptr, height);
452 break;
453
454 case 24:
455 qtrle_decode_24bpp(s, row_ptr, height);
456 break;
457
458 case 32:
459 qtrle_decode_32bpp(s, row_ptr, height);
460 break;
461
462 default:
463 av_log (s->avctx, AV_LOG_ERROR, "Unsupported colorspace: %d bits/sample?\n",
464 avctx->bits_per_coded_sample);
465 break;
466 }
467
468 if(has_palette) {
469 const uint8_t *pal = av_packet_get_side_data(avpkt, AV_PKT_DATA_PALETTE, NULL);
470
471 if (pal) {
472 s->frame.palette_has_changed = 1;
473 memcpy(s->pal, pal, AVPALETTE_SIZE);
474 }
475
476 /* make the palette available on the way out */
477 memcpy(s->frame.data[1], s->pal, AVPALETTE_SIZE);
478 }
479
480 done:
481 *data_size = sizeof(AVFrame);
482 *(AVFrame*)data = s->frame;
483
484 /* always report that the buffer was completely consumed */
485 return avpkt->size;
486 }
487
488 static av_cold int qtrle_decode_end(AVCodecContext *avctx)
489 {
490 QtrleContext *s = avctx->priv_data;
491
492 if (s->frame.data[0])
493 avctx->release_buffer(avctx, &s->frame);
494
495 return 0;
496 }
497
498 AVCodec ff_qtrle_decoder = {
499 .name = "qtrle",
500 .type = AVMEDIA_TYPE_VIDEO,
501 .id = AV_CODEC_ID_QTRLE,
502 .priv_data_size = sizeof(QtrleContext),
503 .init = qtrle_decode_init,
504 .close = qtrle_decode_end,
505 .decode = qtrle_decode_frame,
506 .capabilities = CODEC_CAP_DR1,
507 .long_name = NULL_IF_CONFIG_SMALL("QuickTime Animation (RLE) video"),
508 };