Check pointers before writing to memory, fix possible integer overflows
[libav.git] / libavcodec / 8bps.c
1 /*
2 * Quicktime Planar RGB (8BPS) Video Decoder
3 * Copyright (C) 2003 Roberto Togni
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 *
19 */
20
21 /**
22 * @file 8bps.c
23 * QT 8BPS Video Decoder by Roberto Togni <rtogni at bresciaonline dot it>
24 * For more information about the 8BPS format, visit:
25 * http://www.pcisys.net/~melanson/codecs/
26 *
27 * Supports: PAL8 (RGB 8bpp, paletted)
28 * : BGR24 (RGB 24bpp) (can also output it as RGBA32)
29 * : RGBA32 (RGB 32bpp, 4th plane is probably alpha and it's ignored)
30 *
31 */
32
33 #include <stdio.h>
34 #include <stdlib.h>
35
36 #include "common.h"
37 #include "avcodec.h"
38
39
40 const enum PixelFormat pixfmt_rgb24[] = {PIX_FMT_BGR24, PIX_FMT_RGBA32, -1};
41
42 /*
43 * Decoder context
44 */
45 typedef struct EightBpsContext {
46
47 AVCodecContext *avctx;
48 AVFrame pic;
49
50 unsigned char planes;
51 unsigned char planemap[4];
52 } EightBpsContext;
53
54
55 /*
56 *
57 * Decode a frame
58 *
59 */
60 static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8_t *buf, int buf_size)
61 {
62 EightBpsContext * const c = (EightBpsContext *)avctx->priv_data;
63 unsigned char *encoded = (unsigned char *)buf;
64 unsigned char *pixptr, *pixptr_end;
65 unsigned int height = avctx->height; // Real image height
66 unsigned int dlen, p, row;
67 unsigned char *lp, *dp;
68 unsigned char count;
69 unsigned int px_inc;
70 unsigned int planes = c->planes;
71 unsigned char *planemap = c->planemap;
72
73
74 /* no supplementary picture */
75 if (buf_size == 0)
76 return 0;
77
78 if(c->pic.data[0])
79 avctx->release_buffer(avctx, &c->pic);
80
81 c->pic.reference = 0;
82 c->pic.buffer_hints = FF_BUFFER_HINTS_VALID;
83 if(avctx->get_buffer(avctx, &c->pic) < 0){
84 av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
85 return -1;
86 }
87
88 /* Set data pointer after line lengths */
89 dp = encoded + planes * (height << 1);
90
91 /* Ignore alpha plane, don't know what to do with it */
92 if (planes == 4)
93 planes--;
94
95 px_inc = planes + (avctx->pix_fmt == PIX_FMT_RGBA32);
96
97 for (p = 0; p < planes; p++) {
98 /* Lines length pointer for this plane */
99 lp = encoded + p * (height << 1);
100
101 /* Decode a plane */
102 for(row = 0; row < height; row++) {
103 pixptr = c->pic.data[0] + row * c->pic.linesize[0] + planemap[p];
104 pixptr_end = pixptr + c->pic.linesize[0];
105 dlen = be2me_16(*(unsigned short *)(lp+row*2));
106 /* Decode a row of this plane */
107 while(dlen > 0) {
108 if ((count = *dp++) <= 127) {
109 count++;
110 dlen -= count + 1;
111 if (pixptr + count * px_inc > pixptr_end)
112 break;
113 while(count--) {
114 *pixptr = *dp++;
115 pixptr += px_inc;
116 }
117 } else {
118 count = 257 - count;
119 if (pixptr + count * px_inc > pixptr_end)
120 break;
121 while(count--) {
122 *pixptr = *dp;
123 pixptr += px_inc;
124 }
125 dp++;
126 dlen -= 2;
127 }
128 }
129 }
130 }
131
132 if (avctx->palctrl) {
133 memcpy (c->pic.data[1], avctx->palctrl->palette, AVPALETTE_SIZE);
134 if (avctx->palctrl->palette_changed) {
135 c->pic.palette_has_changed = 1;
136 avctx->palctrl->palette_changed = 0;
137 } else
138 c->pic.palette_has_changed = 0;
139 }
140
141 *data_size = sizeof(AVFrame);
142 *(AVFrame*)data = c->pic;
143
144 /* always report that the buffer was completely consumed */
145 return buf_size;
146 }
147
148
149 /*
150 *
151 * Init 8BPS decoder
152 *
153 */
154 static int decode_init(AVCodecContext *avctx)
155 {
156 EightBpsContext * const c = (EightBpsContext *)avctx->priv_data;
157
158 c->avctx = avctx;
159 avctx->has_b_frames = 0;
160
161 c->pic.data[0] = NULL;
162
163 // FIXME: find a better way to prevent integer overflow
164 if (((unsigned int)avctx->width > 32000) || ((unsigned int)avctx->height > 32000)) {
165 av_log(avctx, AV_LOG_ERROR, "Bad image size (w = %d, h = %d).\n", avctx->width, avctx->height);
166 return 1;
167 }
168
169 switch (avctx->bits_per_sample) {
170 case 8:
171 avctx->pix_fmt = PIX_FMT_PAL8;
172 c->planes = 1;
173 c->planemap[0] = 0; // 1st plane is palette indexes
174 if (avctx->palctrl == NULL) {
175 av_log(avctx, AV_LOG_ERROR, "Error: PAL8 format but no palette from demuxer.\n");
176 return -1;
177 }
178 break;
179 case 24:
180 avctx->pix_fmt = avctx->get_format(avctx, pixfmt_rgb24);
181 c->planes = 3;
182 c->planemap[0] = 2; // 1st plane is red
183 c->planemap[1] = 1; // 2nd plane is green
184 c->planemap[2] = 0; // 3rd plane is blue
185 break;
186 case 32:
187 avctx->pix_fmt = PIX_FMT_RGBA32;
188 c->planes = 4;
189 #ifdef WORDS_BIGENDIAN
190 c->planemap[0] = 1; // 1st plane is red
191 c->planemap[1] = 2; // 2nd plane is green
192 c->planemap[2] = 3; // 3rd plane is blue
193 c->planemap[3] = 0; // 4th plane is alpha???
194 #else
195 c->planemap[0] = 2; // 1st plane is red
196 c->planemap[1] = 1; // 2nd plane is green
197 c->planemap[2] = 0; // 3rd plane is blue
198 c->planemap[3] = 3; // 4th plane is alpha???
199 #endif
200 break;
201 default:
202 av_log(avctx, AV_LOG_ERROR, "Error: Unsupported color depth: %u.\n", avctx->bits_per_sample);
203 return -1;
204 }
205
206 return 0;
207 }
208
209
210
211
212 /*
213 *
214 * Uninit 8BPS decoder
215 *
216 */
217 static int decode_end(AVCodecContext *avctx)
218 {
219 EightBpsContext * const c = (EightBpsContext *)avctx->priv_data;
220
221 if (c->pic.data[0])
222 avctx->release_buffer(avctx, &c->pic);
223
224 return 0;
225 }
226
227
228
229 AVCodec eightbps_decoder = {
230 "8bps",
231 CODEC_TYPE_VIDEO,
232 CODEC_ID_8BPS,
233 sizeof(EightBpsContext),
234 decode_init,
235 NULL,
236 decode_end,
237 decode_frame,
238 CODEC_CAP_DR1,
239 };