pcx: convert VLAs to malloc/free
[libav.git] / libavcodec / pcx.c
1 /*
2 * PC Paintbrush PCX (.pcx) image decoder
3 * Copyright (c) 2007, 2008 Ivo van Poorten
4 *
5 * This decoder does not support CGA palettes. I am unable to find samples
6 * and Netpbm cannot generate them.
7 *
8 * This file is part of FFmpeg.
9 *
10 * FFmpeg is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public
12 * License as published by the Free Software Foundation; either
13 * version 2.1 of the License, or (at your option) any later version.
14 *
15 * FFmpeg is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * Lesser General Public License for more details.
19 *
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with FFmpeg; if not, write to the Free Software
22 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 */
24
25 #include "avcodec.h"
26 #include "bytestream.h"
27 #include "get_bits.h"
28
29 typedef struct PCXContext {
30 AVFrame picture;
31 } PCXContext;
32
33 static av_cold int pcx_init(AVCodecContext *avctx) {
34 PCXContext *s = avctx->priv_data;
35
36 avcodec_get_frame_defaults(&s->picture);
37 avctx->coded_frame= &s->picture;
38
39 return 0;
40 }
41
42 /**
43 * @return advanced src pointer
44 */
45 static const uint8_t *pcx_rle_decode(const uint8_t *src, uint8_t *dst,
46 unsigned int bytes_per_scanline, int compressed) {
47 unsigned int i = 0;
48 unsigned char run, value;
49
50 if (compressed) {
51 while (i<bytes_per_scanline) {
52 run = 1;
53 value = *src++;
54 if (value >= 0xc0) {
55 run = value & 0x3f;
56 value = *src++;
57 }
58 while (i<bytes_per_scanline && run--)
59 dst[i++] = value;
60 }
61 } else {
62 memcpy(dst, src, bytes_per_scanline);
63 src += bytes_per_scanline;
64 }
65
66 return src;
67 }
68
69 static void pcx_palette(const uint8_t **src, uint32_t *dst, unsigned int pallen) {
70 unsigned int i;
71
72 for (i=0; i<pallen; i++)
73 *dst++ = bytestream_get_be24(src);
74 if (pallen < 256)
75 memset(dst, 0, (256 - pallen) * sizeof(*dst));
76 }
77
78 static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
79 AVPacket *avpkt) {
80 const uint8_t *buf = avpkt->data;
81 int buf_size = avpkt->size;
82 PCXContext * const s = avctx->priv_data;
83 AVFrame *picture = data;
84 AVFrame * const p = &s->picture;
85 int compressed, xmin, ymin, xmax, ymax;
86 unsigned int w, h, bits_per_pixel, bytes_per_line, nplanes, stride, y, x,
87 bytes_per_scanline;
88 uint8_t *ptr;
89 uint8_t const *bufstart = buf;
90 uint8_t *scanline;
91 int ret = -1;
92
93 if (buf[0] != 0x0a || buf[1] > 5) {
94 av_log(avctx, AV_LOG_ERROR, "this is not PCX encoded data\n");
95 return -1;
96 }
97
98 compressed = buf[2];
99 xmin = AV_RL16(buf+ 4);
100 ymin = AV_RL16(buf+ 6);
101 xmax = AV_RL16(buf+ 8);
102 ymax = AV_RL16(buf+10);
103
104 if (xmax < xmin || ymax < ymin) {
105 av_log(avctx, AV_LOG_ERROR, "invalid image dimensions\n");
106 return -1;
107 }
108
109 w = xmax - xmin + 1;
110 h = ymax - ymin + 1;
111
112 bits_per_pixel = buf[3];
113 bytes_per_line = AV_RL16(buf+66);
114 nplanes = buf[65];
115 bytes_per_scanline = nplanes * bytes_per_line;
116
117 if (bytes_per_scanline < w * bits_per_pixel * nplanes / 8) {
118 av_log(avctx, AV_LOG_ERROR, "PCX data is corrupted\n");
119 return -1;
120 }
121
122 switch ((nplanes<<8) + bits_per_pixel) {
123 case 0x0308:
124 avctx->pix_fmt = PIX_FMT_RGB24;
125 break;
126 case 0x0108:
127 case 0x0104:
128 case 0x0102:
129 case 0x0101:
130 case 0x0401:
131 case 0x0301:
132 case 0x0201:
133 avctx->pix_fmt = PIX_FMT_PAL8;
134 break;
135 default:
136 av_log(avctx, AV_LOG_ERROR, "invalid PCX file\n");
137 return -1;
138 }
139
140 buf += 128;
141
142 if (p->data[0])
143 avctx->release_buffer(avctx, p);
144
145 if (avcodec_check_dimensions(avctx, w, h))
146 return -1;
147 if (w != avctx->width || h != avctx->height)
148 avcodec_set_dimensions(avctx, w, h);
149 if (avctx->get_buffer(avctx, p) < 0) {
150 av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
151 return -1;
152 }
153
154 p->pict_type = FF_I_TYPE;
155
156 ptr = p->data[0];
157 stride = p->linesize[0];
158
159 scanline = av_malloc(bytes_per_scanline);
160 if (!scanline)
161 return AVERROR(ENOMEM);
162
163 if (nplanes == 3 && bits_per_pixel == 8) {
164 for (y=0; y<h; y++) {
165 buf = pcx_rle_decode(buf, scanline, bytes_per_scanline, compressed);
166
167 for (x=0; x<w; x++) {
168 ptr[3*x ] = scanline[x ];
169 ptr[3*x+1] = scanline[x+ bytes_per_line ];
170 ptr[3*x+2] = scanline[x+(bytes_per_line<<1)];
171 }
172
173 ptr += stride;
174 }
175
176 } else if (nplanes == 1 && bits_per_pixel == 8) {
177 const uint8_t *palstart = bufstart + buf_size - 769;
178
179 for (y=0; y<h; y++, ptr+=stride) {
180 buf = pcx_rle_decode(buf, scanline, bytes_per_scanline, compressed);
181 memcpy(ptr, scanline, w);
182 }
183
184 if (buf != palstart) {
185 av_log(avctx, AV_LOG_WARNING, "image data possibly corrupted\n");
186 buf = palstart;
187 }
188 if (*buf++ != 12) {
189 av_log(avctx, AV_LOG_ERROR, "expected palette after image data\n");
190 goto end;
191 }
192
193 } else if (nplanes == 1) { /* all packed formats, max. 16 colors */
194 GetBitContext s;
195
196 for (y=0; y<h; y++) {
197 init_get_bits(&s, scanline, bytes_per_scanline<<3);
198
199 buf = pcx_rle_decode(buf, scanline, bytes_per_scanline, compressed);
200
201 for (x=0; x<w; x++)
202 ptr[x] = get_bits(&s, bits_per_pixel);
203 ptr += stride;
204 }
205
206 } else { /* planar, 4, 8 or 16 colors */
207 int i;
208
209 for (y=0; y<h; y++) {
210 buf = pcx_rle_decode(buf, scanline, bytes_per_scanline, compressed);
211
212 for (x=0; x<w; x++) {
213 int m = 0x80 >> (x&7), v = 0;
214 for (i=nplanes - 1; i>=0; i--) {
215 v <<= 1;
216 v += !!(scanline[i*bytes_per_line + (x>>3)] & m);
217 }
218 ptr[x] = v;
219 }
220 ptr += stride;
221 }
222 }
223
224 if (nplanes == 1 && bits_per_pixel == 8) {
225 pcx_palette(&buf, (uint32_t *) p->data[1], 256);
226 } else if (bits_per_pixel < 8) {
227 const uint8_t *palette = bufstart+16;
228 pcx_palette(&palette, (uint32_t *) p->data[1], 16);
229 }
230
231 *picture = s->picture;
232 *data_size = sizeof(AVFrame);
233
234 ret = buf - bufstart;
235 end:
236 av_free(scanline);
237 return ret;
238 }
239
240 static av_cold int pcx_end(AVCodecContext *avctx) {
241 PCXContext *s = avctx->priv_data;
242
243 if(s->picture.data[0])
244 avctx->release_buffer(avctx, &s->picture);
245
246 return 0;
247 }
248
249 AVCodec pcx_decoder = {
250 "pcx",
251 AVMEDIA_TYPE_VIDEO,
252 CODEC_ID_PCX,
253 sizeof(PCXContext),
254 pcx_init,
255 NULL,
256 pcx_end,
257 pcx_decode_frame,
258 CODEC_CAP_DR1,
259 NULL,
260 .long_name = NULL_IF_CONFIG_SMALL("PC Paintbrush PCX image"),
261 };