pcx: convert VLAs to malloc/free
[libav.git] / libavcodec / pcx.c
CommitLineData
b4abe1d1
IP
1/*
2 * PC Paintbrush PCX (.pcx) image decoder
3 * Copyright (c) 2007, 2008 Ivo van Poorten
4 *
5 * This decoder does not support CGA palettes. I am unable to find samples
6 * and Netpbm cannot generate them.
7 *
8 * This file is part of FFmpeg.
9 *
10 * FFmpeg is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public
12 * License as published by the Free Software Foundation; either
13 * version 2.1 of the License, or (at your option) any later version.
14 *
15 * FFmpeg is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * Lesser General Public License for more details.
19 *
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with FFmpeg; if not, write to the Free Software
22 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 */
24
25#include "avcodec.h"
26#include "bytestream.h"
9106a698 27#include "get_bits.h"
b4abe1d1
IP
28
29typedef struct PCXContext {
30 AVFrame picture;
31} PCXContext;
32
98a6fff9 33static av_cold int pcx_init(AVCodecContext *avctx) {
b4abe1d1
IP
34 PCXContext *s = avctx->priv_data;
35
36 avcodec_get_frame_defaults(&s->picture);
37 avctx->coded_frame= &s->picture;
38
39 return 0;
40}
41
42/**
43 * @return advanced src pointer
44 */
3f8adf26 45static const uint8_t *pcx_rle_decode(const uint8_t *src, uint8_t *dst,
dae0d1e2 46 unsigned int bytes_per_scanline, int compressed) {
b4abe1d1
IP
47 unsigned int i = 0;
48 unsigned char run, value;
49
dae0d1e2
PR
50 if (compressed) {
51 while (i<bytes_per_scanline) {
52 run = 1;
b4abe1d1 53 value = *src++;
dae0d1e2
PR
54 if (value >= 0xc0) {
55 run = value & 0x3f;
56 value = *src++;
57 }
58 while (i<bytes_per_scanline && run--)
59 dst[i++] = value;
b4abe1d1 60 }
dae0d1e2
PR
61 } else {
62 memcpy(dst, src, bytes_per_scanline);
63 src += bytes_per_scanline;
b4abe1d1
IP
64 }
65
66 return src;
67}
68
3c71bc55 69static void pcx_palette(const uint8_t **src, uint32_t *dst, unsigned int pallen) {
b4abe1d1
IP
70 unsigned int i;
71
72 for (i=0; i<pallen; i++)
73 *dst++ = bytestream_get_be24(src);
25b6d0bc 74 if (pallen < 256)
ff54bb97 75 memset(dst, 0, (256 - pallen) * sizeof(*dst));
b4abe1d1
IP
76}
77
78static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
7a00bbad
TB
79 AVPacket *avpkt) {
80 const uint8_t *buf = avpkt->data;
81 int buf_size = avpkt->size;
b4abe1d1
IP
82 PCXContext * const s = avctx->priv_data;
83 AVFrame *picture = data;
91be76a6 84 AVFrame * const p = &s->picture;
dae0d1e2 85 int compressed, xmin, ymin, xmax, ymax;
b4abe1d1
IP
86 unsigned int w, h, bits_per_pixel, bytes_per_line, nplanes, stride, y, x,
87 bytes_per_scanline;
3c71bc55
MN
88 uint8_t *ptr;
89 uint8_t const *bufstart = buf;
70f2314d
MR
90 uint8_t *scanline;
91 int ret = -1;
b4abe1d1 92
dae0d1e2 93 if (buf[0] != 0x0a || buf[1] > 5) {
b4abe1d1
IP
94 av_log(avctx, AV_LOG_ERROR, "this is not PCX encoded data\n");
95 return -1;
96 }
97
dae0d1e2 98 compressed = buf[2];
b4abe1d1
IP
99 xmin = AV_RL16(buf+ 4);
100 ymin = AV_RL16(buf+ 6);
101 xmax = AV_RL16(buf+ 8);
102 ymax = AV_RL16(buf+10);
103
104 if (xmax < xmin || ymax < ymin) {
105 av_log(avctx, AV_LOG_ERROR, "invalid image dimensions\n");
106 return -1;
107 }
108
109 w = xmax - xmin + 1;
110 h = ymax - ymin + 1;
111
112 bits_per_pixel = buf[3];
113 bytes_per_line = AV_RL16(buf+66);
114 nplanes = buf[65];
115 bytes_per_scanline = nplanes * bytes_per_line;
116
117 if (bytes_per_scanline < w * bits_per_pixel * nplanes / 8) {
118 av_log(avctx, AV_LOG_ERROR, "PCX data is corrupted\n");
119 return -1;
120 }
121
122 switch ((nplanes<<8) + bits_per_pixel) {
123 case 0x0308:
124 avctx->pix_fmt = PIX_FMT_RGB24;
125 break;
126 case 0x0108:
127 case 0x0104:
128 case 0x0102:
129 case 0x0101:
130 case 0x0401:
131 case 0x0301:
132 case 0x0201:
133 avctx->pix_fmt = PIX_FMT_PAL8;
134 break;
135 default:
136 av_log(avctx, AV_LOG_ERROR, "invalid PCX file\n");
137 return -1;
138 }
139
140 buf += 128;
141
142 if (p->data[0])
143 avctx->release_buffer(avctx, p);
144
145 if (avcodec_check_dimensions(avctx, w, h))
146 return -1;
147 if (w != avctx->width || h != avctx->height)
148 avcodec_set_dimensions(avctx, w, h);
149 if (avctx->get_buffer(avctx, p) < 0) {
150 av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
151 return -1;
152 }
153
154 p->pict_type = FF_I_TYPE;
155
156 ptr = p->data[0];
157 stride = p->linesize[0];
158
70f2314d
MR
159 scanline = av_malloc(bytes_per_scanline);
160 if (!scanline)
161 return AVERROR(ENOMEM);
b4abe1d1 162
70f2314d 163 if (nplanes == 3 && bits_per_pixel == 8) {
b4abe1d1 164 for (y=0; y<h; y++) {
dae0d1e2 165 buf = pcx_rle_decode(buf, scanline, bytes_per_scanline, compressed);
b4abe1d1
IP
166
167 for (x=0; x<w; x++) {
168 ptr[3*x ] = scanline[x ];
169 ptr[3*x+1] = scanline[x+ bytes_per_line ];
170 ptr[3*x+2] = scanline[x+(bytes_per_line<<1)];
171 }
172
173 ptr += stride;
174 }
175
176 } else if (nplanes == 1 && bits_per_pixel == 8) {
3c71bc55 177 const uint8_t *palstart = bufstart + buf_size - 769;
b4abe1d1
IP
178
179 for (y=0; y<h; y++, ptr+=stride) {
dae0d1e2 180 buf = pcx_rle_decode(buf, scanline, bytes_per_scanline, compressed);
b4abe1d1
IP
181 memcpy(ptr, scanline, w);
182 }
183
184 if (buf != palstart) {
185 av_log(avctx, AV_LOG_WARNING, "image data possibly corrupted\n");
186 buf = palstart;
187 }
188 if (*buf++ != 12) {
189 av_log(avctx, AV_LOG_ERROR, "expected palette after image data\n");
70f2314d 190 goto end;
b4abe1d1
IP
191 }
192
193 } else if (nplanes == 1) { /* all packed formats, max. 16 colors */
b4abe1d1
IP
194 GetBitContext s;
195
196 for (y=0; y<h; y++) {
197 init_get_bits(&s, scanline, bytes_per_scanline<<3);
198
dae0d1e2 199 buf = pcx_rle_decode(buf, scanline, bytes_per_scanline, compressed);
b4abe1d1
IP
200
201 for (x=0; x<w; x++)
202 ptr[x] = get_bits(&s, bits_per_pixel);
203 ptr += stride;
204 }
205
206 } else { /* planar, 4, 8 or 16 colors */
b4abe1d1
IP
207 int i;
208
209 for (y=0; y<h; y++) {
dae0d1e2 210 buf = pcx_rle_decode(buf, scanline, bytes_per_scanline, compressed);
b4abe1d1
IP
211
212 for (x=0; x<w; x++) {
213 int m = 0x80 >> (x&7), v = 0;
214 for (i=nplanes - 1; i>=0; i--) {
215 v <<= 1;
216 v += !!(scanline[i*bytes_per_line + (x>>3)] & m);
217 }
218 ptr[x] = v;
219 }
220 ptr += stride;
221 }
222 }
223
224 if (nplanes == 1 && bits_per_pixel == 8) {
225 pcx_palette(&buf, (uint32_t *) p->data[1], 256);
226 } else if (bits_per_pixel < 8) {
3c71bc55 227 const uint8_t *palette = bufstart+16;
b4abe1d1
IP
228 pcx_palette(&palette, (uint32_t *) p->data[1], 16);
229 }
230
231 *picture = s->picture;
232 *data_size = sizeof(AVFrame);
233
70f2314d
MR
234 ret = buf - bufstart;
235end:
236 av_free(scanline);
237 return ret;
b4abe1d1
IP
238}
239
98a6fff9 240static av_cold int pcx_end(AVCodecContext *avctx) {
b4abe1d1
IP
241 PCXContext *s = avctx->priv_data;
242
243 if(s->picture.data[0])
244 avctx->release_buffer(avctx, &s->picture);
245
246 return 0;
247}
248
249AVCodec pcx_decoder = {
250 "pcx",
72415b2a 251 AVMEDIA_TYPE_VIDEO,
b4abe1d1
IP
252 CODEC_ID_PCX,
253 sizeof(PCXContext),
254 pcx_init,
255 NULL,
256 pcx_end,
257 pcx_decode_frame,
f1cdd863 258 CODEC_CAP_DR1,
bcdb2378 259 NULL,
fe4bf374 260 .long_name = NULL_IF_CONFIG_SMALL("PC Paintbrush PCX image"),
b4abe1d1 261};