various security fixes and precautionary checks
[libav.git] / libavcodec / pnm.c
CommitLineData
5b6d5596
MN
1/*
2 * PNM image format
3 * Copyright (c) 2002, 2003 Fabrice Bellard.
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19#include "avcodec.h"
99f06236 20#include "mpegvideo.h" //only for ParseContext
5b6d5596
MN
21
22typedef struct PNMContext {
23 uint8_t *bytestream;
24 uint8_t *bytestream_start;
25 uint8_t *bytestream_end;
26 AVFrame picture;
27} PNMContext;
28
29static inline int pnm_space(int c)
30{
31 return (c == ' ' || c == '\n' || c == '\r' || c == '\t');
32}
33
34static void pnm_get(PNMContext *sc, char *str, int buf_size)
35{
36 char *s;
37 int c;
38
39 /* skip spaces and comments */
40 for(;;) {
41 c = *sc->bytestream++;
42 if (c == '#') {
43 do {
44 c = *sc->bytestream++;
45 } while (c != '\n' && sc->bytestream < sc->bytestream_end);
46 } else if (!pnm_space(c)) {
47 break;
48 }
49 }
50
51 s = str;
52 while (sc->bytestream < sc->bytestream_end && !pnm_space(c)) {
53 if ((s - str) < buf_size - 1)
54 *s++ = c;
55 c = *sc->bytestream++;
56 }
57 *s = '\0';
58}
59
60static int common_init(AVCodecContext *avctx){
61 PNMContext *s = avctx->priv_data;
62
63 avcodec_get_frame_defaults((AVFrame*)&s->picture);
64 avctx->coded_frame= (AVFrame*)&s->picture;
65
66 return 0;
67}
68
99f06236 69static int pnm_decode_header(AVCodecContext *avctx, PNMContext * const s){
311c7008
MN
70 char buf1[32], tuple_type[32];
71 int h, w, depth, maxval;;
5b6d5596
MN
72
73 pnm_get(s, buf1, sizeof(buf1));
74 if (!strcmp(buf1, "P4")) {
75 avctx->pix_fmt = PIX_FMT_MONOWHITE;
76 } else if (!strcmp(buf1, "P5")) {
77 if (avctx->codec_id == CODEC_ID_PGMYUV)
78 avctx->pix_fmt = PIX_FMT_YUV420P;
79 else
80 avctx->pix_fmt = PIX_FMT_GRAY8;
81 } else if (!strcmp(buf1, "P6")) {
82 avctx->pix_fmt = PIX_FMT_RGB24;
311c7008
MN
83 } else if (!strcmp(buf1, "P7")) {
84 w = -1;
85 h = -1;
86 maxval = -1;
87 depth = -1;
88 tuple_type[0] = '\0';
89 for(;;) {
90 pnm_get(s, buf1, sizeof(buf1));
91 if (!strcmp(buf1, "WIDTH")) {
92 pnm_get(s, buf1, sizeof(buf1));
93 w = strtol(buf1, NULL, 10);
94 } else if (!strcmp(buf1, "HEIGHT")) {
95 pnm_get(s, buf1, sizeof(buf1));
96 h = strtol(buf1, NULL, 10);
97 } else if (!strcmp(buf1, "DEPTH")) {
98 pnm_get(s, buf1, sizeof(buf1));
99 depth = strtol(buf1, NULL, 10);
100 } else if (!strcmp(buf1, "MAXVAL")) {
101 pnm_get(s, buf1, sizeof(buf1));
102 maxval = strtol(buf1, NULL, 10);
103 } else if (!strcmp(buf1, "TUPLETYPE")) {
104 pnm_get(s, tuple_type, sizeof(tuple_type));
105 } else if (!strcmp(buf1, "ENDHDR")) {
106 break;
107 } else {
108 return -1;
109 }
110 }
111 /* check that all tags are present */
0ecca7a4 112 if (w <= 0 || h <= 0 || maxval <= 0 || depth <= 0 || tuple_type[0] == '\0' || avcodec_check_dimensions(avctx, w, h))
311c7008 113 return -1;
0ecca7a4 114
311c7008
MN
115 avctx->width = w;
116 avctx->height = h;
117 if (depth == 1) {
118 if (maxval == 1)
119 avctx->pix_fmt = PIX_FMT_MONOWHITE;
120 else
121 avctx->pix_fmt = PIX_FMT_GRAY8;
122 } else if (depth == 3) {
123 avctx->pix_fmt = PIX_FMT_RGB24;
124 } else if (depth == 4) {
125 avctx->pix_fmt = PIX_FMT_RGBA32;
126 } else {
127 return -1;
128 }
129 return 0;
5b6d5596
MN
130 } else {
131 return -1;
132 }
133 pnm_get(s, buf1, sizeof(buf1));
134 avctx->width = atoi(buf1);
135 if (avctx->width <= 0)
136 return -1;
137 pnm_get(s, buf1, sizeof(buf1));
138 avctx->height = atoi(buf1);
0ecca7a4 139 if(avcodec_check_dimensions(avctx, avctx->width, avctx->height))
5b6d5596
MN
140 return -1;
141 if (avctx->pix_fmt != PIX_FMT_MONOWHITE) {
142 pnm_get(s, buf1, sizeof(buf1));
143 }
144
145 /* more check if YUV420 */
146 if (avctx->pix_fmt == PIX_FMT_YUV420P) {
147 if ((avctx->width & 1) != 0)
148 return -1;
149 h = (avctx->height * 2);
150 if ((h % 3) != 0)
151 return -1;
152 h /= 3;
153 avctx->height = h;
154 }
99f06236
MN
155 return 0;
156}
157
158static int pnm_decode_frame(AVCodecContext *avctx,
159 void *data, int *data_size,
160 uint8_t *buf, int buf_size)
161{
162 PNMContext * const s = avctx->priv_data;
163 AVFrame *picture = data;
164 AVFrame * const p= (AVFrame*)&s->picture;
165 int i, n, linesize, h;
166 unsigned char *ptr;
167
168 /* special case for last picture */
169 if (buf_size == 0) {
170 return 0;
171 }
172
173 s->bytestream_start=
174 s->bytestream= buf;
175 s->bytestream_end= buf + buf_size;
176
177 if(pnm_decode_header(avctx, s) < 0)
311c7008 178 return -1;
5b6d5596
MN
179
180 if(p->data[0])
181 avctx->release_buffer(avctx, p);
182
183 p->reference= 0;
184 if(avctx->get_buffer(avctx, p) < 0){
185 av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
186 return -1;
187 }
188 p->pict_type= FF_I_TYPE;
189 p->key_frame= 1;
190
191 switch(avctx->pix_fmt) {
192 default:
193 return -1;
194 case PIX_FMT_RGB24:
195 n = avctx->width * 3;
196 goto do_read;
197 case PIX_FMT_GRAY8:
198 n = avctx->width;
199 goto do_read;
200 case PIX_FMT_MONOWHITE:
311c7008 201 case PIX_FMT_MONOBLACK:
5b6d5596
MN
202 n = (avctx->width + 7) >> 3;
203 do_read:
204 ptr = p->data[0];
205 linesize = p->linesize[0];
206 for(i = 0; i < avctx->height; i++) {
207 memcpy(ptr, s->bytestream, n);
208 s->bytestream += n;
209 ptr += linesize;
210 }
211 break;
212 case PIX_FMT_YUV420P:
213 {
214 unsigned char *ptr1, *ptr2;
215
216 n = avctx->width;
217 ptr = p->data[0];
218 linesize = p->linesize[0];
219 for(i = 0; i < avctx->height; i++) {
220 memcpy(ptr, s->bytestream, n);
221 s->bytestream += n;
222 ptr += linesize;
223 }
224 ptr1 = p->data[1];
225 ptr2 = p->data[2];
226 n >>= 1;
227 h = avctx->height >> 1;
228 for(i = 0; i < h; i++) {
229 memcpy(ptr1, s->bytestream, n);
230 s->bytestream += n;
231 memcpy(ptr2, s->bytestream, n);
232 s->bytestream += n;
233 ptr1 += p->linesize[1];
234 ptr2 += p->linesize[2];
235 }
236 }
237 break;
311c7008
MN
238 case PIX_FMT_RGBA32:
239 ptr = p->data[0];
240 linesize = p->linesize[0];
241 for(i = 0; i < avctx->height; i++) {
242 int j, r, g, b, a;
243
244 for(j = 0;j < avctx->width; j++) {
245 r = *s->bytestream++;
246 g = *s->bytestream++;
247 b = *s->bytestream++;
248 a = *s->bytestream++;
249 ((uint32_t *)ptr)[j] = (a << 24) | (r << 16) | (g << 8) | b;
250 }
251 ptr += linesize;
252 }
253 break;
5b6d5596
MN
254 }
255 *picture= *(AVFrame*)&s->picture;
256 *data_size = sizeof(AVPicture);
257
258 return s->bytestream - s->bytestream_start;
259}
260
261static int pnm_encode_frame(AVCodecContext *avctx, unsigned char *outbuf, int buf_size, void *data){
262 PNMContext *s = avctx->priv_data;
263 AVFrame *pict = data;
264 AVFrame * const p= (AVFrame*)&s->picture;
265 int i, h, h1, c, n, linesize;
266 uint8_t *ptr, *ptr1, *ptr2;
267
0ecca7a4
MN
268 if(buf_size < avpicture_get_size(avctx->pix_fmt, avctx->width, avctx->height) + 200){
269 av_log(avctx, AV_LOG_ERROR, "encoded frame too large\n");
270 return -1;
271 }
272
5b6d5596
MN
273 *p = *pict;
274 p->pict_type= FF_I_TYPE;
275 p->key_frame= 1;
276
277 s->bytestream_start=
278 s->bytestream= outbuf;
279 s->bytestream_end= outbuf+buf_size;
280
281 h = avctx->height;
282 h1 = h;
283 switch(avctx->pix_fmt) {
284 case PIX_FMT_MONOWHITE:
285 c = '4';
286 n = (avctx->width + 7) >> 3;
287 break;
288 case PIX_FMT_GRAY8:
289 c = '5';
290 n = avctx->width;
291 break;
292 case PIX_FMT_RGB24:
293 c = '6';
294 n = avctx->width * 3;
295 break;
296 case PIX_FMT_YUV420P:
297 c = '5';
298 n = avctx->width;
299 h1 = (h * 3) / 2;
300 break;
301 default:
302 return -1;
303 }
304 snprintf(s->bytestream, s->bytestream_end - s->bytestream,
305 "P%c\n%d %d\n",
306 c, avctx->width, h1);
307 s->bytestream += strlen(s->bytestream);
308 if (avctx->pix_fmt != PIX_FMT_MONOWHITE) {
309 snprintf(s->bytestream, s->bytestream_end - s->bytestream,
310 "%d\n", 255);
311 s->bytestream += strlen(s->bytestream);
312 }
313
314 ptr = p->data[0];
315 linesize = p->linesize[0];
316 for(i=0;i<h;i++) {
317 memcpy(s->bytestream, ptr, n);
318 s->bytestream += n;
319 ptr += linesize;
320 }
321
322 if (avctx->pix_fmt == PIX_FMT_YUV420P) {
323 h >>= 1;
324 n >>= 1;
325 ptr1 = p->data[1];
326 ptr2 = p->data[2];
327 for(i=0;i<h;i++) {
328 memcpy(s->bytestream, ptr1, n);
329 s->bytestream += n;
330 memcpy(s->bytestream, ptr2, n);
331 s->bytestream += n;
332 ptr1 += p->linesize[1];
333 ptr2 += p->linesize[2];
334 }
335 }
336 return s->bytestream - s->bytestream_start;
337}
338
5b6d5596
MN
339static int pam_encode_frame(AVCodecContext *avctx, unsigned char *outbuf, int buf_size, void *data){
340 PNMContext *s = avctx->priv_data;
341 AVFrame *pict = data;
342 AVFrame * const p= (AVFrame*)&s->picture;
343 int i, h, w, n, linesize, depth, maxval;
344 const char *tuple_type;
345 uint8_t *ptr;
346
0ecca7a4
MN
347 if(buf_size < avpicture_get_size(avctx->pix_fmt, avctx->width, avctx->height) + 200){
348 av_log(avctx, AV_LOG_ERROR, "encoded frame too large\n");
349 return -1;
350 }
351
5b6d5596
MN
352 *p = *pict;
353 p->pict_type= FF_I_TYPE;
354 p->key_frame= 1;
355
356 s->bytestream_start=
357 s->bytestream= outbuf;
358 s->bytestream_end= outbuf+buf_size;
359
360 h = avctx->height;
361 w = avctx->width;
362 switch(avctx->pix_fmt) {
363 case PIX_FMT_MONOWHITE:
364 n = (w + 7) >> 3;
365 depth = 1;
366 maxval = 1;
367 tuple_type = "BLACKANDWHITE";
368 break;
369 case PIX_FMT_GRAY8:
370 n = w;
371 depth = 1;
372 maxval = 255;
373 tuple_type = "GRAYSCALE";
374 break;
375 case PIX_FMT_RGB24:
376 n = w * 3;
377 depth = 3;
378 maxval = 255;
379 tuple_type = "RGB";
380 break;
381 case PIX_FMT_RGBA32:
382 n = w * 4;
383 depth = 4;
384 maxval = 255;
385 tuple_type = "RGB_ALPHA";
386 break;
387 default:
388 return -1;
389 }
390 snprintf(s->bytestream, s->bytestream_end - s->bytestream,
391 "P7\nWIDTH %d\nHEIGHT %d\nDEPTH %d\nMAXVAL %d\nTUPLETYPE %s\nENDHDR\n",
392 w, h, depth, maxval, tuple_type);
393 s->bytestream += strlen(s->bytestream);
394
395 ptr = p->data[0];
396 linesize = p->linesize[0];
397
398 if (avctx->pix_fmt == PIX_FMT_RGBA32) {
399 int j;
400 unsigned int v;
401
402 for(i=0;i<h;i++) {
403 for(j=0;j<w;j++) {
404 v = ((uint32_t *)ptr)[j];
405 *s->bytestream++ = v >> 16;
406 *s->bytestream++ = v >> 8;
407 *s->bytestream++ = v;
408 *s->bytestream++ = v >> 24;
409 }
410 ptr += linesize;
411 }
412 } else {
413 for(i=0;i<h;i++) {
414 memcpy(s->bytestream, ptr, n);
415 s->bytestream += n;
416 ptr += linesize;
417 }
418 }
419 return s->bytestream - s->bytestream_start;
420}
421
422#if 0
423static int pnm_probe(AVProbeData *pd)
424{
425 const char *p = pd->buf;
426 if (pd->buf_size >= 8 &&
427 p[0] == 'P' &&
428 p[1] >= '4' && p[1] <= '6' &&
429 pnm_space(p[2]) )
430 return AVPROBE_SCORE_MAX - 1; /* to permit pgmyuv probe */
431 else
432 return 0;
433}
434
435static int pgmyuv_probe(AVProbeData *pd)
436{
437 if (match_ext(pd->filename, "pgmyuv"))
438 return AVPROBE_SCORE_MAX;
439 else
440 return 0;
441}
442
443static int pam_probe(AVProbeData *pd)
444{
445 const char *p = pd->buf;
446 if (pd->buf_size >= 8 &&
447 p[0] == 'P' &&
448 p[1] == '7' &&
449 p[2] == '\n')
450 return AVPROBE_SCORE_MAX;
451 else
452 return 0;
453}
454#endif
455
99f06236
MN
456static int pnm_parse(AVCodecParserContext *s,
457 AVCodecContext *avctx,
458 uint8_t **poutbuf, int *poutbuf_size,
459 const uint8_t *buf, int buf_size)
460{
461 ParseContext *pc = s->priv_data;
462 PNMContext pnmctx;
463 int next;
464
465 for(; pc->overread>0; pc->overread--){
466 pc->buffer[pc->index++]= pc->buffer[pc->overread_index++];
467 }
468retry:
469 if(pc->index){
470 pnmctx.bytestream_start=
471 pnmctx.bytestream= pc->buffer;
472 pnmctx.bytestream_end= pc->buffer + pc->index;
473 }else{
474 pnmctx.bytestream_start=
475 pnmctx.bytestream= buf;
476 pnmctx.bytestream_end= buf + buf_size;
477 }
478 if(pnm_decode_header(avctx, &pnmctx) < 0){
479 if(pnmctx.bytestream < pnmctx.bytestream_end){
480 if(pc->index){
481 pc->index=0;
482 }else{
483 buf++;
484 buf_size--;
485 }
486 goto retry;
487 }
488#if 0
489 if(pc->index && pc->index*2 + FF_INPUT_BUFFER_PADDING_SIZE < pc->buffer_size && buf_size > pc->index){
490 memcpy(pc->buffer + pc->index, buf, pc->index);
491 pc->index += pc->index;
492 buf += pc->index;
493 buf_size -= pc->index;
494 goto retry;
495 }
496#endif
497 next= END_NOT_FOUND;
498 }else{
499 next= pnmctx.bytestream - pnmctx.bytestream_start
500 + avpicture_get_size(avctx->pix_fmt, avctx->width, avctx->height);
501 if(pnmctx.bytestream_start!=buf)
502 next-= pc->index;
503 if(next > buf_size)
504 next= END_NOT_FOUND;
505 }
506
507 if(ff_combine_frame(pc, next, (uint8_t **)&buf, &buf_size)<0){
508 *poutbuf = NULL;
509 *poutbuf_size = 0;
510 return buf_size;
511 }
512 *poutbuf = (uint8_t *)buf;
513 *poutbuf_size = buf_size;
514 return next;
515}
516
517AVCodecParser pnm_parser = {
311c7008 518 { CODEC_ID_PGM, CODEC_ID_PGMYUV, CODEC_ID_PPM, CODEC_ID_PBM, CODEC_ID_PAM},
99f06236
MN
519 sizeof(ParseContext),
520 NULL,
521 pnm_parse,
522 ff_parse_close,
523};
524
5b6d5596
MN
525AVCodec pgm_encoder = {
526 "pgm",
527 CODEC_TYPE_VIDEO,
528 CODEC_ID_PGM,
529 sizeof(PNMContext),
530 common_init,
531 pnm_encode_frame,
532 NULL, //encode_end,
533 pnm_decode_frame,
534 .pix_fmts= (enum PixelFormat[]){PIX_FMT_GRAY8, -1},
535};
536
537AVCodec pgmyuv_encoder = {
538 "pgmyuv",
539 CODEC_TYPE_VIDEO,
540 CODEC_ID_PGMYUV,
541 sizeof(PNMContext),
542 common_init,
543 pnm_encode_frame,
544 NULL, //encode_end,
545 pnm_decode_frame,
546 .pix_fmts= (enum PixelFormat[]){PIX_FMT_YUV420P, -1},
547};
548
549AVCodec ppm_encoder = {
550 "ppm",
551 CODEC_TYPE_VIDEO,
552 CODEC_ID_PPM,
553 sizeof(PNMContext),
554 common_init,
555 pnm_encode_frame,
556 NULL, //encode_end,
557 pnm_decode_frame,
558 .pix_fmts= (enum PixelFormat[]){PIX_FMT_RGB24, -1},
559};
560
561AVCodec pbm_encoder = {
562 "pbm",
563 CODEC_TYPE_VIDEO,
564 CODEC_ID_PBM,
565 sizeof(PNMContext),
566 common_init,
567 pnm_encode_frame,
568 NULL, //encode_end,
569 pnm_decode_frame,
570 .pix_fmts= (enum PixelFormat[]){PIX_FMT_MONOWHITE, -1},
571};
572
573AVCodec pam_encoder = {
574 "pam",
575 CODEC_TYPE_VIDEO,
576 CODEC_ID_PAM,
577 sizeof(PNMContext),
578 common_init,
579 pam_encode_frame,
580 NULL, //encode_end,
311c7008 581 pnm_decode_frame,
5b6d5596
MN
582 .pix_fmts= (enum PixelFormat[]){PIX_FMT_RGB24, PIX_FMT_RGBA32, PIX_FMT_GRAY8, PIX_FMT_MONOWHITE, -1},
583};