projects / libav.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xwddec: prevent overflow of lsize * avctx->height
[libav.git] / libavcodec / xwddec.c
CommitLineData
27ed027b
PM		 1/*
2 * XWD image format
3 *
4 * Copyright (c) 2012 Paul B Mahol
5 *
6 * This file is part of Libav.
7 *
8 * Libav is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * Libav is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with Libav; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
cc8163e1
DB		 23#include <inttypes.h>
24
27ed027b
PM		 25#include "libavutil/imgutils.h"
26#include "avcodec.h"
27#include "bytestream.h"
594d4d5d 28#include "internal.h"
27ed027b
PM		 29#include "xwd.h"
30
27ed027b 31static int xwd_decode_frame(AVCodecContext *avctx, void *data,
df9b9567 32 int *got_frame, AVPacket *avpkt)
27ed027b 33{
759001c5 34 AVFrame *p = data;
27ed027b
PM		 35 const uint8_t *buf = avpkt->data;
36 int i, ret, buf_size = avpkt->size;
37 uint32_t version, header_size, vclass, ncolors;
38 uint32_t xoffset, be, bpp, lsize, rsize;
39 uint32_t pixformat, pixdepth, bunit, bitorder, bpad;
40 uint32_t rgb[3];
41 uint8_t *ptr;
abe3c697 42 GetByteContext gb;
27ed027b
PM		 43
44 if (buf_size < XWD_HEADER_SIZE)
45 return AVERROR_INVALIDDATA;
46
abe3c697
RB		 47 bytestream2_init(&gb, buf, buf_size);
48 header_size = bytestream2_get_be32u(&gb);
27ed027b 49
abe3c697 50 version = bytestream2_get_be32u(&gb);
27ed027b
PM		 51 if (version != XWD_VERSION) {
52 av_log(avctx, AV_LOG_ERROR, "unsupported version\n");
53 return AVERROR_INVALIDDATA;
54 }
55
abe3c697 56 if (buf_size < header_size || header_size < XWD_HEADER_SIZE) {
27ed027b
PM		 57 av_log(avctx, AV_LOG_ERROR, "invalid header size\n");
58 return AVERROR_INVALIDDATA;
59 }
60
abe3c697
RB		 61 pixformat = bytestream2_get_be32u(&gb);
62 pixdepth = bytestream2_get_be32u(&gb);
63 avctx->width = bytestream2_get_be32u(&gb);
64 avctx->height = bytestream2_get_be32u(&gb);
65 xoffset = bytestream2_get_be32u(&gb);
66 be = bytestream2_get_be32u(&gb);
67 bunit = bytestream2_get_be32u(&gb);
68 bitorder = bytestream2_get_be32u(&gb);
69 bpad = bytestream2_get_be32u(&gb);
70 bpp = bytestream2_get_be32u(&gb);
71 lsize = bytestream2_get_be32u(&gb);
72 vclass = bytestream2_get_be32u(&gb);
73 rgb[0] = bytestream2_get_be32u(&gb);
74 rgb[1] = bytestream2_get_be32u(&gb);
75 rgb[2] = bytestream2_get_be32u(&gb);
76 bytestream2_skipu(&gb, 8);
77 ncolors = bytestream2_get_be32u(&gb);
78 bytestream2_skipu(&gb, header_size - (XWD_HEADER_SIZE - 20));
27ed027b 79
cc8163e1
DB		 80 av_log(avctx, AV_LOG_DEBUG,
81 "pixformat %"PRIu32", pixdepth %"PRIu32", bunit %"PRIu32", bitorder %"PRIu32", bpad %"PRIu32"\n",
27ed027b 82 pixformat, pixdepth, bunit, bitorder, bpad);
cc8163e1
DB		 83 av_log(avctx, AV_LOG_DEBUG,
84 "vclass %"PRIu32", ncolors %"PRIu32", bpp %"PRIu32", be %"PRIu32", lsize %"PRIu32", xoffset %"PRIu32"\n",
27ed027b 85 vclass, ncolors, bpp, be, lsize, xoffset);
cba4e606
DB		 86 av_log(avctx, AV_LOG_DEBUG,
87 "red %0"PRIx32", green %0"PRIx32", blue %0"PRIx32"\n",
88 rgb[0], rgb[1], rgb[2]);
27ed027b
PM		 89
90 if (pixformat > XWD_Z_PIXMAP) {
91 av_log(avctx, AV_LOG_ERROR, "invalid pixmap format\n");
92 return AVERROR_INVALIDDATA;
93 }
94
95 if (pixdepth == 0 || pixdepth > 32) {
96 av_log(avctx, AV_LOG_ERROR, "invalid pixmap depth\n");
97 return AVERROR_INVALIDDATA;
98 }
99
100 if (xoffset) {
cc8163e1 101 avpriv_request_sample(avctx, "xoffset %"PRIu32"", xoffset);
27ed027b
PM		 102 return AVERROR_PATCHWELCOME;
103 }
104
105 if (be > 1) {
106 av_log(avctx, AV_LOG_ERROR, "invalid byte order\n");
107 return AVERROR_INVALIDDATA;
108 }
109
110 if (bitorder > 1) {
111 av_log(avctx, AV_LOG_ERROR, "invalid bitmap bit order\n");
112 return AVERROR_INVALIDDATA;
113 }
114
115 if (bunit != 8 && bunit != 16 && bunit != 32) {
116 av_log(avctx, AV_LOG_ERROR, "invalid bitmap unit\n");
117 return AVERROR_INVALIDDATA;
118 }
119
120 if (bpad != 8 && bpad != 16 && bpad != 32) {
121 av_log(avctx, AV_LOG_ERROR, "invalid bitmap scan-line pad\n");
122 return AVERROR_INVALIDDATA;
123 }
124
125 if (bpp == 0 || bpp > 32) {
126 av_log(avctx, AV_LOG_ERROR, "invalid bits per pixel\n");
127 return AVERROR_INVALIDDATA;
128 }
129
130 if (ncolors > 256) {
131 av_log(avctx, AV_LOG_ERROR, "invalid number of entries in colormap\n");
132 return AVERROR_INVALIDDATA;
133 }
134
135 if ((ret = av_image_check_size(avctx->width, avctx->height, 0, NULL)) < 0)
136 return ret;
137
138 rsize = FFALIGN(avctx->width * bpp, bpad) / 8;
139 if (lsize < rsize) {
140 av_log(avctx, AV_LOG_ERROR, "invalid bytes per scan-line\n");
141 return AVERROR_INVALIDDATA;
142 }
143
8431629d 144 if (bytestream2_get_bytes_left(&gb) < ncolors * XWD_CMAP_SIZE + (uint64_t)avctx->height * lsize) {
27ed027b
PM		 145 av_log(avctx, AV_LOG_ERROR, "input buffer too small\n");
146 return AVERROR_INVALIDDATA;
147 }
148
149 if (pixformat != XWD_Z_PIXMAP) {
cc8163e1 150 av_log(avctx, AV_LOG_ERROR, "pixmap format %"PRIu32" unsupported\n", pixformat);
27ed027b
PM		 151 return AVERROR_PATCHWELCOME;
152 }
153
716d413c 154 avctx->pix_fmt = AV_PIX_FMT_NONE;
27ed027b
PM		 155 switch (vclass) {
156 case XWD_STATIC_GRAY:
157 case XWD_GRAY_SCALE:
158 if (bpp != 1)
159 return AVERROR_INVALIDDATA;
160 if (pixdepth == 1)
716d413c 161 avctx->pix_fmt = AV_PIX_FMT_MONOWHITE;
27ed027b
PM		 162 break;
163 case XWD_STATIC_COLOR:
164 case XWD_PSEUDO_COLOR:
165 if (bpp == 8)
716d413c 166 avctx->pix_fmt = AV_PIX_FMT_PAL8;
27ed027b
PM		 167 break;
168 case XWD_TRUE_COLOR:
169 case XWD_DIRECT_COLOR:
170 if (bpp != 16 && bpp != 24 && bpp != 32)
171 return AVERROR_INVALIDDATA;
172 if (bpp == 16 && pixdepth == 15) {
173 if (rgb[0] == 0x7C00 && rgb[1] == 0x3E0 && rgb[2] == 0x1F)
716d413c 174 avctx->pix_fmt = be ? AV_PIX_FMT_RGB555BE : AV_PIX_FMT_RGB555LE;
27ed027b 175 else if (rgb[0] == 0x1F && rgb[1] == 0x3E0 && rgb[2] == 0x7C00)
716d413c 176 avctx->pix_fmt = be ? AV_PIX_FMT_BGR555BE : AV_PIX_FMT_BGR555LE;
27ed027b
PM		 177 } else if (bpp == 16 && pixdepth == 16) {
178 if (rgb[0] == 0xF800 && rgb[1] == 0x7E0 && rgb[2] == 0x1F)
716d413c 179 avctx->pix_fmt = be ? AV_PIX_FMT_RGB565BE : AV_PIX_FMT_RGB565LE;
27ed027b 180 else if (rgb[0] == 0x1F && rgb[1] == 0x7E0 && rgb[2] == 0xF800)
716d413c 181 avctx->pix_fmt = be ? AV_PIX_FMT_BGR565BE : AV_PIX_FMT_BGR565LE;
27ed027b
PM		 182 } else if (bpp == 24) {
183 if (rgb[0] == 0xFF0000 && rgb[1] == 0xFF00 && rgb[2] == 0xFF)
716d413c 184 avctx->pix_fmt = be ? AV_PIX_FMT_RGB24 : AV_PIX_FMT_BGR24;
27ed027b 185 else if (rgb[0] == 0xFF && rgb[1] == 0xFF00 && rgb[2] == 0xFF0000)
716d413c 186 avctx->pix_fmt = be ? AV_PIX_FMT_BGR24 : AV_PIX_FMT_RGB24;
27ed027b
PM		 187 } else if (bpp == 32) {
188 if (rgb[0] == 0xFF0000 && rgb[1] == 0xFF00 && rgb[2] == 0xFF)
716d413c 189 avctx->pix_fmt = be ? AV_PIX_FMT_ARGB : AV_PIX_FMT_BGRA;
27ed027b 190 else if (rgb[0] == 0xFF && rgb[1] == 0xFF00 && rgb[2] == 0xFF0000)
716d413c 191 avctx->pix_fmt = be ? AV_PIX_FMT_ABGR : AV_PIX_FMT_RGBA;
27ed027b 192 }
abe3c697 193 bytestream2_skipu(&gb, ncolors * XWD_CMAP_SIZE);
27ed027b
PM		 194 break;
195 default:
196 av_log(avctx, AV_LOG_ERROR, "invalid visual class\n");
197 return AVERROR_INVALIDDATA;
198 }
199
716d413c 200 if (avctx->pix_fmt == AV_PIX_FMT_NONE) {
6d97484d 201 avpriv_request_sample(avctx,
cc8163e1 202 "Unknown file: bpp %"PRIu32", pixdepth %"PRIu32", vclass %"PRIu32"",
6d97484d 203 bpp, pixdepth, vclass);
27ed027b
PM		 204 return AVERROR_PATCHWELCOME;
205 }
206
759001c5 207 if ((ret = ff_get_buffer(avctx, p, 0)) < 0) {
27ed027b
PM		 208 av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
209 return ret;
210 }
211
212 p->key_frame = 1;
213 p->pict_type = AV_PICTURE_TYPE_I;
214
716d413c 215 if (avctx->pix_fmt == AV_PIX_FMT_PAL8) {
27ed027b
PM		 216 uint32_t *dst = (uint32_t *)p->data[1];
217 uint8_t red, green, blue;
218
219 for (i = 0; i < ncolors; i++) {
220
abe3c697
RB		 221 bytestream2_skipu(&gb, 4); // skip colormap entry number
222 red = bytestream2_get_byteu(&gb);
223 bytestream2_skipu(&gb, 1);
224 green = bytestream2_get_byteu(&gb);
225 bytestream2_skipu(&gb, 1);
226 blue = bytestream2_get_byteu(&gb);
227 bytestream2_skipu(&gb, 3); // skip bitmask flag and padding
27ed027b
PM		 228
229 dst[i] = red << 16 | green << 8 | blue;
230 }
231 }
232
233 ptr = p->data[0];
234 for (i = 0; i < avctx->height; i++) {
abe3c697
RB		 235 bytestream2_get_bufferu(&gb, ptr, rsize);
236 bytestream2_skipu(&gb, lsize - rsize);
27ed027b
PM		 237 ptr += p->linesize[0];
238 }
239
df9b9567 240 *got_frame = 1;
27ed027b
PM		 241
242 return buf_size;
243}
244
27ed027b
PM		 245AVCodec ff_xwd_decoder = {
246 .name = "xwd",
b2bed932 247 .long_name = NULL_IF_CONFIG_SMALL("XWD (X Window Dump) image"),
27ed027b 248 .type = AVMEDIA_TYPE_VIDEO,
36ef5369 249 .id = AV_CODEC_ID_XWD,
27ed027b 250 .decode = xwd_decode_frame,
def97856 251 .capabilities = AV_CODEC_CAP_DR1,
27ed027b 252};
Libav master