mmvideo: Make sure the rle does not write over the frame boundaries
[libav.git] / libavcodec / mmvideo.c
1 /*
2 * American Laser Games MM Video Decoder
3 * Copyright (c) 2006,2008 Peter Ross
4 *
5 * This file is part of Libav.
6 *
7 * Libav is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * Libav is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with Libav; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22 /**
23 * @file
24 * American Laser Games MM Video Decoder
25 * by Peter Ross (pross@xvid.org)
26 *
27 * The MM format was used by IBM-PC ports of ALG's "arcade shooter" games,
28 * including Mad Dog McCree and Crime Patrol.
29 *
30 * Technical details here:
31 * http://wiki.multimedia.cx/index.php?title=American_Laser_Games_MM
32 */
33
34 #include "libavutil/intreadwrite.h"
35 #include "avcodec.h"
36 #include "bytestream.h"
37 #include "internal.h"
38
39 #define MM_PREAMBLE_SIZE 6
40
41 #define MM_TYPE_INTER 0x5
42 #define MM_TYPE_INTRA 0x8
43 #define MM_TYPE_INTRA_HH 0xc
44 #define MM_TYPE_INTER_HH 0xd
45 #define MM_TYPE_INTRA_HHV 0xe
46 #define MM_TYPE_INTER_HHV 0xf
47 #define MM_TYPE_PALETTE 0x31
48
49 typedef struct MmContext {
50 AVCodecContext *avctx;
51 AVFrame *frame;
52 int palette[AVPALETTE_COUNT];
53 GetByteContext gb;
54 } MmContext;
55
56 static av_cold int mm_decode_init(AVCodecContext *avctx)
57 {
58 MmContext *s = avctx->priv_data;
59
60 s->avctx = avctx;
61
62 avctx->pix_fmt = AV_PIX_FMT_PAL8;
63
64 if (!avctx->width || !avctx->height ||
65 (avctx->width & 1) || (avctx->height & 1)) {
66 av_log(avctx, AV_LOG_ERROR, "Invalid video dimensions: %dx%d\n",
67 avctx->width, avctx->height);
68 return AVERROR(EINVAL);
69 }
70
71 s->frame = av_frame_alloc();
72 if (!s->frame)
73 return AVERROR(ENOMEM);
74
75 return 0;
76 }
77
78 static int mm_decode_pal(MmContext *s)
79 {
80 int i;
81
82 bytestream2_skip(&s->gb, 4);
83 for (i = 0; i < 128; i++) {
84 s->palette[i] = bytestream2_get_be24(&s->gb);
85 s->palette[i+128] = s->palette[i]<<2;
86 }
87
88 return 0;
89 }
90
91 /**
92 * @param half_horiz Half horizontal resolution (0 or 1)
93 * @param half_vert Half vertical resolution (0 or 1)
94 */
95 static int mm_decode_intra(MmContext * s, int half_horiz, int half_vert)
96 {
97 int x = 0, y = 0;
98
99 while (bytestream2_get_bytes_left(&s->gb) > 0) {
100 int run_length, color;
101
102 // writes one more line when half_vert is true
103 if (y >= s->avctx->height + !!half_vert)
104 return 0;
105
106 color = bytestream2_get_byte(&s->gb);
107 if (color & 0x80) {
108 run_length = 1;
109 }else{
110 run_length = (color & 0x7f) + 2;
111 color = bytestream2_get_byte(&s->gb);
112 }
113
114 if (half_horiz)
115 run_length *=2;
116
117 if (s->avctx->width - x < run_length)
118 return AVERROR_INVALIDDATA;
119
120 if (color) {
121 memset(s->frame->data[0] + y*s->frame->linesize[0] + x, color, run_length);
122 if (half_vert)
123 memset(s->frame->data[0] + (y+1)*s->frame->linesize[0] + x, color, run_length);
124 }
125 x+= run_length;
126
127 if (x >= s->avctx->width) {
128 x=0;
129 y += 1 + half_vert;
130 }
131 }
132
133 return 0;
134 }
135
136 /*
137 * @param half_horiz Half horizontal resolution (0 or 1)
138 * @param half_vert Half vertical resolution (0 or 1)
139 */
140 static int mm_decode_inter(MmContext * s, int half_horiz, int half_vert)
141 {
142 int data_off = bytestream2_get_le16(&s->gb);
143 int y = 0;
144 GetByteContext data_ptr;
145
146 if (bytestream2_get_bytes_left(&s->gb) < data_off)
147 return AVERROR_INVALIDDATA;
148
149 bytestream2_init(&data_ptr, s->gb.buffer + data_off, bytestream2_get_bytes_left(&s->gb) - data_off);
150 while (s->gb.buffer < data_ptr.buffer_start) {
151 int i, j;
152 int length = bytestream2_get_byte(&s->gb);
153 int x = bytestream2_get_byte(&s->gb) + ((length & 0x80) << 1);
154 length &= 0x7F;
155
156 if (length==0) {
157 y += x;
158 continue;
159 }
160
161 if (y + half_vert >= s->avctx->height)
162 return 0;
163
164 for(i=0; i<length; i++) {
165 int replace_array = bytestream2_get_byte(&s->gb);
166 for(j=0; j<8; j++) {
167 int replace = (replace_array >> (7-j)) & 1;
168 if (x + half_horiz >= s->avctx->width)
169 return AVERROR_INVALIDDATA;
170 if (replace) {
171 int color = bytestream2_get_byte(&data_ptr);
172 s->frame->data[0][y*s->frame->linesize[0] + x] = color;
173 if (half_horiz)
174 s->frame->data[0][y*s->frame->linesize[0] + x + 1] = color;
175 if (half_vert) {
176 s->frame->data[0][(y+1)*s->frame->linesize[0] + x] = color;
177 if (half_horiz)
178 s->frame->data[0][(y+1)*s->frame->linesize[0] + x + 1] = color;
179 }
180 }
181 x += 1 + half_horiz;
182 }
183 }
184
185 y += 1 + half_vert;
186 }
187
188 return 0;
189 }
190
191 static int mm_decode_frame(AVCodecContext *avctx,
192 void *data, int *got_frame,
193 AVPacket *avpkt)
194 {
195 const uint8_t *buf = avpkt->data;
196 int buf_size = avpkt->size;
197 MmContext *s = avctx->priv_data;
198 int type, res;
199
200 if (buf_size < MM_PREAMBLE_SIZE)
201 return AVERROR_INVALIDDATA;
202 type = AV_RL16(&buf[0]);
203 buf += MM_PREAMBLE_SIZE;
204 buf_size -= MM_PREAMBLE_SIZE;
205 bytestream2_init(&s->gb, buf, buf_size);
206
207 if ((res = ff_reget_buffer(avctx, s->frame)) < 0) {
208 av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
209 return res;
210 }
211
212 switch(type) {
213 case MM_TYPE_PALETTE : res = mm_decode_pal(s); return buf_size;
214 case MM_TYPE_INTRA : res = mm_decode_intra(s, 0, 0); break;
215 case MM_TYPE_INTRA_HH : res = mm_decode_intra(s, 1, 0); break;
216 case MM_TYPE_INTRA_HHV : res = mm_decode_intra(s, 1, 1); break;
217 case MM_TYPE_INTER : res = mm_decode_inter(s, 0, 0); break;
218 case MM_TYPE_INTER_HH : res = mm_decode_inter(s, 1, 0); break;
219 case MM_TYPE_INTER_HHV : res = mm_decode_inter(s, 1, 1); break;
220 default:
221 res = AVERROR_INVALIDDATA;
222 break;
223 }
224 if (res < 0)
225 return res;
226
227 memcpy(s->frame->data[1], s->palette, AVPALETTE_SIZE);
228
229 if ((res = av_frame_ref(data, s->frame)) < 0)
230 return res;
231
232 *got_frame = 1;
233
234 return buf_size;
235 }
236
237 static av_cold int mm_decode_end(AVCodecContext *avctx)
238 {
239 MmContext *s = avctx->priv_data;
240
241 av_frame_free(&s->frame);
242
243 return 0;
244 }
245
246 AVCodec ff_mmvideo_decoder = {
247 .name = "mmvideo",
248 .long_name = NULL_IF_CONFIG_SMALL("American Laser Games MM Video"),
249 .type = AVMEDIA_TYPE_VIDEO,
250 .id = AV_CODEC_ID_MMVIDEO,
251 .priv_data_size = sizeof(MmContext),
252 .init = mm_decode_init,
253 .close = mm_decode_end,
254 .decode = mm_decode_frame,
255 .capabilities = AV_CODEC_CAP_DR1,
256 };