mmvideo: Make sure the rle does not write over the frame boundaries
[libav.git] / libavcodec / mmvideo.c
CommitLineData
4e114829
MM
1/*
2 * American Laser Games MM Video Decoder
60f451b6 3 * Copyright (c) 2006,2008 Peter Ross
4e114829 4 *
2912e87a 5 * This file is part of Libav.
b78e7197 6 *
2912e87a 7 * Libav is free software; you can redistribute it and/or
4e114829
MM
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
b78e7197 10 * version 2.1 of the License, or (at your option) any later version.
4e114829 11 *
2912e87a 12 * Libav is distributed in the hope that it will be useful,
4e114829
MM
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
2912e87a 18 * License along with Libav; if not, write to the Free Software
e5a389a1 19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
4e114829
MM
20 */
21
22/**
ba87f080 23 * @file
4e114829 24 * American Laser Games MM Video Decoder
8600106a 25 * by Peter Ross (pross@xvid.org)
4e114829
MM
26 *
27 * The MM format was used by IBM-PC ports of ALG's "arcade shooter" games,
28 * including Mad Dog McCree and Crime Patrol.
29 *
30 * Technical details here:
31 * http://wiki.multimedia.cx/index.php?title=American_Laser_Games_MM
32 */
33
6a5d31ac 34#include "libavutil/intreadwrite.h"
4e114829 35#include "avcodec.h"
a55d5bdc 36#include "bytestream.h"
759001c5 37#include "internal.h"
4e114829
MM
38
39#define MM_PREAMBLE_SIZE 6
40
41#define MM_TYPE_INTER 0x5
42#define MM_TYPE_INTRA 0x8
43#define MM_TYPE_INTRA_HH 0xc
44#define MM_TYPE_INTER_HH 0xd
45#define MM_TYPE_INTRA_HHV 0xe
46#define MM_TYPE_INTER_HHV 0xf
60f451b6 47#define MM_TYPE_PALETTE 0x31
4e114829
MM
48
49typedef struct MmContext {
50 AVCodecContext *avctx;
5c96f029 51 AVFrame *frame;
60f451b6 52 int palette[AVPALETTE_COUNT];
a55d5bdc 53 GetByteContext gb;
4e114829
MM
54} MmContext;
55
98a6fff9 56static av_cold int mm_decode_init(AVCodecContext *avctx)
4e114829
MM
57{
58 MmContext *s = avctx->priv_data;
59
60 s->avctx = avctx;
61
716d413c 62 avctx->pix_fmt = AV_PIX_FMT_PAL8;
4e114829 63
17ba719d
AK
64 if (!avctx->width || !avctx->height ||
65 (avctx->width & 1) || (avctx->height & 1)) {
66 av_log(avctx, AV_LOG_ERROR, "Invalid video dimensions: %dx%d\n",
67 avctx->width, avctx->height);
68 return AVERROR(EINVAL);
69 }
70
5c96f029
AK
71 s->frame = av_frame_alloc();
72 if (!s->frame)
73 return AVERROR(ENOMEM);
3b199d29 74
4e114829
MM
75 return 0;
76}
77
a55d5bdc 78static int mm_decode_pal(MmContext *s)
60f451b6
PR
79{
80 int i;
a55d5bdc
RB
81
82 bytestream2_skip(&s->gb, 4);
83 for (i = 0; i < 128; i++) {
84 s->palette[i] = bytestream2_get_be24(&s->gb);
60f451b6 85 s->palette[i+128] = s->palette[i]<<2;
60f451b6 86 }
a55d5bdc
RB
87
88 return 0;
60f451b6
PR
89}
90
091bc6ca
PR
91/**
92 * @param half_horiz Half horizontal resolution (0 or 1)
93 * @param half_vert Half vertical resolution (0 or 1)
94 */
a55d5bdc 95static int mm_decode_intra(MmContext * s, int half_horiz, int half_vert)
4e114829 96{
246b050f 97 int x = 0, y = 0;
4e114829 98
a55d5bdc 99 while (bytestream2_get_bytes_left(&s->gb) > 0) {
4e114829
MM
100 int run_length, color;
101
9b5a4a9c
LB
102 // writes one more line when half_vert is true
103 if (y >= s->avctx->height + !!half_vert)
a55d5bdc 104 return 0;
94e58e57 105
a55d5bdc
RB
106 color = bytestream2_get_byte(&s->gb);
107 if (color & 0x80) {
4e114829 108 run_length = 1;
4e114829 109 }else{
a55d5bdc
RB
110 run_length = (color & 0x7f) + 2;
111 color = bytestream2_get_byte(&s->gb);
4e114829
MM
112 }
113
114 if (half_horiz)
115 run_length *=2;
116
9b5a4a9c
LB
117 if (s->avctx->width - x < run_length)
118 return AVERROR_INVALIDDATA;
119
4e114829 120 if (color) {
5c96f029 121 memset(s->frame->data[0] + y*s->frame->linesize[0] + x, color, run_length);
4e114829 122 if (half_vert)
5c96f029 123 memset(s->frame->data[0] + (y+1)*s->frame->linesize[0] + x, color, run_length);
4e114829
MM
124 }
125 x+= run_length;
126
127 if (x >= s->avctx->width) {
128 x=0;
091bc6ca 129 y += 1 + half_vert;
4e114829
MM
130 }
131 }
a55d5bdc
RB
132
133 return 0;
4e114829
MM
134}
135
091bc6ca
PR
136/*
137 * @param half_horiz Half horizontal resolution (0 or 1)
138 * @param half_vert Half vertical resolution (0 or 1)
139 */
a55d5bdc 140static int mm_decode_inter(MmContext * s, int half_horiz, int half_vert)
4e114829 141{
fc06ee6e
VG
142 int data_off = bytestream2_get_le16(&s->gb);
143 int y = 0;
a55d5bdc 144 GetByteContext data_ptr;
4e114829 145
a55d5bdc
RB
146 if (bytestream2_get_bytes_left(&s->gb) < data_off)
147 return AVERROR_INVALIDDATA;
148
149 bytestream2_init(&data_ptr, s->gb.buffer + data_off, bytestream2_get_bytes_left(&s->gb) - data_off);
150 while (s->gb.buffer < data_ptr.buffer_start) {
4e114829 151 int i, j;
a55d5bdc
RB
152 int length = bytestream2_get_byte(&s->gb);
153 int x = bytestream2_get_byte(&s->gb) + ((length & 0x80) << 1);
154 length &= 0x7F;
4e114829
MM
155
156 if (length==0) {
157 y += x;
158 continue;
159 }
160
94e58e57 161 if (y + half_vert >= s->avctx->height)
a55d5bdc 162 return 0;
94e58e57 163
4e114829 164 for(i=0; i<length; i++) {
a55d5bdc 165 int replace_array = bytestream2_get_byte(&s->gb);
4e114829 166 for(j=0; j<8; j++) {
a55d5bdc 167 int replace = (replace_array >> (7-j)) & 1;
70cd3b8e
MN
168 if (x + half_horiz >= s->avctx->width)
169 return AVERROR_INVALIDDATA;
4e114829 170 if (replace) {
a55d5bdc 171 int color = bytestream2_get_byte(&data_ptr);
5c96f029 172 s->frame->data[0][y*s->frame->linesize[0] + x] = color;
4e114829 173 if (half_horiz)
5c96f029 174 s->frame->data[0][y*s->frame->linesize[0] + x + 1] = color;
4e114829 175 if (half_vert) {
5c96f029 176 s->frame->data[0][(y+1)*s->frame->linesize[0] + x] = color;
4e114829 177 if (half_horiz)
5c96f029 178 s->frame->data[0][(y+1)*s->frame->linesize[0] + x + 1] = color;
4e114829 179 }
4e114829 180 }
091bc6ca 181 x += 1 + half_horiz;
4e114829
MM
182 }
183 }
184
091bc6ca 185 y += 1 + half_vert;
4e114829 186 }
a55d5bdc
RB
187
188 return 0;
4e114829
MM
189}
190
191static int mm_decode_frame(AVCodecContext *avctx,
df9b9567 192 void *data, int *got_frame,
7a00bbad 193 AVPacket *avpkt)
4e114829 194{
7a00bbad
TB
195 const uint8_t *buf = avpkt->data;
196 int buf_size = avpkt->size;
4e114829 197 MmContext *s = avctx->priv_data;
a55d5bdc 198 int type, res;
4e114829 199
a55d5bdc
RB
200 if (buf_size < MM_PREAMBLE_SIZE)
201 return AVERROR_INVALIDDATA;
fead30d4 202 type = AV_RL16(&buf[0]);
4e114829
MM
203 buf += MM_PREAMBLE_SIZE;
204 buf_size -= MM_PREAMBLE_SIZE;
a55d5bdc 205 bytestream2_init(&s->gb, buf, buf_size);
4e114829 206
5c96f029 207 if ((res = ff_reget_buffer(avctx, s->frame)) < 0) {
6e7cf13b 208 av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
759001c5 209 return res;
6e7cf13b
PR
210 }
211
4e114829 212 switch(type) {
a55d5bdc
RB
213 case MM_TYPE_PALETTE : res = mm_decode_pal(s); return buf_size;
214 case MM_TYPE_INTRA : res = mm_decode_intra(s, 0, 0); break;
215 case MM_TYPE_INTRA_HH : res = mm_decode_intra(s, 1, 0); break;
216 case MM_TYPE_INTRA_HHV : res = mm_decode_intra(s, 1, 1); break;
217 case MM_TYPE_INTER : res = mm_decode_inter(s, 0, 0); break;
218 case MM_TYPE_INTER_HH : res = mm_decode_inter(s, 1, 0); break;
219 case MM_TYPE_INTER_HHV : res = mm_decode_inter(s, 1, 1); break;
220 default:
221 res = AVERROR_INVALIDDATA;
222 break;
4e114829 223 }
a55d5bdc
RB
224 if (res < 0)
225 return res;
4e114829 226
5c96f029 227 memcpy(s->frame->data[1], s->palette, AVPALETTE_SIZE);
60f451b6 228
5c96f029 229 if ((res = av_frame_ref(data, s->frame)) < 0)
759001c5
AK
230 return res;
231
df9b9567 232 *got_frame = 1;
4e114829
MM
233
234 return buf_size;
235}
236
98a6fff9 237static av_cold int mm_decode_end(AVCodecContext *avctx)
4e114829
MM
238{
239 MmContext *s = avctx->priv_data;
240
5c96f029 241 av_frame_free(&s->frame);
4e114829
MM
242
243 return 0;
244}
245
d36beb3f 246AVCodec ff_mmvideo_decoder = {
ec6402b7 247 .name = "mmvideo",
b2bed932 248 .long_name = NULL_IF_CONFIG_SMALL("American Laser Games MM Video"),
ec6402b7 249 .type = AVMEDIA_TYPE_VIDEO,
36ef5369 250 .id = AV_CODEC_ID_MMVIDEO,
ec6402b7
AK
251 .priv_data_size = sizeof(MmContext),
252 .init = mm_decode_init,
253 .close = mm_decode_end,
254 .decode = mm_decode_frame,
def97856 255 .capabilities = AV_CODEC_CAP_DR1,
4e114829 256};