mmvideo: fix uninitialized variable use in mm_decode_intra
[libav.git] / libavcodec / mmvideo.c
CommitLineData
4e114829
MM
1/*
2 * American Laser Games MM Video Decoder
60f451b6 3 * Copyright (c) 2006,2008 Peter Ross
4e114829 4 *
2912e87a 5 * This file is part of Libav.
b78e7197 6 *
2912e87a 7 * Libav is free software; you can redistribute it and/or
4e114829
MM
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
b78e7197 10 * version 2.1 of the License, or (at your option) any later version.
4e114829 11 *
2912e87a 12 * Libav is distributed in the hope that it will be useful,
4e114829
MM
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
2912e87a 18 * License along with Libav; if not, write to the Free Software
e5a389a1 19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
4e114829
MM
20 */
21
22/**
ba87f080 23 * @file
4e114829 24 * American Laser Games MM Video Decoder
8600106a 25 * by Peter Ross (pross@xvid.org)
4e114829
MM
26 *
27 * The MM format was used by IBM-PC ports of ALG's "arcade shooter" games,
28 * including Mad Dog McCree and Crime Patrol.
29 *
30 * Technical details here:
31 * http://wiki.multimedia.cx/index.php?title=American_Laser_Games_MM
32 */
33
6a5d31ac 34#include "libavutil/intreadwrite.h"
4e114829 35#include "avcodec.h"
a55d5bdc 36#include "bytestream.h"
759001c5 37#include "internal.h"
4e114829
MM
38
39#define MM_PREAMBLE_SIZE 6
40
41#define MM_TYPE_INTER 0x5
42#define MM_TYPE_INTRA 0x8
43#define MM_TYPE_INTRA_HH 0xc
44#define MM_TYPE_INTER_HH 0xd
45#define MM_TYPE_INTRA_HHV 0xe
46#define MM_TYPE_INTER_HHV 0xf
60f451b6 47#define MM_TYPE_PALETTE 0x31
4e114829
MM
48
49typedef struct MmContext {
50 AVCodecContext *avctx;
51 AVFrame frame;
60f451b6 52 int palette[AVPALETTE_COUNT];
a55d5bdc 53 GetByteContext gb;
4e114829
MM
54} MmContext;
55
98a6fff9 56static av_cold int mm_decode_init(AVCodecContext *avctx)
4e114829
MM
57{
58 MmContext *s = avctx->priv_data;
59
60 s->avctx = avctx;
61
716d413c 62 avctx->pix_fmt = AV_PIX_FMT_PAL8;
4e114829 63
3b199d29
AK
64 avcodec_get_frame_defaults(&s->frame);
65
4e114829
MM
66 return 0;
67}
68
a55d5bdc 69static int mm_decode_pal(MmContext *s)
60f451b6
PR
70{
71 int i;
a55d5bdc
RB
72
73 bytestream2_skip(&s->gb, 4);
74 for (i = 0; i < 128; i++) {
75 s->palette[i] = bytestream2_get_be24(&s->gb);
60f451b6 76 s->palette[i+128] = s->palette[i]<<2;
60f451b6 77 }
a55d5bdc
RB
78
79 return 0;
60f451b6
PR
80}
81
091bc6ca
PR
82/**
83 * @param half_horiz Half horizontal resolution (0 or 1)
84 * @param half_vert Half vertical resolution (0 or 1)
85 */
a55d5bdc 86static int mm_decode_intra(MmContext * s, int half_horiz, int half_vert)
4e114829 87{
246b050f 88 int x = 0, y = 0;
4e114829 89
a55d5bdc 90 while (bytestream2_get_bytes_left(&s->gb) > 0) {
4e114829
MM
91 int run_length, color;
92
94e58e57 93 if (y >= s->avctx->height)
a55d5bdc 94 return 0;
94e58e57 95
a55d5bdc
RB
96 color = bytestream2_get_byte(&s->gb);
97 if (color & 0x80) {
4e114829 98 run_length = 1;
4e114829 99 }else{
a55d5bdc
RB
100 run_length = (color & 0x7f) + 2;
101 color = bytestream2_get_byte(&s->gb);
4e114829
MM
102 }
103
104 if (half_horiz)
105 run_length *=2;
106
107 if (color) {
108 memset(s->frame.data[0] + y*s->frame.linesize[0] + x, color, run_length);
109 if (half_vert)
110 memset(s->frame.data[0] + (y+1)*s->frame.linesize[0] + x, color, run_length);
111 }
112 x+= run_length;
113
114 if (x >= s->avctx->width) {
115 x=0;
091bc6ca 116 y += 1 + half_vert;
4e114829
MM
117 }
118 }
a55d5bdc
RB
119
120 return 0;
4e114829
MM
121}
122
091bc6ca
PR
123/*
124 * @param half_horiz Half horizontal resolution (0 or 1)
125 * @param half_vert Half vertical resolution (0 or 1)
126 */
a55d5bdc 127static int mm_decode_inter(MmContext * s, int half_horiz, int half_vert)
4e114829 128{
fc06ee6e
VG
129 int data_off = bytestream2_get_le16(&s->gb);
130 int y = 0;
a55d5bdc 131 GetByteContext data_ptr;
4e114829 132
a55d5bdc
RB
133 if (bytestream2_get_bytes_left(&s->gb) < data_off)
134 return AVERROR_INVALIDDATA;
135
136 bytestream2_init(&data_ptr, s->gb.buffer + data_off, bytestream2_get_bytes_left(&s->gb) - data_off);
137 while (s->gb.buffer < data_ptr.buffer_start) {
4e114829 138 int i, j;
a55d5bdc
RB
139 int length = bytestream2_get_byte(&s->gb);
140 int x = bytestream2_get_byte(&s->gb) + ((length & 0x80) << 1);
141 length &= 0x7F;
4e114829
MM
142
143 if (length==0) {
144 y += x;
145 continue;
146 }
147
94e58e57 148 if (y + half_vert >= s->avctx->height)
a55d5bdc 149 return 0;
94e58e57 150
4e114829 151 for(i=0; i<length; i++) {
a55d5bdc 152 int replace_array = bytestream2_get_byte(&s->gb);
4e114829 153 for(j=0; j<8; j++) {
a55d5bdc 154 int replace = (replace_array >> (7-j)) & 1;
4e114829 155 if (replace) {
a55d5bdc 156 int color = bytestream2_get_byte(&data_ptr);
4e114829
MM
157 s->frame.data[0][y*s->frame.linesize[0] + x] = color;
158 if (half_horiz)
159 s->frame.data[0][y*s->frame.linesize[0] + x + 1] = color;
160 if (half_vert) {
161 s->frame.data[0][(y+1)*s->frame.linesize[0] + x] = color;
162 if (half_horiz)
163 s->frame.data[0][(y+1)*s->frame.linesize[0] + x + 1] = color;
164 }
4e114829 165 }
091bc6ca 166 x += 1 + half_horiz;
4e114829
MM
167 }
168 }
169
091bc6ca 170 y += 1 + half_vert;
4e114829 171 }
a55d5bdc
RB
172
173 return 0;
4e114829
MM
174}
175
176static int mm_decode_frame(AVCodecContext *avctx,
df9b9567 177 void *data, int *got_frame,
7a00bbad 178 AVPacket *avpkt)
4e114829 179{
7a00bbad
TB
180 const uint8_t *buf = avpkt->data;
181 int buf_size = avpkt->size;
4e114829 182 MmContext *s = avctx->priv_data;
a55d5bdc 183 int type, res;
4e114829 184
a55d5bdc
RB
185 if (buf_size < MM_PREAMBLE_SIZE)
186 return AVERROR_INVALIDDATA;
fead30d4 187 type = AV_RL16(&buf[0]);
4e114829
MM
188 buf += MM_PREAMBLE_SIZE;
189 buf_size -= MM_PREAMBLE_SIZE;
a55d5bdc 190 bytestream2_init(&s->gb, buf, buf_size);
4e114829 191
759001c5 192 if ((res = ff_reget_buffer(avctx, &s->frame)) < 0) {
6e7cf13b 193 av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
759001c5 194 return res;
6e7cf13b
PR
195 }
196
4e114829 197 switch(type) {
a55d5bdc
RB
198 case MM_TYPE_PALETTE : res = mm_decode_pal(s); return buf_size;
199 case MM_TYPE_INTRA : res = mm_decode_intra(s, 0, 0); break;
200 case MM_TYPE_INTRA_HH : res = mm_decode_intra(s, 1, 0); break;
201 case MM_TYPE_INTRA_HHV : res = mm_decode_intra(s, 1, 1); break;
202 case MM_TYPE_INTER : res = mm_decode_inter(s, 0, 0); break;
203 case MM_TYPE_INTER_HH : res = mm_decode_inter(s, 1, 0); break;
204 case MM_TYPE_INTER_HHV : res = mm_decode_inter(s, 1, 1); break;
205 default:
206 res = AVERROR_INVALIDDATA;
207 break;
4e114829 208 }
a55d5bdc
RB
209 if (res < 0)
210 return res;
4e114829 211
60f451b6
PR
212 memcpy(s->frame.data[1], s->palette, AVPALETTE_SIZE);
213
759001c5
AK
214 if ((res = av_frame_ref(data, &s->frame)) < 0)
215 return res;
216
df9b9567 217 *got_frame = 1;
4e114829
MM
218
219 return buf_size;
220}
221
98a6fff9 222static av_cold int mm_decode_end(AVCodecContext *avctx)
4e114829
MM
223{
224 MmContext *s = avctx->priv_data;
225
759001c5 226 av_frame_unref(&s->frame);
4e114829
MM
227
228 return 0;
229}
230
d36beb3f 231AVCodec ff_mmvideo_decoder = {
ec6402b7 232 .name = "mmvideo",
b2bed932 233 .long_name = NULL_IF_CONFIG_SMALL("American Laser Games MM Video"),
ec6402b7 234 .type = AVMEDIA_TYPE_VIDEO,
36ef5369 235 .id = AV_CODEC_ID_MMVIDEO,
ec6402b7
AK
236 .priv_data_size = sizeof(MmContext),
237 .init = mm_decode_init,
238 .close = mm_decode_end,
239 .decode = mm_decode_frame,
240 .capabilities = CODEC_CAP_DR1,
4e114829 241};