* fixes for broken builds on Solaris, OS2 and all bingendian
[libav.git] / libavcodec / msrle.c
CommitLineData
2fdf638b
MM
1/*
2 * Micrsoft RLE Video Decoder
3 * Copyright (C) 2003 the ffmpeg project
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20/**
21 * @file msrle.c
22 * MS RLE Video Decoder by Mike Melanson (melanson@pcisys.net)
23 * For more information about the MS RLE format, visit:
24 * http://www.pcisys.net/~melanson/codecs/
25 *
26 * The MS RLE decoder outputs PAL8 colorspace data.
27 *
28 * Note that this decoder expects the palette colors from the end of the
875efafa 29 * BITMAPINFO header passed through palctrl.
2fdf638b
MM
30 */
31
32#include <stdio.h>
33#include <stdlib.h>
34#include <string.h>
35#include <unistd.h>
36
37#include "common.h"
38#include "avcodec.h"
39#include "dsputil.h"
40
41typedef struct MsrleContext {
42 AVCodecContext *avctx;
43 AVFrame frame;
44 AVFrame prev_frame;
45
46 unsigned char *buf;
47 int size;
48
2fdf638b
MM
49} MsrleContext;
50
51#define FETCH_NEXT_STREAM_BYTE() \
52 if (stream_ptr >= s->size) \
53 { \
9b879566 54 av_log(s->avctx, AV_LOG_ERROR, " MS RLE: stream ptr just went out of bounds (1)\n"); \
2fdf638b
MM
55 return; \
56 } \
57 stream_byte = s->buf[stream_ptr++];
58
59static void msrle_decode_pal8(MsrleContext *s)
60{
61 int stream_ptr = 0;
62 unsigned char rle_code;
63 unsigned char extra_byte;
64 unsigned char stream_byte;
65 int pixel_ptr = 0;
66 int row_dec = s->frame.linesize[0];
67 int row_ptr = (s->avctx->height - 1) * row_dec;
68 int frame_size = row_dec * s->avctx->height;
69
70 while (row_ptr >= 0) {
71 FETCH_NEXT_STREAM_BYTE();
72 rle_code = stream_byte;
73 if (rle_code == 0) {
74 /* fetch the next byte to see how to handle escape code */
75 FETCH_NEXT_STREAM_BYTE();
76 if (stream_byte == 0) {
77 /* line is done, goto the next one */
78 row_ptr -= row_dec;
79 pixel_ptr = 0;
80 } else if (stream_byte == 1) {
81 /* decode is done */
82 return;
83 } else if (stream_byte == 2) {
84 /* reposition frame decode coordinates */
85 FETCH_NEXT_STREAM_BYTE();
86 pixel_ptr += stream_byte;
87 FETCH_NEXT_STREAM_BYTE();
88 row_ptr -= stream_byte * row_dec;
89 } else {
90 /* copy pixels from encoded stream */
91 if ((row_ptr + pixel_ptr + stream_byte > frame_size) ||
92 (row_ptr < 0)) {
9b879566 93 av_log(s->avctx, AV_LOG_ERROR, " MS RLE: frame ptr just went out of bounds (1)\n");
2fdf638b
MM
94 return;
95 }
96
97 rle_code = stream_byte;
98 extra_byte = stream_byte & 0x01;
99 if (stream_ptr + rle_code + extra_byte > s->size) {
9b879566 100 av_log(s->avctx, AV_LOG_ERROR, " MS RLE: stream ptr just went out of bounds (2)\n");
2fdf638b
MM
101 return;
102 }
103
104 while (rle_code--) {
105 FETCH_NEXT_STREAM_BYTE();
106 s->frame.data[0][row_ptr + pixel_ptr] = stream_byte;
107 pixel_ptr++;
108 }
109
110 /* if the RLE code is odd, skip a byte in the stream */
111 if (extra_byte)
112 stream_ptr++;
113 }
114 } else {
115 /* decode a run of data */
116 if ((row_ptr + pixel_ptr + stream_byte > frame_size) ||
117 (row_ptr < 0)) {
9b879566 118 av_log(s->avctx, AV_LOG_ERROR, " MS RLE: frame ptr just went out of bounds (2)\n");
2fdf638b
MM
119 return;
120 }
121
122 FETCH_NEXT_STREAM_BYTE();
123
124 while(rle_code--) {
125 s->frame.data[0][row_ptr + pixel_ptr] = stream_byte;
126 pixel_ptr++;
127 }
128 }
129 }
130
131 /* make the palette available */
875efafa
RT
132 memcpy(s->frame.data[1], s->avctx->palctrl->palette, AVPALETTE_SIZE);
133 if (s->avctx->palctrl->palette_changed) {
134 s->frame.palette_has_changed = 1;
135 s->avctx->palctrl->palette_changed = 0;
136 }
2fdf638b
MM
137
138 /* one last sanity check on the way out */
139 if (stream_ptr < s->size)
9b879566 140 av_log(s->avctx, AV_LOG_ERROR, " MS RLE: ended frame decode with bytes left over (%d < %d)\n",
2fdf638b
MM
141 stream_ptr, s->size);
142}
143
144static int msrle_decode_init(AVCodecContext *avctx)
145{
146 MsrleContext *s = (MsrleContext *)avctx->priv_data;
2fdf638b
MM
147
148 s->avctx = avctx;
149
150 avctx->pix_fmt = PIX_FMT_PAL8;
151 avctx->has_b_frames = 0;
152 s->frame.data[0] = s->prev_frame.data[0] = NULL;
153
2fdf638b
MM
154 return 0;
155}
156
157static int msrle_decode_frame(AVCodecContext *avctx,
158 void *data, int *data_size,
159 uint8_t *buf, int buf_size)
160{
161 MsrleContext *s = (MsrleContext *)avctx->priv_data;
162
163 s->buf = buf;
164 s->size = buf_size;
165
04939fb7 166 s->frame.reference = 1;
2fdf638b 167 if (avctx->get_buffer(avctx, &s->frame)) {
9b879566 168 av_log(avctx, AV_LOG_ERROR, " MS RLE: get_buffer() failed\n");
2fdf638b
MM
169 return -1;
170 }
171
875efafa 172 if (s->prev_frame.data[0] && (s->frame.linesize[0] != s->prev_frame.linesize[0]))
9b879566 173 av_log(avctx, AV_LOG_ERROR, " MS RLE: Buffer linesize changed: current %u, previous %u.\n"
875efafa
RT
174 " Expect wrong image and/or crash!\n",
175 s->frame.linesize[0], s->prev_frame.linesize[0]);
176
2fdf638b 177 /* grossly inefficient, but...oh well */
bc0219fd
RT
178 if (s->prev_frame.data[0] != NULL)
179 memcpy(s->frame.data[0], s->prev_frame.data[0],
2fdf638b
MM
180 s->frame.linesize[0] * s->avctx->height);
181
182 msrle_decode_pal8(s);
183
04939fb7
RT
184 if (s->prev_frame.data[0])
185 avctx->release_buffer(avctx, &s->prev_frame);
2fdf638b
MM
186
187 /* shuffle frames */
188 s->prev_frame = s->frame;
189
190 *data_size = sizeof(AVFrame);
191 *(AVFrame*)data = s->frame;
192
193 /* report that the buffer was completely consumed */
194 return buf_size;
195}
196
197static int msrle_decode_end(AVCodecContext *avctx)
198{
199 MsrleContext *s = (MsrleContext *)avctx->priv_data;
200
201 /* release the last frame */
202 if (s->prev_frame.data[0])
203 avctx->release_buffer(avctx, &s->prev_frame);
204
205 return 0;
206}
207
208AVCodec msrle_decoder = {
209 "msrle",
210 CODEC_TYPE_VIDEO,
211 CODEC_ID_MSRLE,
212 sizeof(MsrleContext),
213 msrle_decode_init,
214 NULL,
215 msrle_decode_end,
216 msrle_decode_frame,
217 CODEC_CAP_DR1,
218};