sgi: set the row boundary to the correct value
[libav.git] / libavcodec / sgidec.c
1 /*
2 * SGI image decoder
3 * Todd Kirby <doubleshot@pacbell.net>
4 *
5 * This file is part of Libav.
6 *
7 * Libav is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * Libav is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with Libav; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22 #include "libavutil/imgutils.h"
23 #include "avcodec.h"
24 #include "bytestream.h"
25 #include "internal.h"
26 #include "sgi.h"
27
28 typedef struct SgiState {
29 AVCodecContext *avctx;
30 unsigned int width;
31 unsigned int height;
32 unsigned int depth;
33 unsigned int bytes_per_channel;
34 int linesize;
35 GetByteContext g;
36 } SgiState;
37
38 /**
39 * Expand an RLE row into a channel.
40 * @param s the current image state
41 * @param out_buf Points to one line after the output buffer.
42 * @param len length of out_buf in bytes
43 * @param pixelstride pixel stride of input buffer
44 * @return size of output in bytes, -1 if buffer overflows
45 */
46 static int expand_rle_row(SgiState *s, uint8_t *out_buf,
47 int len, int pixelstride)
48 {
49 unsigned char pixel, count;
50 unsigned char *orig = out_buf;
51 uint8_t *out_end = out_buf + len;
52
53 while (out_buf < out_end) {
54 if (bytestream2_get_bytes_left(&s->g) < 1)
55 return AVERROR_INVALIDDATA;
56 pixel = bytestream2_get_byteu(&s->g);
57 if (!(count = (pixel & 0x7f))) {
58 break;
59 }
60
61 /* Check for buffer overflow. */
62 if (pixelstride * (count - 1) >= len) {
63 av_log(s->avctx, AV_LOG_ERROR, "Invalid pixel count.\n");
64 return AVERROR_INVALIDDATA;
65 }
66
67 if (pixel & 0x80) {
68 while (count--) {
69 *out_buf = bytestream2_get_byte(&s->g);
70 out_buf += pixelstride;
71 }
72 } else {
73 pixel = bytestream2_get_byte(&s->g);
74
75 while (count--) {
76 *out_buf = pixel;
77 out_buf += pixelstride;
78 }
79 }
80 }
81 return (out_buf - orig) / pixelstride;
82 }
83
84 /**
85 * Read a run length encoded SGI image.
86 * @param out_buf output buffer
87 * @param s the current image state
88 * @return 0 if no error, else return error number.
89 */
90 static int read_rle_sgi(uint8_t *out_buf, SgiState *s)
91 {
92 uint8_t *dest_row;
93 unsigned int len = s->height * s->depth * 4;
94 GetByteContext g_table = s->g;
95 unsigned int y, z;
96 unsigned int start_offset;
97
98 /* size of RLE offset and length tables */
99 if (len * 2 > bytestream2_get_bytes_left(&s->g)) {
100 return AVERROR_INVALIDDATA;
101 }
102
103 for (z = 0; z < s->depth; z++) {
104 dest_row = out_buf;
105 for (y = 0; y < s->height; y++) {
106 dest_row -= s->linesize;
107 start_offset = bytestream2_get_be32(&g_table);
108 bytestream2_seek(&s->g, start_offset, SEEK_SET);
109 if (expand_rle_row(s, dest_row + z, s->width * s->depth,
110 s->depth) != s->width) {
111 return AVERROR_INVALIDDATA;
112 }
113 }
114 }
115 return 0;
116 }
117
118 /**
119 * Read an uncompressed SGI image.
120 * @param out_buf output buffer
121 * @param out_end end ofoutput buffer
122 * @param s the current image state
123 * @return 0 if read success, otherwise return -1.
124 */
125 static int read_uncompressed_sgi(unsigned char *out_buf, uint8_t *out_end,
126 SgiState *s)
127 {
128 int x, y, z;
129 unsigned int offset = s->height * s->width * s->bytes_per_channel;
130 GetByteContext gp[4];
131
132 /* Test buffer size. */
133 if (offset * s->depth > bytestream2_get_bytes_left(&s->g))
134 return AVERROR_INVALIDDATA;
135
136 /* Create a reader for each plane */
137 for (z = 0; z < s->depth; z++) {
138 gp[z] = s->g;
139 bytestream2_skip(&gp[z], z * offset);
140 }
141
142 for (y = s->height - 1; y >= 0; y--) {
143 out_end = out_buf + (y * s->linesize);
144 if (s->bytes_per_channel == 1) {
145 for (x = s->width; x > 0; x--)
146 for (z = 0; z < s->depth; z++)
147 *out_end++ = bytestream2_get_byteu(&gp[z]);
148 } else {
149 uint16_t *out16 = (uint16_t *)out_end;
150 for (x = s->width; x > 0; x--)
151 for (z = 0; z < s->depth; z++)
152 *out16++ = bytestream2_get_ne16u(&gp[z]);
153 }
154 }
155 return 0;
156 }
157
158 static int decode_frame(AVCodecContext *avctx,
159 void *data, int *got_frame,
160 AVPacket *avpkt)
161 {
162 SgiState *s = avctx->priv_data;
163 AVFrame *p = data;
164 unsigned int dimension, rle;
165 int ret = 0;
166 uint8_t *out_buf, *out_end;
167
168 bytestream2_init(&s->g, avpkt->data, avpkt->size);
169 if (bytestream2_get_bytes_left(&s->g) < SGI_HEADER_SIZE) {
170 av_log(avctx, AV_LOG_ERROR, "buf_size too small (%d)\n", avpkt->size);
171 return AVERROR_INVALIDDATA;
172 }
173
174 /* Test for SGI magic. */
175 if (bytestream2_get_be16(&s->g) != SGI_MAGIC) {
176 av_log(avctx, AV_LOG_ERROR, "bad magic number\n");
177 return AVERROR_INVALIDDATA;
178 }
179
180 rle = bytestream2_get_byte(&s->g);
181 s->bytes_per_channel = bytestream2_get_byte(&s->g);
182 dimension = bytestream2_get_be16(&s->g);
183 s->width = bytestream2_get_be16(&s->g);
184 s->height = bytestream2_get_be16(&s->g);
185 s->depth = bytestream2_get_be16(&s->g);
186
187 if (s->bytes_per_channel != 1 && (s->bytes_per_channel != 2 || rle)) {
188 av_log(avctx, AV_LOG_ERROR, "wrong channel number\n");
189 return -1;
190 }
191
192 /* Check for supported image dimensions. */
193 if (dimension != 2 && dimension != 3) {
194 av_log(avctx, AV_LOG_ERROR, "wrong dimension number\n");
195 return -1;
196 }
197
198 if (s->depth == SGI_GRAYSCALE) {
199 avctx->pix_fmt = s->bytes_per_channel == 2 ? AV_PIX_FMT_GRAY16BE : AV_PIX_FMT_GRAY8;
200 } else if (s->depth == SGI_RGB) {
201 avctx->pix_fmt = s->bytes_per_channel == 2 ? AV_PIX_FMT_RGB48BE : AV_PIX_FMT_RGB24;
202 } else if (s->depth == SGI_RGBA && s->bytes_per_channel == 1) {
203 avctx->pix_fmt = AV_PIX_FMT_RGBA;
204 } else {
205 av_log(avctx, AV_LOG_ERROR, "wrong picture format\n");
206 return -1;
207 }
208
209 ret = ff_set_dimensions(avctx, s->width, s->height);
210 if (ret < 0)
211 return ret;
212
213 if (ff_get_buffer(avctx, p, 0) < 0) {
214 av_log(avctx, AV_LOG_ERROR, "get_buffer() failed.\n");
215 return -1;
216 }
217
218 p->pict_type = AV_PICTURE_TYPE_I;
219 p->key_frame = 1;
220 out_buf = p->data[0];
221
222 out_end = out_buf + p->linesize[0] * s->height;
223
224 s->linesize = p->linesize[0];
225
226 /* Skip header. */
227 bytestream2_seek(&s->g, SGI_HEADER_SIZE, SEEK_SET);
228 if (rle) {
229 ret = read_rle_sgi(out_end, s);
230 } else {
231 ret = read_uncompressed_sgi(out_buf, out_end, s);
232 }
233
234 if (ret == 0) {
235 *got_frame = 1;
236 return avpkt->size;
237 } else {
238 return ret;
239 }
240 }
241
242 static av_cold int sgi_decode_init(AVCodecContext *avctx)
243 {
244 SgiState *s = avctx->priv_data;
245
246 s->avctx = avctx;
247
248 return 0;
249 }
250
251 AVCodec ff_sgi_decoder = {
252 .name = "sgi",
253 .long_name = NULL_IF_CONFIG_SMALL("SGI image"),
254 .type = AVMEDIA_TYPE_VIDEO,
255 .id = AV_CODEC_ID_SGI,
256 .priv_data_size = sizeof(SgiState),
257 .decode = decode_frame,
258 .init = sgi_decode_init,
259 .capabilities = CODEC_CAP_DR1,
260 };