lavc: fix decode_frame() third parameter semantics for video decoders
[libav.git] / libavcodec / xxan.c
1 /*
2 * Wing Commander/Xan Video Decoder
3 * Copyright (C) 2011 Konstantin Shishkov
4 * based on work by Mike Melanson
5 *
6 * This file is part of Libav.
7 *
8 * Libav is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * Libav is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with Libav; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23 #include "avcodec.h"
24 #include "libavutil/intreadwrite.h"
25 #include "libavutil/mem.h"
26 #include "bytestream.h"
27 #define BITSTREAM_READER_LE
28 #include "get_bits.h"
29
30 typedef struct XanContext {
31 AVCodecContext *avctx;
32 AVFrame pic;
33
34 uint8_t *y_buffer;
35 uint8_t *scratch_buffer;
36 int buffer_size;
37 GetByteContext gb;
38 } XanContext;
39
40 static av_cold int xan_decode_init(AVCodecContext *avctx)
41 {
42 XanContext *s = avctx->priv_data;
43
44 s->avctx = avctx;
45
46 avctx->pix_fmt = AV_PIX_FMT_YUV420P;
47
48 s->buffer_size = avctx->width * avctx->height;
49 s->y_buffer = av_malloc(s->buffer_size);
50 if (!s->y_buffer)
51 return AVERROR(ENOMEM);
52 s->scratch_buffer = av_malloc(s->buffer_size + 130);
53 if (!s->scratch_buffer) {
54 av_freep(&s->y_buffer);
55 return AVERROR(ENOMEM);
56 }
57
58 return 0;
59 }
60
61 static int xan_unpack_luma(XanContext *s,
62 uint8_t *dst, const int dst_size)
63 {
64 int tree_size, eof;
65 int bits, mask;
66 int tree_root, node;
67 const uint8_t *dst_end = dst + dst_size;
68 GetByteContext tree = s->gb;
69 int start_off = bytestream2_tell(&tree);
70
71 tree_size = bytestream2_get_byte(&s->gb);
72 eof = bytestream2_get_byte(&s->gb);
73 tree_root = eof + tree_size;
74 bytestream2_skip(&s->gb, tree_size * 2);
75
76 node = tree_root;
77 bits = bytestream2_get_byte(&s->gb);
78 mask = 0x80;
79 for (;;) {
80 int bit = !!(bits & mask);
81 mask >>= 1;
82 bytestream2_seek(&tree, start_off + node*2 + bit - eof * 2, SEEK_SET);
83 node = bytestream2_get_byte(&tree);
84 if (node == eof)
85 break;
86 if (node < eof) {
87 *dst++ = node;
88 if (dst > dst_end)
89 break;
90 node = tree_root;
91 }
92 if (!mask) {
93 if (bytestream2_get_bytes_left(&s->gb) <= 0)
94 break;
95 bits = bytestream2_get_byteu(&s->gb);
96 mask = 0x80;
97 }
98 }
99 return dst != dst_end ? AVERROR_INVALIDDATA : 0;
100 }
101
102 /* almost the same as in xan_wc3 decoder */
103 static int xan_unpack(XanContext *s,
104 uint8_t *dest, const int dest_len)
105 {
106 uint8_t opcode;
107 int size;
108 uint8_t *orig_dest = dest;
109 const uint8_t *dest_end = dest + dest_len;
110
111 while (dest < dest_end) {
112 if (bytestream2_get_bytes_left(&s->gb) <= 0)
113 return AVERROR_INVALIDDATA;
114
115 opcode = bytestream2_get_byteu(&s->gb);
116
117 if (opcode < 0xe0) {
118 int size2, back;
119 if ((opcode & 0x80) == 0) {
120 size = opcode & 3;
121 back = ((opcode & 0x60) << 3) + bytestream2_get_byte(&s->gb) + 1;
122 size2 = ((opcode & 0x1c) >> 2) + 3;
123 } else if ((opcode & 0x40) == 0) {
124 size = bytestream2_peek_byte(&s->gb) >> 6;
125 back = (bytestream2_get_be16(&s->gb) & 0x3fff) + 1;
126 size2 = (opcode & 0x3f) + 4;
127 } else {
128 size = opcode & 3;
129 back = ((opcode & 0x10) << 12) + bytestream2_get_be16(&s->gb) + 1;
130 size2 = ((opcode & 0x0c) << 6) + bytestream2_get_byte(&s->gb) + 5;
131 if (size + size2 > dest_end - dest)
132 break;
133 }
134 if (dest + size + size2 > dest_end ||
135 dest - orig_dest + size < back)
136 return -1;
137 bytestream2_get_buffer(&s->gb, dest, size);
138 dest += size;
139 av_memcpy_backptr(dest, back, size2);
140 dest += size2;
141 } else {
142 int finish = opcode >= 0xfc;
143
144 size = finish ? opcode & 3 : ((opcode & 0x1f) << 2) + 4;
145 if (dest_end - dest < size)
146 return -1;
147 bytestream2_get_buffer(&s->gb, dest, size);
148 dest += size;
149 if (finish)
150 break;
151 }
152 }
153 return dest - orig_dest;
154 }
155
156 static int xan_decode_chroma(AVCodecContext *avctx, unsigned chroma_off)
157 {
158 XanContext *s = avctx->priv_data;
159 uint8_t *U, *V;
160 int val, uval, vval;
161 int i, j;
162 const uint8_t *src, *src_end;
163 const uint8_t *table;
164 int mode, offset, dec_size, table_size;
165
166 if (!chroma_off)
167 return 0;
168 if (chroma_off + 4 >= bytestream2_get_bytes_left(&s->gb)) {
169 av_log(avctx, AV_LOG_ERROR, "Invalid chroma block position\n");
170 return -1;
171 }
172 bytestream2_seek(&s->gb, chroma_off + 4, SEEK_SET);
173 mode = bytestream2_get_le16(&s->gb);
174 table = s->gb.buffer;
175 table_size = bytestream2_get_le16(&s->gb);
176 offset = table_size * 2;
177 table_size += 1;
178
179 if (offset >= bytestream2_get_bytes_left(&s->gb)) {
180 av_log(avctx, AV_LOG_ERROR, "Invalid chroma block offset\n");
181 return -1;
182 }
183
184 bytestream2_skip(&s->gb, offset);
185 memset(s->scratch_buffer, 0, s->buffer_size);
186 dec_size = xan_unpack(s, s->scratch_buffer, s->buffer_size);
187 if (dec_size < 0) {
188 av_log(avctx, AV_LOG_ERROR, "Chroma unpacking failed\n");
189 return -1;
190 }
191
192 U = s->pic.data[1];
193 V = s->pic.data[2];
194 src = s->scratch_buffer;
195 src_end = src + dec_size;
196 if (mode) {
197 for (j = 0; j < avctx->height >> 1; j++) {
198 for (i = 0; i < avctx->width >> 1; i++) {
199 val = *src++;
200 if (val && val < table_size) {
201 val = AV_RL16(table + (val << 1));
202 uval = (val >> 3) & 0xF8;
203 vval = (val >> 8) & 0xF8;
204 U[i] = uval | (uval >> 5);
205 V[i] = vval | (vval >> 5);
206 }
207 if (src == src_end)
208 return 0;
209 }
210 U += s->pic.linesize[1];
211 V += s->pic.linesize[2];
212 }
213 } else {
214 uint8_t *U2 = U + s->pic.linesize[1];
215 uint8_t *V2 = V + s->pic.linesize[2];
216
217 for (j = 0; j < avctx->height >> 2; j++) {
218 for (i = 0; i < avctx->width >> 1; i += 2) {
219 val = *src++;
220 if (val && val < table_size) {
221 val = AV_RL16(table + (val << 1));
222 uval = (val >> 3) & 0xF8;
223 vval = (val >> 8) & 0xF8;
224 U[i] = U[i+1] = U2[i] = U2[i+1] = uval | (uval >> 5);
225 V[i] = V[i+1] = V2[i] = V2[i+1] = vval | (vval >> 5);
226 }
227 }
228 U += s->pic.linesize[1] * 2;
229 V += s->pic.linesize[2] * 2;
230 U2 += s->pic.linesize[1] * 2;
231 V2 += s->pic.linesize[2] * 2;
232 }
233 }
234
235 return 0;
236 }
237
238 static int xan_decode_frame_type0(AVCodecContext *avctx)
239 {
240 XanContext *s = avctx->priv_data;
241 uint8_t *ybuf, *prev_buf, *src = s->scratch_buffer;
242 unsigned chroma_off, corr_off;
243 int cur, last;
244 int i, j;
245 int ret;
246
247 chroma_off = bytestream2_get_le32(&s->gb);
248 corr_off = bytestream2_get_le32(&s->gb);
249
250 if ((ret = xan_decode_chroma(avctx, chroma_off)) != 0)
251 return ret;
252
253 if (corr_off >= (s->gb.buffer_end - s->gb.buffer_start)) {
254 av_log(avctx, AV_LOG_WARNING, "Ignoring invalid correction block position\n");
255 corr_off = 0;
256 }
257 bytestream2_seek(&s->gb, 12, SEEK_SET);
258 ret = xan_unpack_luma(s, src, s->buffer_size >> 1);
259 if (ret) {
260 av_log(avctx, AV_LOG_ERROR, "Luma decoding failed\n");
261 return ret;
262 }
263
264 ybuf = s->y_buffer;
265 last = *src++;
266 ybuf[0] = last << 1;
267 for (j = 1; j < avctx->width - 1; j += 2) {
268 cur = (last + *src++) & 0x1F;
269 ybuf[j] = last + cur;
270 ybuf[j+1] = cur << 1;
271 last = cur;
272 }
273 ybuf[j] = last << 1;
274 prev_buf = ybuf;
275 ybuf += avctx->width;
276
277 for (i = 1; i < avctx->height; i++) {
278 last = ((prev_buf[0] >> 1) + *src++) & 0x1F;
279 ybuf[0] = last << 1;
280 for (j = 1; j < avctx->width - 1; j += 2) {
281 cur = ((prev_buf[j + 1] >> 1) + *src++) & 0x1F;
282 ybuf[j] = last + cur;
283 ybuf[j+1] = cur << 1;
284 last = cur;
285 }
286 ybuf[j] = last << 1;
287 prev_buf = ybuf;
288 ybuf += avctx->width;
289 }
290
291 if (corr_off) {
292 int dec_size;
293
294 bytestream2_seek(&s->gb, 8 + corr_off, SEEK_SET);
295 dec_size = xan_unpack(s, s->scratch_buffer, s->buffer_size);
296 if (dec_size < 0)
297 dec_size = 0;
298 for (i = 0; i < dec_size; i++)
299 s->y_buffer[i*2+1] = (s->y_buffer[i*2+1] + (s->scratch_buffer[i] << 1)) & 0x3F;
300 }
301
302 src = s->y_buffer;
303 ybuf = s->pic.data[0];
304 for (j = 0; j < avctx->height; j++) {
305 for (i = 0; i < avctx->width; i++)
306 ybuf[i] = (src[i] << 2) | (src[i] >> 3);
307 src += avctx->width;
308 ybuf += s->pic.linesize[0];
309 }
310
311 return 0;
312 }
313
314 static int xan_decode_frame_type1(AVCodecContext *avctx)
315 {
316 XanContext *s = avctx->priv_data;
317 uint8_t *ybuf, *src = s->scratch_buffer;
318 int cur, last;
319 int i, j;
320 int ret;
321
322 if ((ret = xan_decode_chroma(avctx, bytestream2_get_le32(&s->gb))) != 0)
323 return ret;
324
325 bytestream2_seek(&s->gb, 16, SEEK_SET);
326 ret = xan_unpack_luma(s, src,
327 s->buffer_size >> 1);
328 if (ret) {
329 av_log(avctx, AV_LOG_ERROR, "Luma decoding failed\n");
330 return ret;
331 }
332
333 ybuf = s->y_buffer;
334 for (i = 0; i < avctx->height; i++) {
335 last = (ybuf[0] + (*src++ << 1)) & 0x3F;
336 ybuf[0] = last;
337 for (j = 1; j < avctx->width - 1; j += 2) {
338 cur = (ybuf[j + 1] + (*src++ << 1)) & 0x3F;
339 ybuf[j] = (last + cur) >> 1;
340 ybuf[j+1] = cur;
341 last = cur;
342 }
343 ybuf[j] = last;
344 ybuf += avctx->width;
345 }
346
347 src = s->y_buffer;
348 ybuf = s->pic.data[0];
349 for (j = 0; j < avctx->height; j++) {
350 for (i = 0; i < avctx->width; i++)
351 ybuf[i] = (src[i] << 2) | (src[i] >> 3);
352 src += avctx->width;
353 ybuf += s->pic.linesize[0];
354 }
355
356 return 0;
357 }
358
359 static int xan_decode_frame(AVCodecContext *avctx,
360 void *data, int *got_frame,
361 AVPacket *avpkt)
362 {
363 XanContext *s = avctx->priv_data;
364 int ftype;
365 int ret;
366
367 s->pic.reference = 1;
368 s->pic.buffer_hints = FF_BUFFER_HINTS_VALID |
369 FF_BUFFER_HINTS_PRESERVE |
370 FF_BUFFER_HINTS_REUSABLE;
371 if ((ret = avctx->reget_buffer(avctx, &s->pic))) {
372 av_log(s->avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
373 return ret;
374 }
375
376 bytestream2_init(&s->gb, avpkt->data, avpkt->size);
377 ftype = bytestream2_get_le32(&s->gb);
378 switch (ftype) {
379 case 0:
380 ret = xan_decode_frame_type0(avctx);
381 break;
382 case 1:
383 ret = xan_decode_frame_type1(avctx);
384 break;
385 default:
386 av_log(avctx, AV_LOG_ERROR, "Unknown frame type %d\n", ftype);
387 return -1;
388 }
389 if (ret)
390 return ret;
391
392 *got_frame = 1;
393 *(AVFrame*)data = s->pic;
394
395 return avpkt->size;
396 }
397
398 static av_cold int xan_decode_end(AVCodecContext *avctx)
399 {
400 XanContext *s = avctx->priv_data;
401
402 if (s->pic.data[0])
403 avctx->release_buffer(avctx, &s->pic);
404
405 av_freep(&s->y_buffer);
406 av_freep(&s->scratch_buffer);
407
408 return 0;
409 }
410
411 AVCodec ff_xan_wc4_decoder = {
412 .name = "xan_wc4",
413 .type = AVMEDIA_TYPE_VIDEO,
414 .id = AV_CODEC_ID_XAN_WC4,
415 .priv_data_size = sizeof(XanContext),
416 .init = xan_decode_init,
417 .close = xan_decode_end,
418 .decode = xan_decode_frame,
419 .capabilities = CODEC_CAP_DR1,
420 .long_name = NULL_IF_CONFIG_SMALL("Wing Commander IV / Xxan"),
421 };