dxtory: change error code for unexpected slice configuration
[libav.git] / libavcodec / dxtory.c
1 /*
2 * Dxtory decoder
3 *
4 * Copyright (c) 2011 Konstantin Shishkov
5 *
6 * This file is part of Libav.
7 *
8 * Libav is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * Libav is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with Libav; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23 #define BITSTREAM_READER_LE
24 #include "avcodec.h"
25 #include "bytestream.h"
26 #include "get_bits.h"
27 #include "internal.h"
28 #include "unary.h"
29 #include "libavutil/common.h"
30 #include "libavutil/intreadwrite.h"
31
32 static int dxtory_decode_v1(AVCodecContext *avctx, AVFrame *pic,
33 const uint8_t *src, int src_size)
34 {
35 int h, w;
36 uint8_t *Y1, *Y2, *U, *V;
37 int ret;
38
39 if (src_size < avctx->width * avctx->height * 3 / 2) {
40 av_log(avctx, AV_LOG_ERROR, "packet too small\n");
41 return AVERROR_INVALIDDATA;
42 }
43
44 avctx->pix_fmt = AV_PIX_FMT_YUV420P;
45 if ((ret = ff_get_buffer(avctx, pic, 0)) < 0)
46 return ret;
47
48 Y1 = pic->data[0];
49 Y2 = pic->data[0] + pic->linesize[0];
50 U = pic->data[1];
51 V = pic->data[2];
52 for (h = 0; h < avctx->height; h += 2) {
53 for (w = 0; w < avctx->width; w += 2) {
54 AV_COPY16(Y1 + w, src);
55 AV_COPY16(Y2 + w, src + 2);
56 U[w >> 1] = src[4] + 0x80;
57 V[w >> 1] = src[5] + 0x80;
58 src += 6;
59 }
60 Y1 += pic->linesize[0] << 1;
61 Y2 += pic->linesize[0] << 1;
62 U += pic->linesize[1];
63 V += pic->linesize[2];
64 }
65
66 return 0;
67 }
68
69 const uint8_t def_lru[8] = { 0x00, 0x20, 0x40, 0x60, 0x80, 0xA0, 0xC0, 0xFF };
70
71 static inline uint8_t decode_sym(GetBitContext *gb, uint8_t lru[8])
72 {
73 uint8_t c, val;
74
75 c = get_unary(gb, 0, 8);
76 if (!c) {
77 val = get_bits(gb, 8);
78 memmove(lru + 1, lru, sizeof(*lru) * (8 - 1));
79 } else {
80 val = lru[c - 1];
81 memmove(lru + 1, lru, sizeof(*lru) * (c - 1));
82 }
83 lru[0] = val;
84
85 return val;
86 }
87
88 static int dx2_decode_slice(GetBitContext *gb, int width, int height,
89 uint8_t *Y, uint8_t *U, uint8_t *V,
90 int ystride, int ustride, int vstride)
91 {
92 int x, y, i;
93 uint8_t lru[3][8];
94
95 for (i = 0; i < 3; i++)
96 memcpy(lru[i], def_lru, 8 * sizeof(*def_lru));
97
98 for (y = 0; y < height; y+=2) {
99 for (x = 0; x < width; x += 2) {
100 Y[x + 0 + 0 * ystride] = decode_sym(gb, lru[0]);
101 Y[x + 1 + 0 * ystride] = decode_sym(gb, lru[0]);
102 Y[x + 0 + 1 * ystride] = decode_sym(gb, lru[0]);
103 Y[x + 1 + 1 * ystride] = decode_sym(gb, lru[0]);
104 U[x >> 1] = decode_sym(gb, lru[1]) ^ 0x80;
105 V[x >> 1] = decode_sym(gb, lru[2]) ^ 0x80;
106 }
107
108 Y += ystride << 1;
109 U += ustride;
110 V += vstride;
111 }
112
113 return 0;
114 }
115
116 static int dxtory_decode_v2(AVCodecContext *avctx, AVFrame *pic,
117 const uint8_t *src, int src_size)
118 {
119 GetByteContext gb;
120 GetBitContext gb2;
121 int nslices, slice, slice_height, ref_slice_height;
122 int cur_y, next_y;
123 uint32_t off, slice_size;
124 uint8_t *Y, *U, *V;
125 int ret;
126
127 bytestream2_init(&gb, src, src_size);
128 nslices = bytestream2_get_le16(&gb);
129 off = FFALIGN(nslices * 4 + 2, 16);
130 if (src_size < off) {
131 av_log(avctx, AV_LOG_ERROR, "no slice data\n");
132 return AVERROR_INVALIDDATA;
133 }
134
135 if (!nslices || avctx->height % nslices) {
136 avpriv_request_sample(avctx, "%d slices for %dx%d", nslices,
137 avctx->width, avctx->height);
138 return AVERROR_PATCHWELCOME;
139 }
140
141 ref_slice_height = avctx->height / nslices;
142 if ((avctx->width & 1) || (avctx->height & 1)) {
143 avpriv_request_sample(avctx, "Frame dimensions %dx%d",
144 avctx->width, avctx->height);
145 }
146
147 avctx->pix_fmt = AV_PIX_FMT_YUV420P;
148 if ((ret = ff_get_buffer(avctx, pic, 0)) < 0)
149 return ret;
150
151 Y = pic->data[0];
152 U = pic->data[1];
153 V = pic->data[2];
154
155 cur_y = 0;
156 next_y = ref_slice_height;
157 for (slice = 0; slice < nslices; slice++) {
158 slice_size = bytestream2_get_le32(&gb);
159 slice_height = (next_y & ~1) - (cur_y & ~1);
160 if (slice_size > src_size - off) {
161 av_log(avctx, AV_LOG_ERROR,
162 "invalid slice size %d (only %d bytes left)\n",
163 slice_size, src_size - off);
164 return AVERROR_INVALIDDATA;
165 }
166 if (slice_size <= 16) {
167 av_log(avctx, AV_LOG_ERROR, "invalid slice size %d\n", slice_size);
168 return AVERROR_INVALIDDATA;
169 }
170
171 if (AV_RL32(src + off) != slice_size - 16) {
172 av_log(avctx, AV_LOG_ERROR,
173 "Slice sizes mismatch: got %d instead of %d\n",
174 AV_RL32(src + off), slice_size - 16);
175 }
176 init_get_bits(&gb2, src + off + 16, (slice_size - 16) * 8);
177 dx2_decode_slice(&gb2, avctx->width, slice_height, Y, U, V,
178 pic->linesize[0], pic->linesize[1], pic->linesize[2]);
179
180 Y += pic->linesize[0] * slice_height;
181 U += pic->linesize[1] * (slice_height >> 1);
182 V += pic->linesize[2] * (slice_height >> 1);
183 off += slice_size;
184 cur_y = next_y;
185 next_y += ref_slice_height;
186 }
187
188 return 0;
189 }
190
191 static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
192 AVPacket *avpkt)
193 {
194 AVFrame *pic = data;
195 const uint8_t *src = avpkt->data;
196 int ret;
197
198 if (avpkt->size < 16) {
199 av_log(avctx, AV_LOG_ERROR, "packet too small\n");
200 return AVERROR_INVALIDDATA;
201 }
202
203 switch (AV_RB32(src)) {
204 case 0x02000001:
205 ret = dxtory_decode_v1(avctx, pic, src + 16, avpkt->size - 16);
206 break;
207 case 0x02000009:
208 ret = dxtory_decode_v2(avctx, pic, src + 16, avpkt->size - 16);
209 break;
210 default:
211 avpriv_request_sample(avctx, "Frame header %X", AV_RB32(src));
212 return AVERROR_PATCHWELCOME;
213 }
214
215 if (ret)
216 return ret;
217
218 pic->pict_type = AV_PICTURE_TYPE_I;
219 pic->key_frame = 1;
220 *got_frame = 1;
221
222 return avpkt->size;
223 }
224
225 AVCodec ff_dxtory_decoder = {
226 .name = "dxtory",
227 .long_name = NULL_IF_CONFIG_SMALL("Dxtory"),
228 .type = AVMEDIA_TYPE_VIDEO,
229 .id = AV_CODEC_ID_DXTORY,
230 .decode = decode_frame,
231 .capabilities = CODEC_CAP_DR1,
232 };