indeo2: Fix banding artefacts
[libav.git] / libavcodec / indeo2.c
1 /*
2 * Intel Indeo 2 codec
3 * Copyright (c) 2005 Konstantin Shishkov
4 *
5 * This file is part of Libav.
6 *
7 * Libav is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * Libav is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with Libav; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22 /**
23 * @file
24 * Intel Indeo 2 decoder.
25 */
26
27 #define BITSTREAM_READER_LE
28 #include "libavutil/attributes.h"
29 #include "avcodec.h"
30 #include "get_bits.h"
31 #include "indeo2data.h"
32 #include "internal.h"
33 #include "mathops.h"
34
35 typedef struct Ir2Context{
36 AVCodecContext *avctx;
37 AVFrame *picture;
38 GetBitContext gb;
39 int decode_delta;
40 } Ir2Context;
41
42 #define CODE_VLC_BITS 14
43 static VLC ir2_vlc;
44
45 /* Indeo 2 codes are in range 0x01..0x7F and 0x81..0x90 */
46 static inline int ir2_get_code(GetBitContext *gb)
47 {
48 return get_vlc2(gb, ir2_vlc.table, CODE_VLC_BITS, 1) + 1;
49 }
50
51 static int ir2_decode_plane(Ir2Context *ctx, int width, int height, uint8_t *dst,
52 int pitch, const uint8_t *table)
53 {
54 int i;
55 int j;
56 int out = 0;
57 int c;
58 int t;
59
60 if (width & 1)
61 return AVERROR_INVALIDDATA;
62
63 /* first line contain absolute values, other lines contain deltas */
64 while (out < width) {
65 c = ir2_get_code(&ctx->gb);
66 if (c >= 0x80) { /* we have a run */
67 c -= 0x7F;
68 if (out + c*2 > width)
69 return AVERROR_INVALIDDATA;
70 for (i = 0; i < c * 2; i++)
71 dst[out++] = 0x80;
72 } else { /* copy two values from table */
73 dst[out++] = table[c * 2];
74 dst[out++] = table[(c * 2) + 1];
75 }
76 }
77 dst += pitch;
78
79 for (j = 1; j < height; j++) {
80 out = 0;
81 while (out < width) {
82 c = ir2_get_code(&ctx->gb);
83 if (c >= 0x80) { /* we have a skip */
84 c -= 0x7F;
85 if (out + c*2 > width)
86 return AVERROR_INVALIDDATA;
87 for (i = 0; i < c * 2; i++) {
88 dst[out] = dst[out - pitch];
89 out++;
90 }
91 } else { /* add two deltas from table */
92 t = dst[out - pitch] + (table[c * 2] - 128);
93 t = av_clip_uint8(t);
94 dst[out] = t;
95 out++;
96 t = dst[out - pitch] + (table[(c * 2) + 1] - 128);
97 t = av_clip_uint8(t);
98 dst[out] = t;
99 out++;
100 }
101 }
102 dst += pitch;
103 }
104 return 0;
105 }
106
107 static int ir2_decode_plane_inter(Ir2Context *ctx, int width, int height, uint8_t *dst,
108 int pitch, const uint8_t *table)
109 {
110 int j;
111 int out = 0;
112 int c;
113 int t;
114
115 if (width & 1)
116 return AVERROR_INVALIDDATA;
117
118 for (j = 0; j < height; j++) {
119 out = 0;
120 while (out < width) {
121 c = ir2_get_code(&ctx->gb);
122 if (c >= 0x80) { /* we have a skip */
123 c -= 0x7F;
124 out += c * 2;
125 } else { /* add two deltas from table */
126 t = dst[out] + (((table[c * 2] - 128)*3) >> 2);
127 t = av_clip_uint8(t);
128 dst[out] = t;
129 out++;
130 t = dst[out] + (((table[(c * 2) + 1] - 128)*3) >> 2);
131 t = av_clip_uint8(t);
132 dst[out] = t;
133 out++;
134 }
135 }
136 dst += pitch;
137 }
138 return 0;
139 }
140
141 static int ir2_decode_frame(AVCodecContext *avctx,
142 void *data, int *got_frame,
143 AVPacket *avpkt)
144 {
145 Ir2Context * const s = avctx->priv_data;
146 const uint8_t *buf = avpkt->data;
147 int buf_size = avpkt->size;
148 AVFrame *picture = data;
149 AVFrame * const p = s->picture;
150 int start, ret;
151 int ltab, ctab;
152
153 if ((ret = ff_reget_buffer(avctx, p)) < 0) {
154 av_log(s->avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
155 return ret;
156 }
157
158 start = 48; /* hardcoded for now */
159
160 if (start >= buf_size) {
161 av_log(s->avctx, AV_LOG_ERROR, "input buffer size too small (%d)\n", buf_size);
162 return AVERROR_INVALIDDATA;
163 }
164
165 s->decode_delta = buf[18];
166
167 /* decide whether frame uses deltas or not */
168 #ifndef BITSTREAM_READER_LE
169 for (i = 0; i < buf_size; i++)
170 buf[i] = ff_reverse[buf[i]];
171 #endif
172
173 init_get_bits(&s->gb, buf + start, (buf_size - start) * 8);
174
175 ltab = buf[0x22] & 3;
176 ctab = buf[0x22] >> 2;
177 if (s->decode_delta) { /* intraframe */
178 if ((ret = ir2_decode_plane(s, avctx->width, avctx->height,
179 p->data[0], p->linesize[0],
180 ir2_delta_table[ltab])) < 0)
181 return ret;
182
183 /* swapped U and V */
184 if ((ret = ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2,
185 p->data[2], p->linesize[2],
186 ir2_delta_table[ctab])) < 0)
187 return ret;
188 if ((ret = ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2,
189 p->data[1], p->linesize[1],
190 ir2_delta_table[ctab])) < 0)
191 return ret;
192 } else { /* interframe */
193 if ((ret = ir2_decode_plane_inter(s, avctx->width, avctx->height,
194 p->data[0], p->linesize[0],
195 ir2_delta_table[ltab])) < 0)
196 return ret;
197 /* swapped U and V */
198 if ((ret = ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2,
199 p->data[2], p->linesize[2],
200 ir2_delta_table[ctab])) < 0)
201 return ret;
202 if ((ret = ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2,
203 p->data[1], p->linesize[1],
204 ir2_delta_table[ctab])) < 0)
205 return ret;
206 }
207
208 if ((ret = av_frame_ref(picture, p)) < 0)
209 return ret;
210
211 *got_frame = 1;
212
213 return buf_size;
214 }
215
216 static av_cold int ir2_decode_init(AVCodecContext *avctx)
217 {
218 Ir2Context * const ic = avctx->priv_data;
219 static VLC_TYPE vlc_tables[1 << CODE_VLC_BITS][2];
220
221 ic->avctx = avctx;
222
223 avctx->pix_fmt= AV_PIX_FMT_YUV410P;
224
225 ic->picture = av_frame_alloc();
226 if (!ic->picture)
227 return AVERROR(ENOMEM);
228
229 ir2_vlc.table = vlc_tables;
230 ir2_vlc.table_allocated = 1 << CODE_VLC_BITS;
231 #ifdef BITSTREAM_READER_LE
232 init_vlc(&ir2_vlc, CODE_VLC_BITS, IR2_CODES,
233 &ir2_codes[0][1], 4, 2,
234 &ir2_codes[0][0], 4, 2, INIT_VLC_USE_NEW_STATIC | INIT_VLC_LE);
235 #else
236 init_vlc(&ir2_vlc, CODE_VLC_BITS, IR2_CODES,
237 &ir2_codes[0][1], 4, 2,
238 &ir2_codes[0][0], 4, 2, INIT_VLC_USE_NEW_STATIC);
239 #endif
240
241 return 0;
242 }
243
244 static av_cold int ir2_decode_end(AVCodecContext *avctx)
245 {
246 Ir2Context * const ic = avctx->priv_data;
247
248 av_frame_free(&ic->picture);
249
250 return 0;
251 }
252
253 AVCodec ff_indeo2_decoder = {
254 .name = "indeo2",
255 .long_name = NULL_IF_CONFIG_SMALL("Intel Indeo 2"),
256 .type = AVMEDIA_TYPE_VIDEO,
257 .id = AV_CODEC_ID_INDEO2,
258 .priv_data_size = sizeof(Ir2Context),
259 .init = ir2_decode_init,
260 .close = ir2_decode_end,
261 .decode = ir2_decode_frame,
262 .capabilities = AV_CODEC_CAP_DR1,
263 };