ffv1: Check memory allocations
[libav.git] / libavcodec / ffv1.c
1 /*
2 * FFV1 codec for libavcodec
3 *
4 * Copyright (c) 2003 Michael Niedermayer <michaelni@gmx.at>
5 *
6 * This file is part of Libav.
7 *
8 * Libav is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * Libav is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with Libav; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23 /**
24 * @file
25 * FF Video Codec 1 (a lossless codec)
26 */
27
28 #include "libavutil/attributes.h"
29 #include "libavutil/avassert.h"
30 #include "avcodec.h"
31 #include "get_bits.h"
32 #include "put_bits.h"
33 #include "rangecoder.h"
34 #include "golomb.h"
35 #include "mathops.h"
36 #include "ffv1.h"
37
38 const int8_t ffv1_quant5_10bit[256] = {
39 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1,
40 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
41 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
42 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
43 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
44 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
45 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
46 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
47 -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
48 -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
49 -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
50 -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
51 -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1,
52 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
53 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
54 -1, -1, -1, -1, -1, -1, -0, -0, -0, -0, -0, -0, -0, -0, -0, -0,
55 };
56
57 const int8_t ffv1_quant5[256] = {
58 0, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
59 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
60 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
61 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
62 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
63 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
64 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
65 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
66 -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
67 -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
68 -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
69 -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
70 -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
71 -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
72 -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
73 -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1, -1, -1,
74 };
75
76 const int8_t ffv1_quant9_10bit[256] = {
77 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2,
78 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3,
79 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
80 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4,
81 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
82 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
83 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
84 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
85 -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
86 -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
87 -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
88 -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
89 -4, -4, -4, -4, -4, -4, -4, -4, -4, -3, -3, -3, -3, -3, -3, -3,
90 -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3,
91 -3, -3, -3, -3, -3, -3, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
92 -2, -2, -2, -2, -1, -1, -1, -1, -1, -1, -1, -1, -0, -0, -0, -0,
93 };
94
95 const int8_t ffv1_quant11[256] = {
96 0, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4,
97 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
98 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
99 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
100 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
101 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
102 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
103 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
104 -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5,
105 -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5,
106 -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5,
107 -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5,
108 -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5,
109 -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -5, -4, -4,
110 -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4, -4,
111 -4, -4, -4, -4, -4, -3, -3, -3, -3, -3, -3, -3, -2, -2, -2, -1,
112 };
113
114 const uint8_t ffv1_ver2_state[256] = {
115 0, 10, 10, 10, 10, 16, 16, 16, 28, 16, 16, 29, 42, 49, 20, 49,
116 59, 25, 26, 26, 27, 31, 33, 33, 33, 34, 34, 37, 67, 38, 39, 39,
117 40, 40, 41, 79, 43, 44, 45, 45, 48, 48, 64, 50, 51, 52, 88, 52,
118 53, 74, 55, 57, 58, 58, 74, 60, 101, 61, 62, 84, 66, 66, 68, 69,
119 87, 82, 71, 97, 73, 73, 82, 75, 111, 77, 94, 78, 87, 81, 83, 97,
120 85, 83, 94, 86, 99, 89, 90, 99, 111, 92, 93, 134, 95, 98, 105, 98,
121 105, 110, 102, 108, 102, 118, 103, 106, 106, 113, 109, 112, 114, 112, 116, 125,
122 115, 116, 117, 117, 126, 119, 125, 121, 121, 123, 145, 124, 126, 131, 127, 129,
123 165, 130, 132, 138, 133, 135, 145, 136, 137, 139, 146, 141, 143, 142, 144, 148,
124 147, 155, 151, 149, 151, 150, 152, 157, 153, 154, 156, 168, 158, 162, 161, 160,
125 172, 163, 169, 164, 166, 184, 167, 170, 177, 174, 171, 173, 182, 176, 180, 178,
126 175, 189, 179, 181, 186, 183, 192, 185, 200, 187, 191, 188, 190, 197, 193, 196,
127 197, 194, 195, 196, 198, 202, 199, 201, 210, 203, 207, 204, 205, 206, 208, 214,
128 209, 211, 221, 212, 213, 215, 224, 216, 217, 218, 219, 220, 222, 228, 223, 225,
129 226, 224, 227, 229, 240, 230, 231, 232, 233, 234, 235, 236, 238, 239, 237, 242,
130 241, 243, 242, 244, 245, 246, 247, 248, 249, 250, 251, 252, 252, 253, 254, 255,
131 };
132
133
134 av_cold int ffv1_common_init(AVCodecContext *avctx)
135 {
136 FFV1Context *s = avctx->priv_data;
137
138 s->avctx = avctx;
139 s->flags = avctx->flags;
140
141 if (!avctx->width || !avctx->height)
142 return AVERROR_INVALIDDATA;
143
144 s->width = avctx->width;
145 s->height = avctx->height;
146
147 // defaults
148 s->num_h_slices = 1;
149 s->num_v_slices = 1;
150
151 return 0;
152 }
153
154 int ffv1_init_slice_state(FFV1Context *f, FFV1Context *fs)
155 {
156 int j;
157
158 fs->plane_count = f->plane_count;
159 fs->transparency = f->transparency;
160 for (j = 0; j < f->plane_count; j++) {
161 PlaneContext *const p = &fs->plane[j];
162
163 if (fs->ac) {
164 if (!p->state)
165 p->state = av_malloc(CONTEXT_SIZE * p->context_count *
166 sizeof(uint8_t));
167 if (!p->state)
168 return AVERROR(ENOMEM);
169 } else {
170 if (!p->vlc_state)
171 p->vlc_state = av_malloc(p->context_count * sizeof(VlcState));
172 if (!p->vlc_state)
173 return AVERROR(ENOMEM);
174 }
175 }
176
177 if (fs->ac > 1) {
178 //FIXME only redo if state_transition changed
179 for (j = 1; j < 256; j++) {
180 fs->c.one_state[j] = f->state_transition[j];
181 fs->c.zero_state[256 - j] = 256 - fs->c.one_state[j];
182 }
183 }
184
185 return 0;
186 }
187
188 av_cold int ffv1_init_slice_contexts(FFV1Context *f)
189 {
190 int i, j;
191
192 f->slice_count = f->num_h_slices * f->num_v_slices;
193 if (f->slice_count <= 0) {
194 av_log(f->avctx, AV_LOG_ERROR, "Invalid number of slices\n");
195 return AVERROR(EINVAL);
196 }
197
198 for (i = 0; i < f->slice_count; i++) {
199 int sx = i % f->num_h_slices;
200 int sy = i / f->num_h_slices;
201 int sxs = f->avctx->width * sx / f->num_h_slices;
202 int sxe = f->avctx->width * (sx + 1) / f->num_h_slices;
203 int sys = f->avctx->height * sy / f->num_v_slices;
204 int sye = f->avctx->height * (sy + 1) / f->num_v_slices;
205 FFV1Context *fs = av_mallocz(sizeof(*fs));
206 if (!fs)
207 goto memfail;
208
209 f->slice_context[i] = fs;
210 memcpy(fs, f, sizeof(*fs));
211 memset(fs->rc_stat2, 0, sizeof(fs->rc_stat2));
212
213 fs->slice_width = sxe - sxs;
214 fs->slice_height = sye - sys;
215 fs->slice_x = sxs;
216 fs->slice_y = sys;
217
218 fs->sample_buffer = av_malloc(3 * MAX_PLANES * (fs->width + 6) *
219 sizeof(*fs->sample_buffer));
220 if (!fs->sample_buffer) {
221 av_free(fs);
222 goto memfail;
223 }
224 }
225 return 0;
226
227 memfail:
228 for (j = 0; j < i; j++) {
229 av_free(&f->slice_context[j]->sample_buffer);
230 av_free(&f->slice_context[j]);
231 }
232 return AVERROR(ENOMEM);
233 }
234
235 int ffv1_allocate_initial_states(FFV1Context *f)
236 {
237 int i;
238
239 for (i = 0; i < f->quant_table_count; i++) {
240 f->initial_states[i] = av_malloc(f->context_count[i] *
241 sizeof(*f->initial_states[i]));
242 if (!f->initial_states[i])
243 return AVERROR(ENOMEM);
244 memset(f->initial_states[i], 128,
245 f->context_count[i] * sizeof(*f->initial_states[i]));
246 }
247 return 0;
248 }
249
250 void ffv1_clear_slice_state(FFV1Context *f, FFV1Context *fs)
251 {
252 int i, j;
253
254 for (i = 0; i < f->plane_count; i++) {
255 PlaneContext *p = &fs->plane[i];
256
257 p->interlace_bit_state[0] = 128;
258 p->interlace_bit_state[1] = 128;
259
260 if (fs->ac) {
261 if (f->initial_states[p->quant_table_index]) {
262 memcpy(p->state, f->initial_states[p->quant_table_index],
263 CONTEXT_SIZE * p->context_count);
264 } else
265 memset(p->state, 128, CONTEXT_SIZE * p->context_count);
266 } else {
267 for (j = 0; j < p->context_count; j++) {
268 p->vlc_state[j].drift = 0;
269 p->vlc_state[j].error_sum = 4; //FFMAX((RANGE + 32)/64, 2);
270 p->vlc_state[j].bias = 0;
271 p->vlc_state[j].count = 1;
272 }
273 }
274 }
275 }
276
277 av_cold int ffv1_close(AVCodecContext *avctx)
278 {
279 FFV1Context *s = avctx->priv_data;
280 int i, j;
281
282 for (j = 0; j < s->slice_count; j++) {
283 FFV1Context *fs = s->slice_context[j];
284 for (i = 0; i < s->plane_count; i++) {
285 PlaneContext *p = &fs->plane[i];
286
287 av_freep(&p->state);
288 av_freep(&p->vlc_state);
289 }
290 av_freep(&fs->sample_buffer);
291 }
292
293 av_freep(&avctx->stats_out);
294 for (j = 0; j < s->quant_table_count; j++) {
295 av_freep(&s->initial_states[j]);
296 for (i = 0; i < s->slice_count; i++) {
297 FFV1Context *sf = s->slice_context[i];
298 av_freep(&sf->rc_stat2[j]);
299 }
300 av_freep(&s->rc_stat2[j]);
301 }
302
303 for (i = 0; i < s->slice_count; i++)
304 av_freep(&s->slice_context[i]);
305
306 return 0;
307 }