dont return -1 if mpeg4 scalability ...
[libav.git] / libavcodec / mpegvideo.h
CommitLineData
de6d9b64
FB
1/*
2 * Generic DCT based hybrid video encoder
3 * Copyright (c) 2000,2001 Gerard Lantau.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 */
19
20/* Macros for picture code type. */
21#define I_TYPE 1
22#define P_TYPE 2
23#define B_TYPE 3
cc9ba006 24#define S_TYPE 4 //S(GMC)-VOP MPEG4
de6d9b64
FB
25
26enum OutputFormat {
27 FMT_MPEG1,
28 FMT_H263,
29 FMT_MJPEG,
30};
31
32#define MPEG_BUF_SIZE (16 * 1024)
33
2f349de2
MN
34#define QMAT_SHIFT_MMX 19
35#define QMAT_SHIFT 25
36
45870f57
MN
37#define MAX_FCODE 7
38#define MAX_MV 2048
39
9cdd6a24
MN
40typedef struct Predictor{
41 double coeff;
42 double count;
43 double decay;
44} Predictor;
45
de6d9b64 46typedef struct MpegEncContext {
58f26ba9 47 struct AVCodecContext *avctx;
de6d9b64
FB
48 /* the following parameters must be initialized before encoding */
49 int width, height; /* picture size. must be a multiple of 16 */
50 int gop_size;
51 int frame_rate; /* number of frames per second */
52 int intra_only; /* if true, only intra pictures are generated */
53 int bit_rate; /* wanted bit rate */
9cdd6a24 54 int bit_rate_tolerance; /* amount of +- bits (>0)*/
de6d9b64
FB
55 enum OutputFormat out_format; /* output format */
56 int h263_plus; /* h263 plus headers */
57 int h263_rv10; /* use RV10 variation for H263 */
58f26ba9 58 int h263_pred; /* use mpeg4/h263 ac/dc predictions */
de6d9b64
FB
59 int h263_msmpeg4; /* generate MSMPEG4 compatible stream */
60 int h263_intel; /* use I263 intel h263 header */
61 int fixed_qscale; /* fixed qscale if non zero */
9cdd6a24
MN
62 float qcompress; /* amount of qscale change between easy & hard scenes (0.0-1.0) */
63 float qblur; /* amount of qscale smoothing over time (0.0-1.0) */
64 int qmin; /* min qscale */
65 int qmax; /* max qscale */
66 int max_qdiff; /* max qscale difference between frames */
3bb4e23a 67 int encoding; /* true if we are encoding (vs decoding) */
de6d9b64
FB
68 /* the following fields are managed internally by the encoder */
69
70 /* bit output */
71 PutBitContext pb;
72
73 /* sequence parameters */
74 int context_initialized;
75 int picture_number;
76 int fake_picture_number; /* picture number at the bitstream frame rate */
45870f57
MN
77 int gop_picture_number; /* index of the first picture of a GOP based on fake_pic_num & mpeg1 specific */
78 int picture_in_gop_number; /* 0-> first pic in gop, ... */
de6d9b64 79 int mb_width, mb_height;
e03c341e 80 int mb_num; /* number of MBs of a picture */
de6d9b64
FB
81 int linesize; /* line size, in bytes, may be different from width */
82 UINT8 *new_picture[3]; /* picture to be compressed */
83 UINT8 *last_picture[3]; /* previous picture */
84 UINT8 *last_picture_base[3]; /* real start of the picture */
85 UINT8 *next_picture[3]; /* previous picture (for bidir pred) */
86 UINT8 *next_picture_base[3]; /* real start of the picture */
87 UINT8 *aux_picture[3]; /* aux picture (for B frames only) */
88 UINT8 *aux_picture_base[3]; /* real start of the picture */
89 UINT8 *current_picture[3]; /* buffer to store the decompressed current picture */
90 int last_dc[3]; /* last DC values for MPEG1 */
4278e7a6 91 INT16 *dc_val[3]; /* used for mpeg4 DC prediction, all 3 arrays must be continuous */
de6d9b64
FB
92 int y_dc_scale, c_dc_scale;
93 UINT8 *coded_block; /* used for coded block pattern prediction */
4278e7a6 94 INT16 (*ac_val[3])[16]; /* used for for mpeg4 AC prediction, all 3 arrays must be continuous */
de6d9b64 95 int ac_pred;
3bb4e23a
FB
96 int mb_skiped; /* MUST BE SET only during DECODING */
97 UINT8 *mbskip_table; /* used to avoid copy if macroblock
98 skipped (for black regions for example) */
bff6ecaa 99 UINT8 *mbintra_table; /* used to kill a few memsets */
de6d9b64
FB
100
101 int qscale;
102 int pict_type;
6f91bcd1 103 int last_non_b_pict_type; /* used for mpeg4 gmc b-frames */
9cdd6a24 104 int last_pict_type; /* used for bit rate stuff (needs that to update the right predictor) */
de6d9b64
FB
105 int frame_rate_index;
106 /* motion compensation */
107 int unrestricted_mv;
108 int h263_long_vectors; /* use horrible h263v1 long vector mode */
109
110 int f_code; /* resolution */
6f91bcd1
MN
111 int b_code; /* backward resolution for B Frames (mpeg4) */
112 INT16 *mv_table[2]; /* MV table (1MV per MB)*/
113 INT16 (*motion_val)[2]; /* used for MV prediction (4MV per MB)*/
de6d9b64
FB
114 int full_search;
115 int mv_dir;
116#define MV_DIR_BACKWARD 1
117#define MV_DIR_FORWARD 2
6f91bcd1 118#define MV_DIRECT 4 // bidirectional mode where the difference equals the MV of the last P/S/I-Frame (mpeg4)
de6d9b64
FB
119 int mv_type;
120#define MV_TYPE_16X16 0 /* 1 vector for the whole mb */
45870f57 121#define MV_TYPE_8X8 1 /* 4 vectors (h263, mpeg4 4MV) */
de6d9b64
FB
122#define MV_TYPE_16X8 2 /* 2 vectors, one per 16x8 block */
123#define MV_TYPE_FIELD 3 /* 2 vectors, one per field */
124#define MV_TYPE_DMV 4 /* 2 vectors, special mpeg2 Dual Prime Vectors */
125 /* motion vectors for a macroblock
126 first coordinate : 0 = forward 1 = backward
127 second " : depend on type
128 third " : 0 = x, 1 = y
129 */
130 int mv[2][4][2];
131 int field_select[2][2];
132 int last_mv[2][2][2];
45870f57
MN
133 UINT16 (*mv_penalty)[MAX_MV*2+1]; /* amount of bits needed to encode a MV, used for ME */
134 UINT8 *fcode_tab; /* smallest fcode needed for each MV */
de6d9b64
FB
135
136 int has_b_frames;
137 int no_rounding; /* apply no rounding to motion estimation (MPEG4) */
138
139 /* macroblock layer */
140 int mb_x, mb_y;
141 int mb_incr;
142 int mb_intra;
b4dbd87c 143 INT16 *mb_var; /* Table for MB variances */
37fbfd0a 144 char *mb_type; /* Table for MB type */
4278e7a6
MN
145
146 int block_index[6];
147 int block_wrap[6];
148
de6d9b64
FB
149 /* matrix transmitted in the bitstream */
150 UINT16 intra_matrix[64];
151 UINT16 chroma_intra_matrix[64];
152 UINT16 non_intra_matrix[64];
153 UINT16 chroma_non_intra_matrix[64];
154 /* precomputed matrix (combine qscale and DCT renorm) */
155 int q_intra_matrix[64];
156 int q_non_intra_matrix[64];
2f349de2
MN
157 /* identical to the above but for MMX & these are not permutated */
158 UINT16 __align8 q_intra_matrix16[64] ;
159 UINT16 __align8 q_non_intra_matrix16[64];
de6d9b64
FB
160 int block_last_index[6]; /* last non zero coefficient in block */
161
162 void *opaque; /* private data for the user */
163
164 /* bit rate control */
165 int I_frame_bits; /* wanted number of bits per I frame */
166 int P_frame_bits; /* same for P frame */
e03c341e 167 int avg_mb_var; /* average MB variance for current frame */
9cdd6a24
MN
168 int mc_mb_var; /* motion compensated MB variance for current frame */
169 int last_mc_mb_var; /* motion compensated MB variance for last frame */
1a565432
FB
170 INT64 wanted_bits;
171 INT64 total_bits;
9cdd6a24
MN
172 int frame_bits; /* bits used for the current frame */
173 int last_frame_bits; /* bits used for the last frame */
174 Predictor i_pred;
175 Predictor p_pred;
176 double qsum; /* sum of qscales */
177 double qcount; /* count of qscales */
178 double short_term_qsum; /* sum of recent qscales */
179 double short_term_qcount; /* count of recent qscales */
180
102d3908
J
181 /* H.263 specific */
182 int gob_number;
644d98a4 183 int gob_index;
102d3908 184 int first_gob_line;
d140623f 185
544286b3
J
186 /* H.263+ specific */
187 int umvplus;
188 int umvplus_dec;
d140623f
J
189 int h263_aic; /* Advanded INTRA Coding (AIC) */
190 int h263_aic_dir; /* AIC direction: 0 = left, 1 = top */
544286b3 191
de6d9b64 192 /* mpeg4 specific */
6f91bcd1 193 int time_increment_resolution;
de6d9b64 194 int time_increment_bits;
6f91bcd1
MN
195 int time_increment;
196 int time_base;
197 int time;
198 int last_non_b_time[2];
1a565432
FB
199 int shape;
200 int vol_sprite_usage;
cc9ba006
MN
201 int sprite_width;
202 int sprite_height;
203 int sprite_left;
204 int sprite_top;
205 int sprite_brightness_change;
73c8e514 206 int num_sprite_warping_points;
44eb4951 207 int real_sprite_warping_points;
73c8e514
MN
208 int sprite_offset[2][2];
209 int sprite_delta[2][2][2];
210 int sprite_shift[2][2];
211 int mcsel;
1a565432 212 int quant_precision;
cc9ba006
MN
213 int quarter_sample;
214 int scalability;
215 int new_pred;
216 int reduced_res_vop;
217 int aspect_ratio_info;
218 int sprite_warping_accuracy;
219 int low_latency_sprite;
220 int data_partioning;
44eb4951
MN
221
222 /* divx specific, used to workaround (many) bugs in divx5 */
223 int divx_version;
224 int divx_build;
225
de6d9b64
FB
226 /* RV10 specific */
227 int rv10_version; /* RV10 version: 0 or 3 */
228 int rv10_first_dc_coded[3];
229
230 /* MJPEG specific */
231 struct MJpegContext *mjpeg_ctx;
80e103d0
A
232 int mjpeg_vsample[3]; /* vertical sampling factors, default = {2, 1, 1} */
233 int mjpeg_hsample[3]; /* horizontal sampling factors, default = {2, 1, 1} */
234 int mjpeg_write_tables; /* do we want to have quantisation- and
235 huffmantables in the jpeg file ? */
de6d9b64
FB
236
237 /* MSMPEG4 specific */
238 int mv_table_index;
239 int rl_table_index;
240 int rl_chroma_table_index;
241 int dc_table_index;
242 int use_skip_mb_code;
243 int slice_height; /* in macroblocks */
244 int first_slice_line;
ae40484c
MN
245 int flipflop_rounding;
246 int bitrate;
de6d9b64
FB
247 /* decompression specific */
248 GetBitContext gb;
249
250 /* MPEG2 specific - I wish I had not to support this mess. */
251 int progressive_sequence;
252 int mpeg_f_code[2][2];
253 int picture_structure;
254/* picture type */
255#define PICT_TOP_FIELD 1
256#define PICT_BOTTOM_FIELD 2
257#define PICT_FRAME 3
258
259 int intra_dc_precision;
260 int frame_pred_frame_dct;
261 int top_field_first;
262 int concealment_motion_vectors;
263 int q_scale_type;
264 int intra_vlc_format;
265 int alternate_scan;
266 int repeat_first_field;
267 int chroma_420_type;
268 int progressive_frame;
269 int mpeg2;
270 int full_pel[2];
271 int interlaced_dct;
272 int last_qscale;
273 int first_slice;
644d98a4
J
274
275 /* RTP specific */
81401c1f 276 /* These are explained on avcodec.h */
644d98a4
J
277 int rtp_mode;
278 int rtp_payload_size;
81401c1f 279 void (*rtp_callback)(void *data, int size, int packet_number);
644d98a4
J
280 UINT8 *ptr_lastgob;
281 UINT8 *ptr_last_mb_line;
282 UINT32 mb_line_avgsize;
283
21af69f7
FB
284 DCTELEM block[6][64] __align8;
285 void (*dct_unquantize)(struct MpegEncContext *s,
286 DCTELEM *block, int n, int qscale);
de6d9b64
FB
287} MpegEncContext;
288
de6d9b64
FB
289int MPV_common_init(MpegEncContext *s);
290void MPV_common_end(MpegEncContext *s);
291void MPV_decode_mb(MpegEncContext *s, DCTELEM block[6][64]);
292void MPV_frame_start(MpegEncContext *s);
293void MPV_frame_end(MpegEncContext *s);
21af69f7
FB
294#ifdef HAVE_MMX
295void MPV_common_init_mmx(MpegEncContext *s);
296#endif
de6d9b64
FB
297
298/* motion_est.c */
299
300int estimate_motion(MpegEncContext *s,
301 int mb_x, int mb_y,
302 int *mx_ptr, int *my_ptr);
303
304/* mpeg12.c */
3c054a1c
FB
305extern INT16 default_intra_matrix[64];
306extern INT16 default_non_intra_matrix[64];
de6d9b64
FB
307
308void mpeg1_encode_picture_header(MpegEncContext *s, int picture_number);
309void mpeg1_encode_mb(MpegEncContext *s,
310 DCTELEM block[6][64],
311 int motion_x, int motion_y);
11ce8834 312void mpeg1_encode_init(MpegEncContext *s);
de6d9b64
FB
313
314/* h263enc.c */
315
316/* run length table */
317#define MAX_RUN 64
318#define MAX_LEVEL 64
319
320typedef struct RLTable {
321 int n; /* number of entries of table_vlc minus 1 */
322 int last; /* number of values for last = 0 */
323 const UINT16 (*table_vlc)[2];
324 const INT8 *table_run;
325 const INT8 *table_level;
326 UINT8 *index_run[2]; /* encoding only */
327 INT8 *max_level[2]; /* encoding & decoding */
328 INT8 *max_run[2]; /* encoding & decoding */
329 VLC vlc; /* decoding only */
330} RLTable;
331
332void init_rl(RLTable *rl);
333void init_vlc_rl(RLTable *rl);
334
3232bc88 335static inline int get_rl_index(const RLTable *rl, int last, int run, int level)
de6d9b64
FB
336{
337 int index;
338 index = rl->index_run[last][run];
339 if (index >= rl->n)
340 return rl->n;
341 if (level > rl->max_level[last][run])
342 return rl->n;
343 return index + level - 1;
344}
345
346void h263_encode_mb(MpegEncContext *s,
347 DCTELEM block[6][64],
348 int motion_x, int motion_y);
d6231b9e
MN
349void mpeg4_encode_mb(MpegEncContext *s,
350 DCTELEM block[6][64],
351 int motion_x, int motion_y);
de6d9b64 352void h263_encode_picture_header(MpegEncContext *s, int picture_number);
644d98a4 353int h263_encode_gob_header(MpegEncContext * s, int mb_line);
de6d9b64
FB
354void h263_dc_scale(MpegEncContext *s);
355INT16 *h263_pred_motion(MpegEncContext * s, int block,
356 int *px, int *py);
357void mpeg4_pred_ac(MpegEncContext * s, INT16 *block, int n,
358 int dir);
359void mpeg4_encode_picture_header(MpegEncContext *s, int picture_number);
45870f57 360void h263_encode_init(MpegEncContext *s);
de6d9b64
FB
361
362void h263_decode_init_vlc(MpegEncContext *s);
363int h263_decode_picture_header(MpegEncContext *s);
644d98a4 364int h263_decode_gob_header(MpegEncContext *s);
de6d9b64
FB
365int mpeg4_decode_picture_header(MpegEncContext * s);
366int intel_h263_decode_picture_header(MpegEncContext *s);
367int h263_decode_mb(MpegEncContext *s,
368 DCTELEM block[6][64]);
369int h263_get_picture_format(int width, int height);
de6d9b64
FB
370
371/* rv10.c */
372void rv10_encode_picture_header(MpegEncContext *s, int picture_number);
373int rv_decode_dc(MpegEncContext *s, int n);
374
375/* msmpeg4.c */
376void msmpeg4_encode_picture_header(MpegEncContext * s, int picture_number);
ae40484c 377void msmpeg4_encode_ext_header(MpegEncContext * s);
de6d9b64
FB
378void msmpeg4_encode_mb(MpegEncContext * s,
379 DCTELEM block[6][64],
380 int motion_x, int motion_y);
381void msmpeg4_dc_scale(MpegEncContext * s);
382int msmpeg4_decode_picture_header(MpegEncContext * s);
ae40484c 383int msmpeg4_decode_ext_header(MpegEncContext * s, int buf_size);
de6d9b64
FB
384int msmpeg4_decode_mb(MpegEncContext *s,
385 DCTELEM block[6][64]);
386int msmpeg4_decode_init_vlc(MpegEncContext *s);
387
388/* mjpegenc.c */
389
390int mjpeg_init(MpegEncContext *s);
391void mjpeg_close(MpegEncContext *s);
392void mjpeg_encode_mb(MpegEncContext *s,
393 DCTELEM block[6][64]);
394void mjpeg_picture_header(MpegEncContext *s);
395void mjpeg_picture_trailer(MpegEncContext *s);