h264_metadata: Add option to delete filler data
[libav.git] / libavfilter / vf_deinterlace_qsv.c
1 /*
2 * This file is part of Libav.
3 *
4 * Libav is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * Libav is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with Libav; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19 /**
20 * @file
21 * deinterlace video filter - QSV
22 */
23
24 #include <mfx/mfxvideo.h>
25
26 #include <stdio.h>
27 #include <string.h>
28
29 #include "libavutil/avstring.h"
30 #include "libavutil/common.h"
31 #include "libavutil/hwcontext.h"
32 #include "libavutil/hwcontext_qsv.h"
33 #include "libavutil/internal.h"
34 #include "libavutil/mathematics.h"
35 #include "libavutil/opt.h"
36 #include "libavutil/pixdesc.h"
37 #include "libavutil/time.h"
38
39 #include "avfilter.h"
40 #include "formats.h"
41 #include "internal.h"
42 #include "video.h"
43
44 enum {
45 QSVDEINT_MORE_OUTPUT = 1,
46 QSVDEINT_MORE_INPUT,
47 };
48
49 typedef struct QSVFrame {
50 AVFrame *frame;
51 mfxFrameSurface1 surface;
52 int used;
53
54 struct QSVFrame *next;
55 } QSVFrame;
56
57 typedef struct QSVDeintContext {
58 const AVClass *class;
59
60 AVBufferRef *hw_frames_ctx;
61 /* a clone of the main session, used internally for deinterlacing */
62 mfxSession session;
63
64 mfxMemId *mem_ids;
65 int nb_mem_ids;
66
67 mfxFrameSurface1 **surface_ptrs;
68 int nb_surface_ptrs;
69
70 mfxExtOpaqueSurfaceAlloc opaque_alloc;
71 mfxExtVPPDeinterlacing deint_conf;
72 mfxExtBuffer *ext_buffers[2];
73 int num_ext_buffers;
74
75 QSVFrame *work_frames;
76
77 int64_t last_pts;
78
79 int got_output_frame;
80 int eof;
81
82 /* option for Deinterlacing algorithm to be used */
83 int mode;
84 } QSVDeintContext;
85
86 static void qsvdeint_uninit(AVFilterContext *ctx)
87 {
88 QSVDeintContext *s = ctx->priv;
89 QSVFrame *cur;
90
91 if (s->session) {
92 MFXClose(s->session);
93 s->session = NULL;
94 }
95 av_buffer_unref(&s->hw_frames_ctx);
96
97 cur = s->work_frames;
98 while (cur) {
99 s->work_frames = cur->next;
100 av_frame_free(&cur->frame);
101 av_freep(&cur);
102 cur = s->work_frames;
103 }
104
105 av_freep(&s->mem_ids);
106 s->nb_mem_ids = 0;
107
108 av_freep(&s->surface_ptrs);
109 s->nb_surface_ptrs = 0;
110 }
111
112 static int qsvdeint_query_formats(AVFilterContext *ctx)
113 {
114 static const enum AVPixelFormat pixel_formats[] = {
115 AV_PIX_FMT_QSV, AV_PIX_FMT_NONE,
116 };
117 AVFilterFormats *pix_fmts = ff_make_format_list(pixel_formats);
118
119 ff_set_common_formats(ctx, pix_fmts);
120
121 return 0;
122 }
123
124 static mfxStatus frame_alloc(mfxHDL pthis, mfxFrameAllocRequest *req,
125 mfxFrameAllocResponse *resp)
126 {
127 AVFilterContext *ctx = pthis;
128 QSVDeintContext *s = ctx->priv;
129
130 if (!(req->Type & MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET) ||
131 !(req->Type & (MFX_MEMTYPE_FROM_VPPIN | MFX_MEMTYPE_FROM_VPPOUT)) ||
132 !(req->Type & MFX_MEMTYPE_EXTERNAL_FRAME))
133 return MFX_ERR_UNSUPPORTED;
134
135 resp->mids = s->mem_ids;
136 resp->NumFrameActual = s->nb_mem_ids;
137
138 return MFX_ERR_NONE;
139 }
140
141 static mfxStatus frame_free(mfxHDL pthis, mfxFrameAllocResponse *resp)
142 {
143 return MFX_ERR_NONE;
144 }
145
146 static mfxStatus frame_lock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr)
147 {
148 return MFX_ERR_UNSUPPORTED;
149 }
150
151 static mfxStatus frame_unlock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr)
152 {
153 return MFX_ERR_UNSUPPORTED;
154 }
155
156 static mfxStatus frame_get_hdl(mfxHDL pthis, mfxMemId mid, mfxHDL *hdl)
157 {
158 *hdl = mid;
159 return MFX_ERR_NONE;
160 }
161
162 static const mfxHandleType handle_types[] = {
163 MFX_HANDLE_VA_DISPLAY,
164 MFX_HANDLE_D3D9_DEVICE_MANAGER,
165 MFX_HANDLE_D3D11_DEVICE,
166 };
167
168 static int init_out_session(AVFilterContext *ctx)
169 {
170
171 QSVDeintContext *s = ctx->priv;
172 AVHWFramesContext *hw_frames_ctx = (AVHWFramesContext*)s->hw_frames_ctx->data;
173 AVQSVFramesContext *hw_frames_hwctx = hw_frames_ctx->hwctx;
174 AVQSVDeviceContext *device_hwctx = hw_frames_ctx->device_ctx->hwctx;
175
176 int opaque = !!(hw_frames_hwctx->frame_type & MFX_MEMTYPE_OPAQUE_FRAME);
177
178 mfxHDL handle = NULL;
179 mfxHandleType handle_type;
180 mfxVersion ver;
181 mfxIMPL impl;
182 mfxVideoParam par;
183 mfxStatus err;
184 int i;
185
186 /* extract the properties of the "master" session given to us */
187 err = MFXQueryIMPL(device_hwctx->session, &impl);
188 if (err == MFX_ERR_NONE)
189 err = MFXQueryVersion(device_hwctx->session, &ver);
190 if (err != MFX_ERR_NONE) {
191 av_log(ctx, AV_LOG_ERROR, "Error querying the session attributes\n");
192 return AVERROR_UNKNOWN;
193 }
194
195 for (i = 0; i < FF_ARRAY_ELEMS(handle_types); i++) {
196 err = MFXVideoCORE_GetHandle(device_hwctx->session, handle_types[i], &handle);
197 if (err == MFX_ERR_NONE) {
198 handle_type = handle_types[i];
199 break;
200 }
201 }
202
203 /* create a "slave" session with those same properties, to be used for
204 * actual deinterlacing */
205 err = MFXInit(impl, &ver, &s->session);
206 if (err != MFX_ERR_NONE) {
207 av_log(ctx, AV_LOG_ERROR, "Error initializing a session for deinterlacing\n");
208 return AVERROR_UNKNOWN;
209 }
210
211 if (handle) {
212 err = MFXVideoCORE_SetHandle(s->session, handle_type, handle);
213 if (err != MFX_ERR_NONE)
214 return AVERROR_UNKNOWN;
215 }
216
217 memset(&par, 0, sizeof(par));
218
219 s->deint_conf.Header.BufferId = MFX_EXTBUFF_VPP_DEINTERLACING;
220 s->deint_conf.Header.BufferSz = sizeof(s->deint_conf);
221 s->deint_conf.Mode = s->mode;
222
223 s->ext_buffers[s->num_ext_buffers++] = (mfxExtBuffer *)&s->deint_conf;
224
225 if (opaque) {
226 s->surface_ptrs = av_mallocz_array(hw_frames_hwctx->nb_surfaces,
227 sizeof(*s->surface_ptrs));
228 if (!s->surface_ptrs)
229 return AVERROR(ENOMEM);
230 for (i = 0; i < hw_frames_hwctx->nb_surfaces; i++)
231 s->surface_ptrs[i] = hw_frames_hwctx->surfaces + i;
232 s->nb_surface_ptrs = hw_frames_hwctx->nb_surfaces;
233
234 s->opaque_alloc.In.Surfaces = s->surface_ptrs;
235 s->opaque_alloc.In.NumSurface = s->nb_surface_ptrs;
236 s->opaque_alloc.In.Type = hw_frames_hwctx->frame_type;
237
238 s->opaque_alloc.Out = s->opaque_alloc.In;
239
240 s->opaque_alloc.Header.BufferId = MFX_EXTBUFF_OPAQUE_SURFACE_ALLOCATION;
241 s->opaque_alloc.Header.BufferSz = sizeof(s->opaque_alloc);
242
243 s->ext_buffers[s->num_ext_buffers++] = (mfxExtBuffer *)&s->opaque_alloc;
244
245 par.IOPattern = MFX_IOPATTERN_IN_OPAQUE_MEMORY | MFX_IOPATTERN_OUT_OPAQUE_MEMORY;
246 } else {
247 mfxFrameAllocator frame_allocator = {
248 .pthis = ctx,
249 .Alloc = frame_alloc,
250 .Lock = frame_lock,
251 .Unlock = frame_unlock,
252 .GetHDL = frame_get_hdl,
253 .Free = frame_free,
254 };
255
256 s->mem_ids = av_mallocz_array(hw_frames_hwctx->nb_surfaces,
257 sizeof(*s->mem_ids));
258 if (!s->mem_ids)
259 return AVERROR(ENOMEM);
260 for (i = 0; i < hw_frames_hwctx->nb_surfaces; i++)
261 s->mem_ids[i] = hw_frames_hwctx->surfaces[i].Data.MemId;
262 s->nb_mem_ids = hw_frames_hwctx->nb_surfaces;
263
264 err = MFXVideoCORE_SetFrameAllocator(s->session, &frame_allocator);
265 if (err != MFX_ERR_NONE)
266 return AVERROR_UNKNOWN;
267
268 par.IOPattern = MFX_IOPATTERN_IN_VIDEO_MEMORY | MFX_IOPATTERN_OUT_VIDEO_MEMORY;
269 }
270
271 par.ExtParam = s->ext_buffers;
272 par.NumExtParam = s->num_ext_buffers;
273
274 par.AsyncDepth = 1; // TODO async
275
276 par.vpp.In = hw_frames_hwctx->surfaces[0].Info;
277
278 par.vpp.In.CropW = ctx->inputs[0]->w;
279 par.vpp.In.CropH = ctx->inputs[0]->h;
280
281 if (ctx->inputs[0]->frame_rate.num) {
282 par.vpp.In.FrameRateExtN = ctx->inputs[0]->frame_rate.num;
283 par.vpp.In.FrameRateExtD = ctx->inputs[0]->frame_rate.den;
284 } else {
285 par.vpp.In.FrameRateExtN = ctx->inputs[0]->time_base.num;
286 par.vpp.In.FrameRateExtD = ctx->inputs[0]->time_base.den;
287 }
288
289 par.vpp.Out = par.vpp.In;
290
291 if (ctx->outputs[0]->frame_rate.num) {
292 par.vpp.Out.FrameRateExtN = ctx->outputs[0]->frame_rate.num;
293 par.vpp.Out.FrameRateExtD = ctx->outputs[0]->frame_rate.den;
294 } else {
295 par.vpp.Out.FrameRateExtN = ctx->outputs[0]->time_base.num;
296 par.vpp.Out.FrameRateExtD = ctx->outputs[0]->time_base.den;
297 }
298
299 err = MFXVideoVPP_Init(s->session, &par);
300 if (err != MFX_ERR_NONE) {
301 av_log(ctx, AV_LOG_ERROR, "Error opening the VPP for deinterlacing: %d\n", err);
302 return AVERROR_UNKNOWN;
303 }
304
305 return 0;
306 }
307
308 static int qsvdeint_config_props(AVFilterLink *outlink)
309 {
310 AVFilterContext *ctx = outlink->src;
311 AVFilterLink *inlink = ctx->inputs[0];
312 QSVDeintContext *s = ctx->priv;
313 int ret;
314
315 qsvdeint_uninit(ctx);
316
317 s->last_pts = AV_NOPTS_VALUE;
318 outlink->frame_rate = av_mul_q(inlink->frame_rate,
319 (AVRational){ 2, 1 });
320 outlink->time_base = av_mul_q(inlink->time_base,
321 (AVRational){ 1, 2 });
322
323 /* check that we have a hw context */
324 if (!inlink->hw_frames_ctx) {
325 av_log(ctx, AV_LOG_ERROR, "No hw context provided on input\n");
326 return AVERROR(EINVAL);
327 }
328
329 s->hw_frames_ctx = av_buffer_ref(inlink->hw_frames_ctx);
330 if (!s->hw_frames_ctx)
331 return AVERROR(ENOMEM);
332
333 av_buffer_unref(&outlink->hw_frames_ctx);
334 outlink->hw_frames_ctx = av_buffer_ref(inlink->hw_frames_ctx);
335 if (!outlink->hw_frames_ctx) {
336 qsvdeint_uninit(ctx);
337 return AVERROR(ENOMEM);
338 }
339
340 ret = init_out_session(ctx);
341 if (ret < 0)
342 return ret;
343
344
345 return 0;
346 }
347
348 static void clear_unused_frames(QSVDeintContext *s)
349 {
350 QSVFrame *cur = s->work_frames;
351 while (cur) {
352 if (!cur->surface.Data.Locked) {
353 av_frame_free(&cur->frame);
354 cur->used = 0;
355 }
356 cur = cur->next;
357 }
358 }
359
360 static int get_free_frame(QSVDeintContext *s, QSVFrame **f)
361 {
362 QSVFrame *frame, **last;
363
364 clear_unused_frames(s);
365
366 frame = s->work_frames;
367 last = &s->work_frames;
368 while (frame) {
369 if (!frame->used) {
370 *f = frame;
371 return 0;
372 }
373
374 last = &frame->next;
375 frame = frame->next;
376 }
377
378 frame = av_mallocz(sizeof(*frame));
379 if (!frame)
380 return AVERROR(ENOMEM);
381 *last = frame;
382 *f = frame;
383
384 return 0;
385 }
386
387 static int submit_frame(AVFilterContext *ctx, AVFrame *frame,
388 mfxFrameSurface1 **surface)
389 {
390 QSVDeintContext *s = ctx->priv;
391 QSVFrame *qf;
392 int ret;
393
394 ret = get_free_frame(s, &qf);
395 if (ret < 0)
396 return ret;
397
398 qf->frame = frame;
399
400 qf->surface = *(mfxFrameSurface1*)qf->frame->data[3];
401
402 qf->surface.Data.Locked = 0;
403 qf->surface.Info.CropW = qf->frame->width;
404 qf->surface.Info.CropH = qf->frame->height;
405
406 qf->surface.Info.PicStruct = !qf->frame->interlaced_frame ? MFX_PICSTRUCT_PROGRESSIVE :
407 (qf->frame->top_field_first ? MFX_PICSTRUCT_FIELD_TFF :
408 MFX_PICSTRUCT_FIELD_BFF);
409 if (qf->frame->repeat_pict == 1)
410 qf->surface.Info.PicStruct |= MFX_PICSTRUCT_FIELD_REPEATED;
411 else if (qf->frame->repeat_pict == 2)
412 qf->surface.Info.PicStruct |= MFX_PICSTRUCT_FRAME_DOUBLING;
413 else if (qf->frame->repeat_pict == 4)
414 qf->surface.Info.PicStruct |= MFX_PICSTRUCT_FRAME_TRIPLING;
415
416 if (ctx->inputs[0]->frame_rate.num) {
417 qf->surface.Info.FrameRateExtN = ctx->inputs[0]->frame_rate.num;
418 qf->surface.Info.FrameRateExtD = ctx->inputs[0]->frame_rate.den;
419 } else {
420 qf->surface.Info.FrameRateExtN = ctx->inputs[0]->time_base.num;
421 qf->surface.Info.FrameRateExtD = ctx->inputs[0]->time_base.den;
422 }
423
424 qf->surface.Data.TimeStamp = av_rescale_q(qf->frame->pts,
425 ctx->inputs[0]->time_base,
426 (AVRational){1, 90000});
427
428 *surface = &qf->surface;
429 qf->used = 1;
430
431 return 0;
432 }
433
434 static int process_frame(AVFilterContext *ctx, const AVFrame *in,
435 mfxFrameSurface1 *surf_in)
436 {
437 QSVDeintContext *s = ctx->priv;
438 AVFilterLink *inlink = ctx->inputs[0];
439 AVFilterLink *outlink = ctx->outputs[0];
440
441 AVFrame *out;
442 mfxFrameSurface1 *surf_out;
443 mfxSyncPoint sync = NULL;
444 mfxStatus err;
445 int ret, again = 0;
446
447 out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
448 if (!out) {
449 ret = AVERROR(ENOMEM);
450 goto fail;
451 }
452
453 surf_out = (mfxFrameSurface1*)out->data[3];
454 surf_out->Info.CropW = outlink->w;
455 surf_out->Info.CropH = outlink->h;
456 surf_out->Info.PicStruct = MFX_PICSTRUCT_PROGRESSIVE;
457
458 do {
459 err = MFXVideoVPP_RunFrameVPPAsync(s->session, surf_in, surf_out,
460 NULL, &sync);
461 if (err == MFX_WRN_DEVICE_BUSY)
462 av_usleep(1);
463 } while (err == MFX_WRN_DEVICE_BUSY);
464
465 if (err == MFX_ERR_MORE_DATA) {
466 av_frame_free(&out);
467 return QSVDEINT_MORE_INPUT;
468 }
469
470 if ((err < 0 && err != MFX_ERR_MORE_SURFACE) || !sync) {
471 av_log(ctx, AV_LOG_ERROR, "Error during deinterlacing: %d\n", err);
472 ret = AVERROR_UNKNOWN;
473 goto fail;
474 }
475 if (err == MFX_ERR_MORE_SURFACE)
476 again = 1;
477
478 do {
479 err = MFXVideoCORE_SyncOperation(s->session, sync, 1000);
480 } while (err == MFX_WRN_IN_EXECUTION);
481 if (err < 0) {
482 av_log(ctx, AV_LOG_ERROR, "Error synchronizing the operation: %d\n", err);
483 ret = AVERROR_UNKNOWN;
484 goto fail;
485 }
486
487 ret = av_frame_copy_props(out, in);
488 if (ret < 0)
489 goto fail;
490
491 out->width = outlink->w;
492 out->height = outlink->h;
493 out->interlaced_frame = 0;
494
495 out->pts = av_rescale_q(out->pts, inlink->time_base, outlink->time_base);
496 if (out->pts == s->last_pts)
497 out->pts++;
498 s->last_pts = out->pts;
499
500 ret = ff_filter_frame(outlink, out);
501 if (ret < 0)
502 return ret;
503
504 return again ? QSVDEINT_MORE_OUTPUT : 0;
505 fail:
506 av_frame_free(&out);
507 return ret;
508 }
509
510 static int qsvdeint_filter_frame(AVFilterLink *link, AVFrame *in)
511 {
512 AVFilterContext *ctx = link->dst;
513
514 mfxFrameSurface1 *surf_in;
515 int ret;
516
517 ret = submit_frame(ctx, in, &surf_in);
518 if (ret < 0) {
519 av_frame_free(&in);
520 return ret;
521 }
522
523 do {
524 ret = process_frame(ctx, in, surf_in);
525 if (ret < 0)
526 return ret;
527 } while (ret == QSVDEINT_MORE_OUTPUT);
528
529 return 0;
530 }
531
532 static int qsvdeint_request_frame(AVFilterLink *outlink)
533 {
534 AVFilterContext *ctx = outlink->src;
535 QSVDeintContext *s = ctx->priv;
536 int ret = 0;
537
538 s->got_output_frame = 0;
539 while (ret >= 0 && !s->got_output_frame)
540 ret = ff_request_frame(ctx->inputs[0]);
541
542 return ret;
543 }
544
545 #define OFFSET(x) offsetof(QSVDeintContext, x)
546 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM
547 static const AVOption options[] = {
548 { "mode", "set deinterlace mode", OFFSET(mode), AV_OPT_TYPE_INT, {.i64 = MFX_DEINTERLACING_ADVANCED}, MFX_DEINTERLACING_BOB, MFX_DEINTERLACING_ADVANCED, FLAGS, "mode"},
549 { "bob", "bob algorithm", 0, AV_OPT_TYPE_CONST, {.i64 = MFX_DEINTERLACING_BOB}, MFX_DEINTERLACING_BOB, MFX_DEINTERLACING_ADVANCED, FLAGS, "mode"},
550 { "advanced", "Motion adaptive algorithm", 0, AV_OPT_TYPE_CONST, {.i64 = MFX_DEINTERLACING_ADVANCED}, MFX_DEINTERLACING_BOB, MFX_DEINTERLACING_ADVANCED, FLAGS, "mode"},
551 { NULL },
552 };
553
554 static const AVClass qsvdeint_class = {
555 .class_name = "deinterlace_qsv",
556 .item_name = av_default_item_name,
557 .option = options,
558 .version = LIBAVUTIL_VERSION_INT,
559 };
560
561 static const AVFilterPad qsvdeint_inputs[] = {
562 {
563 .name = "default",
564 .type = AVMEDIA_TYPE_VIDEO,
565 .filter_frame = qsvdeint_filter_frame,
566 },
567 { NULL }
568 };
569
570 static const AVFilterPad qsvdeint_outputs[] = {
571 {
572 .name = "default",
573 .type = AVMEDIA_TYPE_VIDEO,
574 .config_props = qsvdeint_config_props,
575 .request_frame = qsvdeint_request_frame,
576 },
577 { NULL }
578 };
579
580 AVFilter ff_vf_deinterlace_qsv = {
581 .name = "deinterlace_qsv",
582 .description = NULL_IF_CONFIG_SMALL("QuickSync video deinterlacing"),
583
584 .uninit = qsvdeint_uninit,
585 .query_formats = qsvdeint_query_formats,
586
587 .priv_size = sizeof(QSVDeintContext),
588 .priv_class = &qsvdeint_class,
589
590 .inputs = qsvdeint_inputs,
591 .outputs = qsvdeint_outputs,
592
593 .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
594 };