qsv: adding Multi Frame Encode support
[libav.git] / libavfilter / vf_deinterlace_qsv.c
CommitLineData
ad71d327
AK
1/*
2 * This file is part of Libav.
3 *
4 * Libav is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * Libav is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with Libav; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19/**
20 * @file
21 * deinterlace video filter - QSV
22 */
23
24#include <mfx/mfxvideo.h>
25
26#include <stdio.h>
27#include <string.h>
28
29#include "libavutil/avstring.h"
30#include "libavutil/common.h"
31#include "libavutil/hwcontext.h"
32#include "libavutil/hwcontext_qsv.h"
33#include "libavutil/internal.h"
34#include "libavutil/mathematics.h"
35#include "libavutil/opt.h"
36#include "libavutil/pixdesc.h"
37#include "libavutil/time.h"
cca5e4f0 38#include "libavfilter/qsvvpp.h"
ad71d327
AK
39
40#include "avfilter.h"
41#include "formats.h"
42#include "internal.h"
43#include "video.h"
44
45enum {
46 QSVDEINT_MORE_OUTPUT = 1,
47 QSVDEINT_MORE_INPUT,
48};
49
50typedef struct QSVFrame {
51 AVFrame *frame;
52 mfxFrameSurface1 surface;
53 int used;
54
55 struct QSVFrame *next;
56} QSVFrame;
57
58typedef struct QSVDeintContext {
59 const AVClass *class;
60
61 AVBufferRef *hw_frames_ctx;
62 /* a clone of the main session, used internally for deinterlacing */
63 mfxSession session;
64
65 mfxMemId *mem_ids;
66 int nb_mem_ids;
67
68 mfxFrameSurface1 **surface_ptrs;
69 int nb_surface_ptrs;
70
71 mfxExtOpaqueSurfaceAlloc opaque_alloc;
550cb6a8
HZ
72 mfxExtVPPDeinterlacing deint_conf;
73 mfxExtBuffer *ext_buffers[2];
74 int num_ext_buffers;
ad71d327
AK
75
76 QSVFrame *work_frames;
77
78 int64_t last_pts;
79
80 int got_output_frame;
81 int eof;
550cb6a8
HZ
82
83 /* option for Deinterlacing algorithm to be used */
84 int mode;
ad71d327
AK
85} QSVDeintContext;
86
87static void qsvdeint_uninit(AVFilterContext *ctx)
88{
89 QSVDeintContext *s = ctx->priv;
90 QSVFrame *cur;
91
92 if (s->session) {
93 MFXClose(s->session);
94 s->session = NULL;
95 }
96 av_buffer_unref(&s->hw_frames_ctx);
97
98 cur = s->work_frames;
99 while (cur) {
100 s->work_frames = cur->next;
101 av_frame_free(&cur->frame);
102 av_freep(&cur);
103 cur = s->work_frames;
104 }
105
106 av_freep(&s->mem_ids);
107 s->nb_mem_ids = 0;
108
109 av_freep(&s->surface_ptrs);
110 s->nb_surface_ptrs = 0;
111}
112
113static int qsvdeint_query_formats(AVFilterContext *ctx)
114{
115 static const enum AVPixelFormat pixel_formats[] = {
116 AV_PIX_FMT_QSV, AV_PIX_FMT_NONE,
117 };
118 AVFilterFormats *pix_fmts = ff_make_format_list(pixel_formats);
119
120 ff_set_common_formats(ctx, pix_fmts);
121
122 return 0;
123}
124
125static mfxStatus frame_alloc(mfxHDL pthis, mfxFrameAllocRequest *req,
126 mfxFrameAllocResponse *resp)
127{
128 AVFilterContext *ctx = pthis;
129 QSVDeintContext *s = ctx->priv;
130
131 if (!(req->Type & MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET) ||
132 !(req->Type & (MFX_MEMTYPE_FROM_VPPIN | MFX_MEMTYPE_FROM_VPPOUT)) ||
133 !(req->Type & MFX_MEMTYPE_EXTERNAL_FRAME))
134 return MFX_ERR_UNSUPPORTED;
135
136 resp->mids = s->mem_ids;
137 resp->NumFrameActual = s->nb_mem_ids;
138
139 return MFX_ERR_NONE;
140}
141
142static mfxStatus frame_free(mfxHDL pthis, mfxFrameAllocResponse *resp)
143{
144 return MFX_ERR_NONE;
145}
146
147static mfxStatus frame_lock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr)
148{
149 return MFX_ERR_UNSUPPORTED;
150}
151
152static mfxStatus frame_unlock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr)
153{
154 return MFX_ERR_UNSUPPORTED;
155}
156
157static mfxStatus frame_get_hdl(mfxHDL pthis, mfxMemId mid, mfxHDL *hdl)
158{
159 *hdl = mid;
160 return MFX_ERR_NONE;
161}
162
163static const mfxHandleType handle_types[] = {
164 MFX_HANDLE_VA_DISPLAY,
165 MFX_HANDLE_D3D9_DEVICE_MANAGER,
166 MFX_HANDLE_D3D11_DEVICE,
167};
168
169static int init_out_session(AVFilterContext *ctx)
170{
171
172 QSVDeintContext *s = ctx->priv;
173 AVHWFramesContext *hw_frames_ctx = (AVHWFramesContext*)s->hw_frames_ctx->data;
174 AVQSVFramesContext *hw_frames_hwctx = hw_frames_ctx->hwctx;
175 AVQSVDeviceContext *device_hwctx = hw_frames_ctx->device_ctx->hwctx;
176
177 int opaque = !!(hw_frames_hwctx->frame_type & MFX_MEMTYPE_OPAQUE_FRAME);
178
179 mfxHDL handle = NULL;
180 mfxHandleType handle_type;
181 mfxVersion ver;
182 mfxIMPL impl;
183 mfxVideoParam par;
184 mfxStatus err;
185 int i;
186
187 /* extract the properties of the "master" session given to us */
188 err = MFXQueryIMPL(device_hwctx->session, &impl);
189 if (err == MFX_ERR_NONE)
190 err = MFXQueryVersion(device_hwctx->session, &ver);
191 if (err != MFX_ERR_NONE) {
192 av_log(ctx, AV_LOG_ERROR, "Error querying the session attributes\n");
193 return AVERROR_UNKNOWN;
194 }
195
196 for (i = 0; i < FF_ARRAY_ELEMS(handle_types); i++) {
197 err = MFXVideoCORE_GetHandle(device_hwctx->session, handle_types[i], &handle);
198 if (err == MFX_ERR_NONE) {
199 handle_type = handle_types[i];
200 break;
201 }
202 }
203
204 /* create a "slave" session with those same properties, to be used for
205 * actual deinterlacing */
206 err = MFXInit(impl, &ver, &s->session);
207 if (err != MFX_ERR_NONE) {
208 av_log(ctx, AV_LOG_ERROR, "Error initializing a session for deinterlacing\n");
209 return AVERROR_UNKNOWN;
210 }
211
212 if (handle) {
213 err = MFXVideoCORE_SetHandle(s->session, handle_type, handle);
214 if (err != MFX_ERR_NONE)
215 return AVERROR_UNKNOWN;
216 }
217
cca5e4f0
MD
218 if (QSV_RUNTIME_VERSION_ATLEAST(ver, 1, 25)) {
219 err = MFXJoinSession(device_hwctx->session, s->session);
220 if (err != MFX_ERR_NONE)
221 return AVERROR_UNKNOWN;
222 }
223
ad71d327
AK
224 memset(&par, 0, sizeof(par));
225
550cb6a8
HZ
226 s->deint_conf.Header.BufferId = MFX_EXTBUFF_VPP_DEINTERLACING;
227 s->deint_conf.Header.BufferSz = sizeof(s->deint_conf);
228 s->deint_conf.Mode = s->mode;
229
230 s->ext_buffers[s->num_ext_buffers++] = (mfxExtBuffer *)&s->deint_conf;
231
ad71d327
AK
232 if (opaque) {
233 s->surface_ptrs = av_mallocz_array(hw_frames_hwctx->nb_surfaces,
234 sizeof(*s->surface_ptrs));
235 if (!s->surface_ptrs)
236 return AVERROR(ENOMEM);
237 for (i = 0; i < hw_frames_hwctx->nb_surfaces; i++)
238 s->surface_ptrs[i] = hw_frames_hwctx->surfaces + i;
239 s->nb_surface_ptrs = hw_frames_hwctx->nb_surfaces;
240
241 s->opaque_alloc.In.Surfaces = s->surface_ptrs;
242 s->opaque_alloc.In.NumSurface = s->nb_surface_ptrs;
243 s->opaque_alloc.In.Type = hw_frames_hwctx->frame_type;
244
245 s->opaque_alloc.Out = s->opaque_alloc.In;
246
247 s->opaque_alloc.Header.BufferId = MFX_EXTBUFF_OPAQUE_SURFACE_ALLOCATION;
248 s->opaque_alloc.Header.BufferSz = sizeof(s->opaque_alloc);
249
550cb6a8 250 s->ext_buffers[s->num_ext_buffers++] = (mfxExtBuffer *)&s->opaque_alloc;
ad71d327
AK
251
252 par.IOPattern = MFX_IOPATTERN_IN_OPAQUE_MEMORY | MFX_IOPATTERN_OUT_OPAQUE_MEMORY;
253 } else {
254 mfxFrameAllocator frame_allocator = {
255 .pthis = ctx,
256 .Alloc = frame_alloc,
257 .Lock = frame_lock,
258 .Unlock = frame_unlock,
259 .GetHDL = frame_get_hdl,
260 .Free = frame_free,
261 };
262
263 s->mem_ids = av_mallocz_array(hw_frames_hwctx->nb_surfaces,
264 sizeof(*s->mem_ids));
265 if (!s->mem_ids)
266 return AVERROR(ENOMEM);
267 for (i = 0; i < hw_frames_hwctx->nb_surfaces; i++)
268 s->mem_ids[i] = hw_frames_hwctx->surfaces[i].Data.MemId;
269 s->nb_mem_ids = hw_frames_hwctx->nb_surfaces;
270
271 err = MFXVideoCORE_SetFrameAllocator(s->session, &frame_allocator);
272 if (err != MFX_ERR_NONE)
273 return AVERROR_UNKNOWN;
274
275 par.IOPattern = MFX_IOPATTERN_IN_VIDEO_MEMORY | MFX_IOPATTERN_OUT_VIDEO_MEMORY;
276 }
277
550cb6a8
HZ
278 par.ExtParam = s->ext_buffers;
279 par.NumExtParam = s->num_ext_buffers;
280
ad71d327
AK
281 par.AsyncDepth = 1; // TODO async
282
283 par.vpp.In = hw_frames_hwctx->surfaces[0].Info;
284
285 par.vpp.In.CropW = ctx->inputs[0]->w;
286 par.vpp.In.CropH = ctx->inputs[0]->h;
287
288 if (ctx->inputs[0]->frame_rate.num) {
289 par.vpp.In.FrameRateExtN = ctx->inputs[0]->frame_rate.num;
290 par.vpp.In.FrameRateExtD = ctx->inputs[0]->frame_rate.den;
291 } else {
292 par.vpp.In.FrameRateExtN = ctx->inputs[0]->time_base.num;
293 par.vpp.In.FrameRateExtD = ctx->inputs[0]->time_base.den;
294 }
295
296 par.vpp.Out = par.vpp.In;
297
298 if (ctx->outputs[0]->frame_rate.num) {
299 par.vpp.Out.FrameRateExtN = ctx->outputs[0]->frame_rate.num;
300 par.vpp.Out.FrameRateExtD = ctx->outputs[0]->frame_rate.den;
301 } else {
302 par.vpp.Out.FrameRateExtN = ctx->outputs[0]->time_base.num;
303 par.vpp.Out.FrameRateExtD = ctx->outputs[0]->time_base.den;
304 }
305
306 err = MFXVideoVPP_Init(s->session, &par);
307 if (err != MFX_ERR_NONE) {
308 av_log(ctx, AV_LOG_ERROR, "Error opening the VPP for deinterlacing: %d\n", err);
309 return AVERROR_UNKNOWN;
310 }
311
312 return 0;
313}
314
315static int qsvdeint_config_props(AVFilterLink *outlink)
316{
317 AVFilterContext *ctx = outlink->src;
318 AVFilterLink *inlink = ctx->inputs[0];
319 QSVDeintContext *s = ctx->priv;
320 int ret;
321
322 qsvdeint_uninit(ctx);
323
324 s->last_pts = AV_NOPTS_VALUE;
325 outlink->frame_rate = av_mul_q(inlink->frame_rate,
326 (AVRational){ 2, 1 });
327 outlink->time_base = av_mul_q(inlink->time_base,
328 (AVRational){ 1, 2 });
329
330 /* check that we have a hw context */
331 if (!inlink->hw_frames_ctx) {
332 av_log(ctx, AV_LOG_ERROR, "No hw context provided on input\n");
333 return AVERROR(EINVAL);
334 }
335
336 s->hw_frames_ctx = av_buffer_ref(inlink->hw_frames_ctx);
337 if (!s->hw_frames_ctx)
338 return AVERROR(ENOMEM);
339
340 av_buffer_unref(&outlink->hw_frames_ctx);
341 outlink->hw_frames_ctx = av_buffer_ref(inlink->hw_frames_ctx);
342 if (!outlink->hw_frames_ctx) {
343 qsvdeint_uninit(ctx);
344 return AVERROR(ENOMEM);
345 }
346
347 ret = init_out_session(ctx);
348 if (ret < 0)
349 return ret;
350
351
352 return 0;
353}
354
355static void clear_unused_frames(QSVDeintContext *s)
356{
357 QSVFrame *cur = s->work_frames;
358 while (cur) {
359 if (!cur->surface.Data.Locked) {
360 av_frame_free(&cur->frame);
361 cur->used = 0;
362 }
363 cur = cur->next;
364 }
365}
366
367static int get_free_frame(QSVDeintContext *s, QSVFrame **f)
368{
369 QSVFrame *frame, **last;
370
371 clear_unused_frames(s);
372
373 frame = s->work_frames;
374 last = &s->work_frames;
375 while (frame) {
376 if (!frame->used) {
377 *f = frame;
378 return 0;
379 }
380
381 last = &frame->next;
382 frame = frame->next;
383 }
384
385 frame = av_mallocz(sizeof(*frame));
386 if (!frame)
387 return AVERROR(ENOMEM);
388 *last = frame;
389 *f = frame;
390
391 return 0;
392}
393
394static int submit_frame(AVFilterContext *ctx, AVFrame *frame,
395 mfxFrameSurface1 **surface)
396{
397 QSVDeintContext *s = ctx->priv;
398 QSVFrame *qf;
399 int ret;
400
401 ret = get_free_frame(s, &qf);
402 if (ret < 0)
403 return ret;
404
405 qf->frame = frame;
406
407 qf->surface = *(mfxFrameSurface1*)qf->frame->data[3];
408
409 qf->surface.Data.Locked = 0;
410 qf->surface.Info.CropW = qf->frame->width;
411 qf->surface.Info.CropH = qf->frame->height;
412
413 qf->surface.Info.PicStruct = !qf->frame->interlaced_frame ? MFX_PICSTRUCT_PROGRESSIVE :
414 (qf->frame->top_field_first ? MFX_PICSTRUCT_FIELD_TFF :
415 MFX_PICSTRUCT_FIELD_BFF);
416 if (qf->frame->repeat_pict == 1)
417 qf->surface.Info.PicStruct |= MFX_PICSTRUCT_FIELD_REPEATED;
418 else if (qf->frame->repeat_pict == 2)
419 qf->surface.Info.PicStruct |= MFX_PICSTRUCT_FRAME_DOUBLING;
420 else if (qf->frame->repeat_pict == 4)
421 qf->surface.Info.PicStruct |= MFX_PICSTRUCT_FRAME_TRIPLING;
422
423 if (ctx->inputs[0]->frame_rate.num) {
424 qf->surface.Info.FrameRateExtN = ctx->inputs[0]->frame_rate.num;
425 qf->surface.Info.FrameRateExtD = ctx->inputs[0]->frame_rate.den;
426 } else {
427 qf->surface.Info.FrameRateExtN = ctx->inputs[0]->time_base.num;
428 qf->surface.Info.FrameRateExtD = ctx->inputs[0]->time_base.den;
429 }
430
431 qf->surface.Data.TimeStamp = av_rescale_q(qf->frame->pts,
432 ctx->inputs[0]->time_base,
433 (AVRational){1, 90000});
434
435 *surface = &qf->surface;
436 qf->used = 1;
437
438 return 0;
439}
440
441static int process_frame(AVFilterContext *ctx, const AVFrame *in,
442 mfxFrameSurface1 *surf_in)
443{
444 QSVDeintContext *s = ctx->priv;
445 AVFilterLink *inlink = ctx->inputs[0];
446 AVFilterLink *outlink = ctx->outputs[0];
447
448 AVFrame *out;
449 mfxFrameSurface1 *surf_out;
450 mfxSyncPoint sync = NULL;
451 mfxStatus err;
452 int ret, again = 0;
453
7e2561fa
MT
454 out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
455 if (!out) {
456 ret = AVERROR(ENOMEM);
ad71d327 457 goto fail;
7e2561fa 458 }
ad71d327
AK
459
460 surf_out = (mfxFrameSurface1*)out->data[3];
461 surf_out->Info.CropW = outlink->w;
462 surf_out->Info.CropH = outlink->h;
463 surf_out->Info.PicStruct = MFX_PICSTRUCT_PROGRESSIVE;
464
465 do {
466 err = MFXVideoVPP_RunFrameVPPAsync(s->session, surf_in, surf_out,
467 NULL, &sync);
468 if (err == MFX_WRN_DEVICE_BUSY)
469 av_usleep(1);
470 } while (err == MFX_WRN_DEVICE_BUSY);
471
472 if (err == MFX_ERR_MORE_DATA) {
473 av_frame_free(&out);
474 return QSVDEINT_MORE_INPUT;
475 }
476
477 if ((err < 0 && err != MFX_ERR_MORE_SURFACE) || !sync) {
478 av_log(ctx, AV_LOG_ERROR, "Error during deinterlacing: %d\n", err);
479 ret = AVERROR_UNKNOWN;
480 goto fail;
481 }
482 if (err == MFX_ERR_MORE_SURFACE)
483 again = 1;
484
485 do {
486 err = MFXVideoCORE_SyncOperation(s->session, sync, 1000);
487 } while (err == MFX_WRN_IN_EXECUTION);
488 if (err < 0) {
489 av_log(ctx, AV_LOG_ERROR, "Error synchronizing the operation: %d\n", err);
490 ret = AVERROR_UNKNOWN;
491 goto fail;
492 }
493
494 ret = av_frame_copy_props(out, in);
495 if (ret < 0)
496 goto fail;
497
498 out->width = outlink->w;
499 out->height = outlink->h;
500 out->interlaced_frame = 0;
501
502 out->pts = av_rescale_q(out->pts, inlink->time_base, outlink->time_base);
503 if (out->pts == s->last_pts)
504 out->pts++;
505 s->last_pts = out->pts;
506
507 ret = ff_filter_frame(outlink, out);
508 if (ret < 0)
509 return ret;
510
511 return again ? QSVDEINT_MORE_OUTPUT : 0;
512fail:
513 av_frame_free(&out);
514 return ret;
515}
516
517static int qsvdeint_filter_frame(AVFilterLink *link, AVFrame *in)
518{
519 AVFilterContext *ctx = link->dst;
520
521 mfxFrameSurface1 *surf_in;
522 int ret;
523
524 ret = submit_frame(ctx, in, &surf_in);
525 if (ret < 0) {
526 av_frame_free(&in);
527 return ret;
528 }
529
530 do {
531 ret = process_frame(ctx, in, surf_in);
532 if (ret < 0)
533 return ret;
534 } while (ret == QSVDEINT_MORE_OUTPUT);
535
536 return 0;
537}
538
539static int qsvdeint_request_frame(AVFilterLink *outlink)
540{
541 AVFilterContext *ctx = outlink->src;
542 QSVDeintContext *s = ctx->priv;
543 int ret = 0;
544
545 s->got_output_frame = 0;
546 while (ret >= 0 && !s->got_output_frame)
547 ret = ff_request_frame(ctx->inputs[0]);
548
549 return ret;
550}
551
552#define OFFSET(x) offsetof(QSVDeintContext, x)
553#define FLAGS AV_OPT_FLAG_VIDEO_PARAM
554static const AVOption options[] = {
550cb6a8
HZ
555 { "mode", "set deinterlace mode", OFFSET(mode), AV_OPT_TYPE_INT, {.i64 = MFX_DEINTERLACING_ADVANCED}, MFX_DEINTERLACING_BOB, MFX_DEINTERLACING_ADVANCED, FLAGS, "mode"},
556 { "bob", "bob algorithm", 0, AV_OPT_TYPE_CONST, {.i64 = MFX_DEINTERLACING_BOB}, MFX_DEINTERLACING_BOB, MFX_DEINTERLACING_ADVANCED, FLAGS, "mode"},
557 { "advanced", "Motion adaptive algorithm", 0, AV_OPT_TYPE_CONST, {.i64 = MFX_DEINTERLACING_ADVANCED}, MFX_DEINTERLACING_BOB, MFX_DEINTERLACING_ADVANCED, FLAGS, "mode"},
ad71d327
AK
558 { NULL },
559};
560
561static const AVClass qsvdeint_class = {
562 .class_name = "deinterlace_qsv",
563 .item_name = av_default_item_name,
564 .option = options,
565 .version = LIBAVUTIL_VERSION_INT,
566};
567
568static const AVFilterPad qsvdeint_inputs[] = {
569 {
570 .name = "default",
571 .type = AVMEDIA_TYPE_VIDEO,
572 .filter_frame = qsvdeint_filter_frame,
573 },
574 { NULL }
575};
576
577static const AVFilterPad qsvdeint_outputs[] = {
578 {
579 .name = "default",
580 .type = AVMEDIA_TYPE_VIDEO,
581 .config_props = qsvdeint_config_props,
582 .request_frame = qsvdeint_request_frame,
583 },
584 { NULL }
585};
586
587AVFilter ff_vf_deinterlace_qsv = {
588 .name = "deinterlace_qsv",
589 .description = NULL_IF_CONFIG_SMALL("QuickSync video deinterlacing"),
590
591 .uninit = qsvdeint_uninit,
592 .query_formats = qsvdeint_query_formats,
593
594 .priv_size = sizeof(QSVDeintContext),
595 .priv_class = &qsvdeint_class,
596
597 .inputs = qsvdeint_inputs,
598 .outputs = qsvdeint_outputs,
e3fb74f7
MT
599
600 .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
ad71d327 601};