2 * Intel MediaSDK QSV codec-independent code
4 * copyright (c) 2013 Luca Barbato
5 * copyright (c) 2015 Anton Khirnov <anton@khirnov.net>
7 * This file is part of Libav.
9 * Libav is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License as published by the Free Software Foundation; either
12 * version 2.1 of the License, or (at your option) any later version.
14 * Libav is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with Libav; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 #include <sys/types.h>
27 #include <mfx/mfxvideo.h>
29 #include "libavutil/common.h"
30 #include "libavutil/hwcontext.h"
31 #include "libavutil/hwcontext_qsv.h"
32 #include "libavutil/mem.h"
33 #include "libavutil/log.h"
34 #include "libavutil/pixdesc.h"
35 #include "libavutil/pixfmt.h"
36 #include "libavutil/time.h"
41 #include "qsv_internal.h"
44 const AVCodecHWConfigInternal
*ff_qsv_hw_configs
[] = {
45 &(const AVCodecHWConfigInternal
) {
47 .pix_fmt
= AV_PIX_FMT_QSV
,
48 .methods
= AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX
|
49 AV_CODEC_HW_CONFIG_METHOD_AD_HOC
,
50 .device_type
= AV_HWDEVICE_TYPE_QSV
,
57 static int qsv_init_session(AVCodecContext
*avctx
, QSVContext
*q
, mfxSession session
,
58 AVBufferRef
*hw_frames_ref
, AVBufferRef
*hw_device_ref
)
64 } else if (hw_frames_ref
) {
65 if (q
->internal_session
) {
66 MFXClose(q
->internal_session
);
67 q
->internal_session
= NULL
;
69 av_buffer_unref(&q
->frames_ctx
.hw_frames_ctx
);
71 q
->frames_ctx
.hw_frames_ctx
= av_buffer_ref(hw_frames_ref
);
72 if (!q
->frames_ctx
.hw_frames_ctx
)
73 return AVERROR(ENOMEM
);
75 ret
= ff_qsv_init_session_frames(avctx
, &q
->internal_session
,
76 &q
->frames_ctx
, q
->load_plugins
,
77 q
->iopattern
== MFX_IOPATTERN_OUT_OPAQUE_MEMORY
);
79 av_buffer_unref(&q
->frames_ctx
.hw_frames_ctx
);
83 q
->session
= q
->internal_session
;
84 } else if (hw_device_ref
) {
85 if (q
->internal_session
) {
86 MFXClose(q
->internal_session
);
87 q
->internal_session
= NULL
;
90 ret
= ff_qsv_init_session_device(avctx
, &q
->internal_session
,
91 hw_device_ref
, q
->load_plugins
);
95 q
->session
= q
->internal_session
;
97 if (!q
->internal_session
) {
98 ret
= ff_qsv_init_internal_session(avctx
, &q
->internal_session
,
104 q
->session
= q
->internal_session
;
107 /* make sure the decoder is uninitialized */
108 MFXVideoDECODE_Close(q
->session
);
113 static inline unsigned int qsv_fifo_item_size(void)
115 return sizeof(mfxSyncPoint
*) + sizeof(QSVFrame
*);
118 static inline unsigned int qsv_fifo_size(const AVFifoBuffer
* fifo
)
120 return av_fifo_size(fifo
) / qsv_fifo_item_size();
123 static int qsv_decode_init(AVCodecContext
*avctx
, QSVContext
*q
)
125 const AVPixFmtDescriptor
*desc
;
126 mfxSession session
= NULL
;
128 mfxVideoParam param
= { 0 };
129 int frame_width
= avctx
->coded_width
;
130 int frame_height
= avctx
->coded_height
;
133 desc
= av_pix_fmt_desc_get(avctx
->sw_pix_fmt
);
137 if (!q
->async_fifo
) {
138 q
->async_fifo
= av_fifo_alloc(q
->async_depth
* qsv_fifo_item_size());
140 return AVERROR(ENOMEM
);
143 if (avctx
->pix_fmt
== AV_PIX_FMT_QSV
&& avctx
->hwaccel_context
) {
144 AVQSVContext
*user_ctx
= avctx
->hwaccel_context
;
145 session
= user_ctx
->session
;
146 iopattern
= user_ctx
->iopattern
;
147 q
->ext_buffers
= user_ctx
->ext_buffers
;
148 q
->nb_ext_buffers
= user_ctx
->nb_ext_buffers
;
151 if (avctx
->hw_frames_ctx
) {
152 AVHWFramesContext
*frames_ctx
= (AVHWFramesContext
*)avctx
->hw_frames_ctx
->data
;
153 AVQSVFramesContext
*frames_hwctx
= frames_ctx
->hwctx
;
156 if (frames_hwctx
->frame_type
& MFX_MEMTYPE_OPAQUE_FRAME
)
157 iopattern
= MFX_IOPATTERN_OUT_OPAQUE_MEMORY
;
158 else if (frames_hwctx
->frame_type
& MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET
)
159 iopattern
= MFX_IOPATTERN_OUT_VIDEO_MEMORY
;
164 iopattern
= MFX_IOPATTERN_OUT_SYSTEM_MEMORY
;
165 q
->iopattern
= iopattern
;
167 ret
= qsv_init_session(avctx
, q
, session
, avctx
->hw_frames_ctx
, avctx
->hw_device_ctx
);
169 av_log(avctx
, AV_LOG_ERROR
, "Error initializing an MFX session\n");
173 ret
= ff_qsv_codec_id_to_mfx(avctx
->codec_id
);
177 param
.mfx
.CodecId
= ret
;
178 param
.mfx
.CodecProfile
= ff_qsv_profile_to_mfx(avctx
->codec_id
, avctx
->profile
);
179 param
.mfx
.CodecLevel
= avctx
->level
== FF_LEVEL_UNKNOWN ? MFX_LEVEL_UNKNOWN
: avctx
->level
;
181 param
.mfx
.FrameInfo
.BitDepthLuma
= desc
->comp
[0].depth
;
182 param
.mfx
.FrameInfo
.BitDepthChroma
= desc
->comp
[0].depth
;
183 param
.mfx
.FrameInfo
.Shift
= desc
->comp
[0].depth
> 8;
184 param
.mfx
.FrameInfo
.FourCC
= q
->fourcc
;
185 param
.mfx
.FrameInfo
.Width
= frame_width
;
186 param
.mfx
.FrameInfo
.Height
= frame_height
;
187 param
.mfx
.FrameInfo
.ChromaFormat
= MFX_CHROMAFORMAT_YUV420
;
189 switch (avctx
->field_order
) {
190 case AV_FIELD_PROGRESSIVE
:
191 param
.mfx
.FrameInfo
.PicStruct
= MFX_PICSTRUCT_PROGRESSIVE
;
194 param
.mfx
.FrameInfo
.PicStruct
= MFX_PICSTRUCT_FIELD_TFF
;
197 param
.mfx
.FrameInfo
.PicStruct
= MFX_PICSTRUCT_FIELD_BFF
;
200 param
.mfx
.FrameInfo
.PicStruct
= MFX_PICSTRUCT_UNKNOWN
;
204 param
.IOPattern
= q
->iopattern
;
205 param
.AsyncDepth
= q
->async_depth
;
206 param
.ExtParam
= q
->ext_buffers
;
207 param
.NumExtParam
= q
->nb_ext_buffers
;
209 ret
= MFXVideoDECODE_Init(q
->session
, ¶m
);
211 return ff_qsv_print_error(avctx
, ret
,
212 "Error initializing the MFX video decoder");
214 q
->frame_info
= param
.mfx
.FrameInfo
;
219 static int alloc_frame(AVCodecContext
*avctx
, QSVContext
*q
, QSVFrame
*frame
)
223 ret
= ff_get_buffer(avctx
, frame
->frame
, AV_GET_BUFFER_FLAG_REF
);
227 if (frame
->frame
->format
== AV_PIX_FMT_QSV
) {
228 frame
->surface
= *(mfxFrameSurface1
*)frame
->frame
->data
[3];
230 frame
->surface
.Info
= q
->frame_info
;
232 frame
->surface
.Data
.PitchLow
= frame
->frame
->linesize
[0];
233 frame
->surface
.Data
.Y
= frame
->frame
->data
[0];
234 frame
->surface
.Data
.UV
= frame
->frame
->data
[1];
237 if (q
->frames_ctx
.mids
) {
238 ret
= ff_qsv_find_surface_idx(&q
->frames_ctx
, frame
);
242 frame
->surface
.Data
.MemId
= &q
->frames_ctx
.mids
[ret
];
244 frame
->surface
.Data
.ExtParam
= &frame
->ext_param
;
245 frame
->surface
.Data
.NumExtParam
= 1;
246 frame
->ext_param
= (mfxExtBuffer
*)&frame
->dec_info
;
247 frame
->dec_info
.Header
.BufferId
= MFX_EXTBUFF_DECODED_FRAME_INFO
;
248 frame
->dec_info
.Header
.BufferSz
= sizeof(frame
->dec_info
);
255 static void qsv_clear_unused_frames(QSVContext
*q
)
257 QSVFrame
*cur
= q
->work_frames
;
259 if (cur
->used
&& !cur
->surface
.Data
.Locked
&& !cur
->queued
) {
261 av_frame_unref(cur
->frame
);
267 static int get_surface(AVCodecContext
*avctx
, QSVContext
*q
, mfxFrameSurface1
**surf
)
269 QSVFrame
*frame
, **last
;
272 qsv_clear_unused_frames(q
);
274 frame
= q
->work_frames
;
275 last
= &q
->work_frames
;
278 ret
= alloc_frame(avctx
, q
, frame
);
281 *surf
= &frame
->surface
;
289 frame
= av_mallocz(sizeof(*frame
));
291 return AVERROR(ENOMEM
);
292 frame
->frame
= av_frame_alloc();
295 return AVERROR(ENOMEM
);
299 ret
= alloc_frame(avctx
, q
, frame
);
303 *surf
= &frame
->surface
;
308 static QSVFrame
*find_frame(QSVContext
*q
, mfxFrameSurface1
*surf
)
310 QSVFrame
*cur
= q
->work_frames
;
312 if (surf
== &cur
->surface
)
319 static int qsv_decode(AVCodecContext
*avctx
, QSVContext
*q
,
320 AVFrame
*frame
, int *got_frame
,
324 mfxFrameSurface1
*insurf
;
325 mfxFrameSurface1
*outsurf
;
327 mfxBitstream bs
= { { { 0 } } };
331 bs
.Data
= avpkt
->data
;
332 bs
.DataLength
= avpkt
->size
;
333 bs
.MaxLength
= bs
.DataLength
;
334 bs
.TimeStamp
= avpkt
->pts
;
335 if (avctx
->field_order
== AV_FIELD_PROGRESSIVE
)
336 bs
.DataFlag
|= MFX_BITSTREAM_COMPLETE_FRAME
;
339 sync
= av_mallocz(sizeof(*sync
));
342 return AVERROR(ENOMEM
);
346 ret
= get_surface(avctx
, q
, &insurf
);
352 ret
= MFXVideoDECODE_DecodeFrameAsync(q
->session
, avpkt
->size ?
&bs
: NULL
,
353 insurf
, &outsurf
, sync
);
354 if (ret
== MFX_WRN_DEVICE_BUSY
)
357 } while (ret
== MFX_WRN_DEVICE_BUSY
|| ret
== MFX_ERR_MORE_SURFACE
);
359 if (ret
!= MFX_ERR_NONE
&&
360 ret
!= MFX_ERR_MORE_DATA
&&
361 ret
!= MFX_WRN_VIDEO_PARAM_CHANGED
&&
362 ret
!= MFX_ERR_MORE_SURFACE
) {
364 return ff_qsv_print_error(avctx
, ret
,
365 "Error during QSV decoding.");
368 /* make sure we do not enter an infinite loop if the SDK
369 * did not consume any data and did not return anything */
370 if (!*sync
&& !bs
.DataOffset
) {
371 bs
.DataOffset
= avpkt
->size
;
372 ++q
->zero_consume_run
;
373 if (q
->zero_consume_run
> 1)
374 ff_qsv_print_warning(avctx
, ret
, "A decode call did not consume any data");
376 q
->zero_consume_run
= 0;
380 QSVFrame
*out_frame
= find_frame(q
, outsurf
);
383 av_log(avctx
, AV_LOG_ERROR
,
384 "The returned surface does not correspond to any frame\n");
389 out_frame
->queued
= 1;
390 av_fifo_generic_write(q
->async_fifo
, &out_frame
, sizeof(out_frame
), NULL
);
391 av_fifo_generic_write(q
->async_fifo
, &sync
, sizeof(sync
), NULL
);
396 if ((qsv_fifo_size(q
->async_fifo
) >= q
->async_depth
) ||
397 (!avpkt
->size
&& av_fifo_size(q
->async_fifo
))) {
400 av_fifo_generic_read(q
->async_fifo
, &out_frame
, sizeof(out_frame
), NULL
);
401 av_fifo_generic_read(q
->async_fifo
, &sync
, sizeof(sync
), NULL
);
402 out_frame
->queued
= 0;
404 if (avctx
->pix_fmt
!= AV_PIX_FMT_QSV
) {
406 ret
= MFXVideoCORE_SyncOperation(q
->session
, *sync
, 1000);
407 } while (ret
== MFX_WRN_IN_EXECUTION
);
412 src_frame
= out_frame
->frame
;
414 ret
= av_frame_ref(frame
, src_frame
);
418 outsurf
= &out_frame
->surface
;
421 FF_DISABLE_DEPRECATION_WARNINGS
422 frame
->pkt_pts
= outsurf
->Data
.TimeStamp
;
423 FF_ENABLE_DEPRECATION_WARNINGS
425 frame
->pts
= outsurf
->Data
.TimeStamp
;
428 outsurf
->Info
.PicStruct
& MFX_PICSTRUCT_FRAME_TRIPLING ?
4 :
429 outsurf
->Info
.PicStruct
& MFX_PICSTRUCT_FRAME_DOUBLING ?
2 :
430 outsurf
->Info
.PicStruct
& MFX_PICSTRUCT_FIELD_REPEATED ?
1 : 0;
431 frame
->top_field_first
=
432 outsurf
->Info
.PicStruct
& MFX_PICSTRUCT_FIELD_TFF
;
433 frame
->interlaced_frame
=
434 !(outsurf
->Info
.PicStruct
& MFX_PICSTRUCT_PROGRESSIVE
);
435 frame
->pict_type
= ff_qsv_map_pictype(out_frame
->dec_info
.FrameType
);
436 //Key frame is IDR frame is only suitable for H264. For HEVC, IRAPs are key frames.
437 if (avctx
->codec_id
== AV_CODEC_ID_H264
)
438 frame
->key_frame
= !!(out_frame
->dec_info
.FrameType
& MFX_FRAMETYPE_IDR
);
440 /* update the surface properties */
441 if (avctx
->pix_fmt
== AV_PIX_FMT_QSV
)
442 ((mfxFrameSurface1
*)frame
->data
[3])->Info
= outsurf
->Info
;
447 return bs
.DataOffset
;
450 int ff_qsv_decode_close(QSVContext
*q
)
452 QSVFrame
*cur
= q
->work_frames
;
455 MFXVideoDECODE_Close(q
->session
);
457 while (q
->async_fifo
&& av_fifo_size(q
->async_fifo
)) {
461 av_fifo_generic_read(q
->async_fifo
, &out_frame
, sizeof(out_frame
), NULL
);
462 av_fifo_generic_read(q
->async_fifo
, &sync
, sizeof(sync
), NULL
);
468 q
->work_frames
= cur
->next
;
469 av_frame_free(&cur
->frame
);
471 cur
= q
->work_frames
;
474 av_fifo_free(q
->async_fifo
);
475 q
->async_fifo
= NULL
;
477 av_parser_close(q
->parser
);
478 avcodec_free_context(&q
->avctx_internal
);
480 if (q
->internal_session
)
481 MFXClose(q
->internal_session
);
483 av_buffer_unref(&q
->frames_ctx
.hw_frames_ctx
);
484 av_buffer_unref(&q
->frames_ctx
.mids_buf
);
489 int ff_qsv_process_data(AVCodecContext
*avctx
, QSVContext
*q
,
490 AVFrame
*frame
, int *got_frame
, AVPacket
*pkt
)
495 const AVPixFmtDescriptor
*desc
;
497 if (!q
->avctx_internal
) {
498 q
->avctx_internal
= avcodec_alloc_context3(NULL
);
499 if (!q
->avctx_internal
)
500 return AVERROR(ENOMEM
);
502 if (avctx
->extradata
) {
503 q
->avctx_internal
->extradata
= av_mallocz(avctx
->extradata_size
+ AV_INPUT_BUFFER_PADDING_SIZE
);
504 if (!q
->avctx_internal
->extradata
)
505 return AVERROR(ENOMEM
);
507 memcpy(q
->avctx_internal
->extradata
, avctx
->extradata
,
508 avctx
->extradata_size
);
509 q
->avctx_internal
->extradata_size
= avctx
->extradata_size
;
512 q
->parser
= av_parser_init(avctx
->codec_id
);
514 return AVERROR(ENOMEM
);
516 q
->parser
->flags
|= PARSER_FLAG_COMPLETE_FRAMES
;
517 q
->orig_pix_fmt
= AV_PIX_FMT_NONE
;
521 return qsv_decode(avctx
, q
, frame
, got_frame
, pkt
);
523 /* we assume the packets are already split properly and want
524 * just the codec parameters here */
525 av_parser_parse2(q
->parser
, q
->avctx_internal
,
526 &dummy_data
, &dummy_size
,
527 pkt
->data
, pkt
->size
, pkt
->pts
, pkt
->dts
,
530 avctx
->field_order
= q
->parser
->field_order
;
531 /* TODO: flush delayed frames on reinit */
532 if (q
->parser
->format
!= q
->orig_pix_fmt
||
533 FFALIGN(q
->parser
->coded_width
, 16) != FFALIGN(avctx
->coded_width
, 16) ||
534 FFALIGN(q
->parser
->coded_height
, 16) != FFALIGN(avctx
->coded_height
, 16)) {
535 enum AVPixelFormat pix_fmts
[3] = { AV_PIX_FMT_QSV
,
538 enum AVPixelFormat qsv_format
;
540 qsv_format
= ff_qsv_map_pixfmt(q
->parser
->format
, &q
->fourcc
);
541 if (qsv_format
< 0) {
542 av_log(avctx
, AV_LOG_ERROR
,
543 "Decoding pixel format '%s' is not supported\n",
544 av_get_pix_fmt_name(q
->parser
->format
));
545 ret
= AVERROR(ENOSYS
);
549 q
->orig_pix_fmt
= q
->parser
->format
;
550 avctx
->pix_fmt
= pix_fmts
[1] = qsv_format
;
551 avctx
->width
= q
->parser
->width
;
552 avctx
->height
= q
->parser
->height
;
553 avctx
->coded_width
= FFALIGN(q
->parser
->coded_width
, 16);
554 avctx
->coded_height
= FFALIGN(q
->parser
->coded_height
, 16);
555 avctx
->level
= q
->avctx_internal
->level
;
556 avctx
->profile
= q
->avctx_internal
->profile
;
558 ret
= ff_get_format(avctx
, pix_fmts
);
562 avctx
->pix_fmt
= ret
;
564 desc
= av_pix_fmt_desc_get(avctx
->pix_fmt
);
568 if (desc
->comp
[0].depth
> 8) {
569 avctx
->coded_width
= FFALIGN(q
->parser
->coded_width
, 32);
570 avctx
->coded_height
= FFALIGN(q
->parser
->coded_height
, 32);
573 ret
= qsv_decode_init(avctx
, q
);
578 return qsv_decode(avctx
, q
, frame
, got_frame
, pkt
);
581 q
->orig_pix_fmt
= q
->parser
->format
= avctx
->pix_fmt
= AV_PIX_FMT_NONE
;
585 void ff_qsv_decode_flush(AVCodecContext
*avctx
, QSVContext
*q
)
587 q
->orig_pix_fmt
= AV_PIX_FMT_NONE
;