avconv_vaapi: fix double-free with some unsupported streams
[libav.git] / avconv_vaapi.c
1 /*
2 * This file is part of Libav.
3 *
4 * Libav is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * Libav is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with Libav; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19 #include "config.h"
20
21 #include <fcntl.h>
22 #include <unistd.h>
23
24 #include <va/va.h>
25 #if HAVE_VAAPI_X11
26 # include <va/va_x11.h>
27 #endif
28 #if HAVE_VAAPI_DRM
29 # include <va/va_drm.h>
30 #endif
31
32 #include "libavutil/avassert.h"
33 #include "libavutil/avconfig.h"
34 #include "libavutil/buffer.h"
35 #include "libavutil/frame.h"
36 #include "libavutil/hwcontext.h"
37 #include "libavutil/hwcontext_vaapi.h"
38 #include "libavutil/imgutils.h"
39 #include "libavutil/opt.h"
40 #include "libavutil/pixfmt.h"
41
42 #include "libavcodec/vaapi.h"
43
44 #include "avconv.h"
45
46
47 static AVClass vaapi_class = {
48 .class_name = "vaapi",
49 .item_name = av_default_item_name,
50 .version = LIBAVUTIL_VERSION_INT,
51 };
52
53 #define DEFAULT_SURFACES 20
54
55 typedef struct VAAPIDecoderContext {
56 const AVClass *class;
57
58 AVBufferRef *device_ref;
59 AVHWDeviceContext *device;
60 AVBufferRef *frames_ref;
61 AVHWFramesContext *frames;
62
63 VAProfile va_profile;
64 VAEntrypoint va_entrypoint;
65 VAConfigID va_config;
66 VAContextID va_context;
67
68 enum AVPixelFormat decode_format;
69 int decode_width;
70 int decode_height;
71 int decode_surfaces;
72
73 // The output need not have the same format, width and height as the
74 // decoded frames - the copy for non-direct-mapped access is actually
75 // a whole vpp instance which can do arbitrary scaling and format
76 // conversion.
77 enum AVPixelFormat output_format;
78
79 struct vaapi_context decoder_vaapi_context;
80 } VAAPIDecoderContext;
81
82
83 static int vaapi_get_buffer(AVCodecContext *avctx, AVFrame *frame, int flags)
84 {
85 InputStream *ist = avctx->opaque;
86 VAAPIDecoderContext *ctx = ist->hwaccel_ctx;
87 int err;
88
89 err = av_hwframe_get_buffer(ctx->frames_ref, frame, 0);
90 if (err < 0) {
91 av_log(ctx, AV_LOG_ERROR, "Failed to allocate decoder surface.\n");
92 } else {
93 av_log(ctx, AV_LOG_DEBUG, "Decoder given surface %#x.\n",
94 (unsigned int)(uintptr_t)frame->data[3]);
95 }
96 return err;
97 }
98
99 static int vaapi_retrieve_data(AVCodecContext *avctx, AVFrame *input)
100 {
101 InputStream *ist = avctx->opaque;
102 VAAPIDecoderContext *ctx = ist->hwaccel_ctx;
103 AVFrame *output = 0;
104 int err;
105
106 av_assert0(input->format == AV_PIX_FMT_VAAPI);
107
108 if (ctx->output_format == AV_PIX_FMT_VAAPI) {
109 // Nothing to do.
110 return 0;
111 }
112
113 av_log(ctx, AV_LOG_DEBUG, "Retrieve data from surface %#x.\n",
114 (unsigned int)(uintptr_t)input->data[3]);
115
116 output = av_frame_alloc();
117 if (!output)
118 return AVERROR(ENOMEM);
119
120 output->format = ctx->output_format;
121
122 err = av_hwframe_transfer_data(output, input, 0);
123 if (err < 0) {
124 av_log(ctx, AV_LOG_ERROR, "Failed to transfer data to "
125 "output frame: %d.\n", err);
126 goto fail;
127 }
128
129 err = av_frame_copy_props(output, input);
130 if (err < 0) {
131 av_frame_unref(output);
132 goto fail;
133 }
134
135 av_frame_unref(input);
136 av_frame_move_ref(input, output);
137 av_frame_free(&output);
138
139 return 0;
140
141 fail:
142 if (output)
143 av_frame_free(&output);
144 return err;
145 }
146
147
148 static const struct {
149 enum AVCodecID codec_id;
150 int codec_profile;
151 VAProfile va_profile;
152 } vaapi_profile_map[] = {
153 #define MAP(c, p, v) { AV_CODEC_ID_ ## c, FF_PROFILE_ ## p, VAProfile ## v }
154 MAP(MPEG2VIDEO, MPEG2_SIMPLE, MPEG2Simple ),
155 MAP(MPEG2VIDEO, MPEG2_MAIN, MPEG2Main ),
156 MAP(H263, UNKNOWN, H263Baseline),
157 MAP(MPEG4, MPEG4_SIMPLE, MPEG4Simple ),
158 MAP(MPEG4, MPEG4_ADVANCED_SIMPLE,
159 MPEG4AdvancedSimple),
160 MAP(MPEG4, MPEG4_MAIN, MPEG4Main ),
161 MAP(H264, H264_CONSTRAINED_BASELINE,
162 H264ConstrainedBaseline),
163 MAP(H264, H264_BASELINE, H264Baseline),
164 MAP(H264, H264_MAIN, H264Main ),
165 MAP(H264, H264_HIGH, H264High ),
166 #if VA_CHECK_VERSION(0, 37, 0)
167 MAP(HEVC, HEVC_MAIN, HEVCMain ),
168 #endif
169 MAP(WMV3, VC1_SIMPLE, VC1Simple ),
170 MAP(WMV3, VC1_MAIN, VC1Main ),
171 MAP(WMV3, VC1_COMPLEX, VC1Advanced ),
172 MAP(WMV3, VC1_ADVANCED, VC1Advanced ),
173 MAP(VC1, VC1_SIMPLE, VC1Simple ),
174 MAP(VC1, VC1_MAIN, VC1Main ),
175 MAP(VC1, VC1_COMPLEX, VC1Advanced ),
176 MAP(VC1, VC1_ADVANCED, VC1Advanced ),
177 #if VA_CHECK_VERSION(0, 35, 0)
178 MAP(VP8, UNKNOWN, VP8Version0_3 ),
179 #endif
180 #if VA_CHECK_VERSION(0, 37, 1)
181 MAP(VP9, VP9_0, VP9Profile0 ),
182 #endif
183 #undef MAP
184 };
185
186 static int vaapi_build_decoder_config(VAAPIDecoderContext *ctx,
187 AVCodecContext *avctx,
188 int fallback_allowed)
189 {
190 AVVAAPIDeviceContext *hwctx = ctx->device->hwctx;
191 AVVAAPIHWConfig *hwconfig = NULL;
192 AVHWFramesConstraints *constraints = NULL;
193 VAStatus vas;
194 int err, i, j;
195 int loglevel = fallback_allowed ? AV_LOG_VERBOSE : AV_LOG_ERROR;
196 const AVCodecDescriptor *codec_desc;
197 const AVPixFmtDescriptor *pix_desc;
198 enum AVPixelFormat pix_fmt;
199 VAProfile profile, *profile_list = NULL;
200 int profile_count, exact_match, alt_profile;
201
202 codec_desc = avcodec_descriptor_get(avctx->codec_id);
203 if (!codec_desc) {
204 err = AVERROR(EINVAL);
205 goto fail;
206 }
207
208 profile_count = vaMaxNumProfiles(hwctx->display);
209 profile_list = av_malloc(profile_count * sizeof(VAProfile));
210 if (!profile_list) {
211 err = AVERROR(ENOMEM);
212 goto fail;
213 }
214
215 vas = vaQueryConfigProfiles(hwctx->display,
216 profile_list, &profile_count);
217 if (vas != VA_STATUS_SUCCESS) {
218 av_log(ctx, loglevel, "Failed to query profiles: %d (%s).\n",
219 vas, vaErrorStr(vas));
220 err = AVERROR(EIO);
221 goto fail;
222 }
223
224 profile = VAProfileNone;
225 exact_match = 0;
226
227 for (i = 0; i < FF_ARRAY_ELEMS(vaapi_profile_map); i++) {
228 int profile_match = 0;
229 if (avctx->codec_id != vaapi_profile_map[i].codec_id)
230 continue;
231 if (avctx->profile == vaapi_profile_map[i].codec_profile)
232 profile_match = 1;
233 profile = vaapi_profile_map[i].va_profile;
234 for (j = 0; j < profile_count; j++) {
235 if (profile == profile_list[j]) {
236 exact_match = profile_match;
237 break;
238 }
239 }
240 if (j < profile_count) {
241 if (exact_match)
242 break;
243 alt_profile = vaapi_profile_map[i].codec_profile;
244 }
245 }
246 av_freep(&profile_list);
247
248 if (profile == VAProfileNone) {
249 av_log(ctx, loglevel, "No VAAPI support for codec %s.\n",
250 codec_desc->name);
251 err = AVERROR(ENOSYS);
252 goto fail;
253 }
254 if (!exact_match) {
255 if (fallback_allowed || !hwaccel_lax_profile_check) {
256 av_log(ctx, loglevel, "No VAAPI support for codec %s "
257 "profile %d.\n", codec_desc->name, avctx->profile);
258 if (!fallback_allowed) {
259 av_log(ctx, AV_LOG_WARNING, "If you want attempt decoding "
260 "anyway with a possibly-incompatible profile, add "
261 "the option -hwaccel_lax_profile_check.\n");
262 }
263 err = AVERROR(EINVAL);
264 goto fail;
265 } else {
266 av_log(ctx, AV_LOG_WARNING, "No VAAPI support for codec %s "
267 "profile %d: trying instead with profile %d.\n",
268 codec_desc->name, avctx->profile, alt_profile);
269 av_log(ctx, AV_LOG_WARNING, "This may fail or give "
270 "incorrect results, depending on your hardware.\n");
271 }
272 }
273
274 ctx->va_profile = profile;
275 ctx->va_entrypoint = VAEntrypointVLD;
276
277 vas = vaCreateConfig(hwctx->display, ctx->va_profile,
278 ctx->va_entrypoint, 0, 0, &ctx->va_config);
279 if (vas != VA_STATUS_SUCCESS) {
280 av_log(ctx, AV_LOG_ERROR, "Failed to create decode pipeline "
281 "configuration: %d (%s).\n", vas, vaErrorStr(vas));
282 err = AVERROR(EIO);
283 goto fail;
284 }
285
286 hwconfig = av_hwdevice_hwconfig_alloc(ctx->device_ref);
287 if (!hwconfig) {
288 err = AVERROR(ENOMEM);
289 goto fail;
290 }
291 hwconfig->config_id = ctx->va_config;
292
293 constraints = av_hwdevice_get_hwframe_constraints(ctx->device_ref,
294 hwconfig);
295 if (!constraints)
296 goto fail;
297
298 // Decide on the decoder target format.
299 // If the user specified something with -hwaccel_output_format then
300 // try to use that to minimise conversions later.
301 ctx->decode_format = AV_PIX_FMT_NONE;
302 if (ctx->output_format != AV_PIX_FMT_NONE &&
303 ctx->output_format != AV_PIX_FMT_VAAPI) {
304 for (i = 0; constraints->valid_sw_formats[i] != AV_PIX_FMT_NONE; i++) {
305 if (constraints->valid_sw_formats[i] == ctx->decode_format) {
306 ctx->decode_format = ctx->output_format;
307 av_log(ctx, AV_LOG_DEBUG, "Using decode format %s (output "
308 "format).\n", av_get_pix_fmt_name(ctx->decode_format));
309 break;
310 }
311 }
312 }
313 // Otherwise, we would like to try to choose something which matches the
314 // decoder output, but there isn't enough information available here to
315 // do so. Assume for now that we are always dealing with YUV 4:2:0, so
316 // pick a format which does that.
317 if (ctx->decode_format == AV_PIX_FMT_NONE) {
318 for (i = 0; constraints->valid_sw_formats[i] != AV_PIX_FMT_NONE; i++) {
319 pix_fmt = constraints->valid_sw_formats[i];
320 pix_desc = av_pix_fmt_desc_get(pix_fmt);
321 if (pix_desc->nb_components == 3 &&
322 pix_desc->log2_chroma_w == 1 &&
323 pix_desc->log2_chroma_h == 1) {
324 ctx->decode_format = pix_fmt;
325 av_log(ctx, AV_LOG_DEBUG, "Using decode format %s (format "
326 "matched).\n", av_get_pix_fmt_name(ctx->decode_format));
327 break;
328 }
329 }
330 }
331 // Otherwise pick the first in the list and hope for the best.
332 if (ctx->decode_format == AV_PIX_FMT_NONE) {
333 ctx->decode_format = constraints->valid_sw_formats[0];
334 av_log(ctx, AV_LOG_DEBUG, "Using decode format %s (first in list).\n",
335 av_get_pix_fmt_name(ctx->decode_format));
336 if (i > 1) {
337 // There was a choice, and we picked randomly. Warn the user
338 // that they might want to choose intelligently instead.
339 av_log(ctx, AV_LOG_WARNING, "Using randomly chosen decode "
340 "format %s.\n", av_get_pix_fmt_name(ctx->decode_format));
341 }
342 }
343
344 // Ensure the picture size is supported by the hardware.
345 ctx->decode_width = avctx->coded_width;
346 ctx->decode_height = avctx->coded_height;
347 if (ctx->decode_width < constraints->min_width ||
348 ctx->decode_height < constraints->min_height ||
349 ctx->decode_width > constraints->max_width ||
350 ctx->decode_height >constraints->max_height) {
351 av_log(ctx, AV_LOG_ERROR, "VAAPI hardware does not support image "
352 "size %dx%d (constraints: width %d-%d height %d-%d).\n",
353 ctx->decode_width, ctx->decode_height,
354 constraints->min_width, constraints->max_width,
355 constraints->min_height, constraints->max_height);
356 err = AVERROR(EINVAL);
357 goto fail;
358 }
359
360 av_hwframe_constraints_free(&constraints);
361 av_freep(&hwconfig);
362
363 // Decide how many reference frames we need. This might be doable more
364 // nicely based on the codec and input stream?
365 ctx->decode_surfaces = DEFAULT_SURFACES;
366 // For frame-threaded decoding, one additional surfaces is needed for
367 // each thread.
368 if (avctx->active_thread_type & FF_THREAD_FRAME)
369 ctx->decode_surfaces += avctx->thread_count;
370
371 return 0;
372
373 fail:
374 av_hwframe_constraints_free(&constraints);
375 av_freep(&hwconfig);
376 vaDestroyConfig(hwctx->display, ctx->va_config);
377 av_freep(&profile_list);
378 return err;
379 }
380
381 static void vaapi_decode_uninit(AVCodecContext *avctx)
382 {
383 InputStream *ist = avctx->opaque;
384 VAAPIDecoderContext *ctx = ist->hwaccel_ctx;
385
386 if (ctx) {
387 AVVAAPIDeviceContext *hwctx = ctx->device->hwctx;
388
389 if (ctx->va_context != VA_INVALID_ID) {
390 vaDestroyContext(hwctx->display, ctx->va_context);
391 ctx->va_context = VA_INVALID_ID;
392 }
393 if (ctx->va_config != VA_INVALID_ID) {
394 vaDestroyConfig(hwctx->display, ctx->va_config);
395 ctx->va_config = VA_INVALID_ID;
396 }
397
398 av_buffer_unref(&ctx->frames_ref);
399 av_buffer_unref(&ctx->device_ref);
400 av_free(ctx);
401 }
402
403 av_buffer_unref(&ist->hw_frames_ctx);
404
405 ist->hwaccel_ctx = 0;
406 ist->hwaccel_uninit = 0;
407 ist->hwaccel_get_buffer = 0;
408 ist->hwaccel_retrieve_data = 0;
409 }
410
411 int vaapi_decode_init(AVCodecContext *avctx)
412 {
413 InputStream *ist = avctx->opaque;
414 AVVAAPIDeviceContext *hwctx;
415 AVVAAPIFramesContext *avfc;
416 VAAPIDecoderContext *ctx;
417 VAStatus vas;
418 int err;
419 int loglevel = (ist->hwaccel_id != HWACCEL_VAAPI ? AV_LOG_VERBOSE
420 : AV_LOG_ERROR);
421
422 if (ist->hwaccel_ctx)
423 vaapi_decode_uninit(avctx);
424
425 // We have -hwaccel without -vaapi_device, so just initialise here with
426 // the device passed as -hwaccel_device (if -vaapi_device was passed, it
427 // will always have been called before now).
428 if (!hw_device_ctx) {
429 err = vaapi_device_init(ist->hwaccel_device);
430 if (err < 0)
431 return err;
432 }
433
434 ctx = av_mallocz(sizeof(*ctx));
435 if (!ctx)
436 return AVERROR(ENOMEM);
437 ctx->class = &vaapi_class;
438
439 ctx->device_ref = av_buffer_ref(hw_device_ctx);
440 ctx->device = (AVHWDeviceContext*)ctx->device_ref->data;
441
442 ctx->va_config = VA_INVALID_ID;
443 ctx->va_context = VA_INVALID_ID;
444
445 hwctx = ctx->device->hwctx;
446
447 ctx->output_format = ist->hwaccel_output_format;
448
449 err = vaapi_build_decoder_config(ctx, avctx,
450 ist->hwaccel_id != HWACCEL_VAAPI);
451 if (err < 0) {
452 av_log(ctx, loglevel, "No supported configuration for this codec.");
453 goto fail;
454 }
455
456 avctx->pix_fmt = ctx->output_format;
457
458 ctx->frames_ref = av_hwframe_ctx_alloc(ctx->device_ref);
459 if (!ctx->frames_ref) {
460 av_log(ctx, loglevel, "Failed to create VAAPI frame context.\n");
461 err = AVERROR(ENOMEM);
462 goto fail;
463 }
464
465 ctx->frames = (AVHWFramesContext*)ctx->frames_ref->data;
466
467 ctx->frames->format = AV_PIX_FMT_VAAPI;
468 ctx->frames->sw_format = ctx->decode_format;
469 ctx->frames->width = ctx->decode_width;
470 ctx->frames->height = ctx->decode_height;
471 ctx->frames->initial_pool_size = ctx->decode_surfaces;
472
473 err = av_hwframe_ctx_init(ctx->frames_ref);
474 if (err < 0) {
475 av_log(ctx, loglevel, "Failed to initialise VAAPI frame "
476 "context: %d\n", err);
477 goto fail;
478 }
479
480 avfc = ctx->frames->hwctx;
481
482 vas = vaCreateContext(hwctx->display, ctx->va_config,
483 ctx->decode_width, ctx->decode_height,
484 VA_PROGRESSIVE,
485 avfc->surface_ids, avfc->nb_surfaces,
486 &ctx->va_context);
487 if (vas != VA_STATUS_SUCCESS) {
488 av_log(ctx, AV_LOG_ERROR, "Failed to create decode pipeline "
489 "context: %d (%s).\n", vas, vaErrorStr(vas));
490 err = AVERROR(EINVAL);
491 goto fail;
492 }
493
494 av_log(ctx, AV_LOG_DEBUG, "VAAPI decoder (re)init complete.\n");
495
496 // We would like to set this on the AVCodecContext for use by whoever gets
497 // the frames from the decoder, but unfortunately the AVCodecContext we
498 // have here need not be the "real" one (H.264 makes many copies for
499 // threading purposes). To avoid the problem, we instead store it in the
500 // InputStream and propagate it from there.
501 ist->hw_frames_ctx = av_buffer_ref(ctx->frames_ref);
502 if (!ist->hw_frames_ctx) {
503 err = AVERROR(ENOMEM);
504 goto fail;
505 }
506
507 ist->hwaccel_ctx = ctx;
508 ist->hwaccel_uninit = vaapi_decode_uninit;
509 ist->hwaccel_get_buffer = vaapi_get_buffer;
510 ist->hwaccel_retrieve_data = vaapi_retrieve_data;
511
512 ctx->decoder_vaapi_context.display = hwctx->display;
513 ctx->decoder_vaapi_context.config_id = ctx->va_config;
514 ctx->decoder_vaapi_context.context_id = ctx->va_context;
515 avctx->hwaccel_context = &ctx->decoder_vaapi_context;
516
517 return 0;
518
519 fail:
520 vaapi_decode_uninit(avctx);
521 return err;
522 }
523
524 static AVClass *vaapi_log = &vaapi_class;
525
526 static av_cold void vaapi_device_uninit(AVHWDeviceContext *hwdev)
527 {
528 AVVAAPIDeviceContext *hwctx = hwdev->hwctx;
529 av_log(&vaapi_log, AV_LOG_VERBOSE, "Terminating VAAPI connection.\n");
530 vaTerminate(hwctx->display);
531 }
532
533 av_cold int vaapi_device_init(const char *device)
534 {
535 AVHWDeviceContext *hwdev;
536 AVVAAPIDeviceContext *hwctx;
537 VADisplay display;
538 VAStatus vas;
539 int major, minor, err;
540
541 display = 0;
542
543 #if HAVE_VAAPI_X11
544 if (!display) {
545 Display *x11_display;
546
547 // Try to open the device as an X11 display.
548 x11_display = XOpenDisplay(device);
549 if (!x11_display) {
550 av_log(&vaapi_log, AV_LOG_WARNING, "Cannot open X11 display "
551 "%s.\n", XDisplayName(device));
552 } else {
553 display = vaGetDisplay(x11_display);
554 if (!display) {
555 av_log(&vaapi_log, AV_LOG_WARNING, "Cannot open a VA display "
556 "from X11 display %s.\n", XDisplayName(device));
557 XCloseDisplay(x11_display);
558 } else {
559 av_log(&vaapi_log, AV_LOG_VERBOSE, "Opened VA display via "
560 "X11 display %s.\n", XDisplayName(device));
561 }
562 }
563 }
564 #endif
565
566 #if HAVE_VAAPI_DRM
567 if (!display && device) {
568 int drm_fd;
569
570 // Try to open the device as a DRM path.
571 drm_fd = open(device, O_RDWR);
572 if (drm_fd < 0) {
573 av_log(&vaapi_log, AV_LOG_WARNING, "Cannot open DRM device %s.\n",
574 device);
575 } else {
576 display = vaGetDisplayDRM(drm_fd);
577 if (!display) {
578 av_log(&vaapi_log, AV_LOG_WARNING, "Cannot open a VA display "
579 "from DRM device %s.\n", device);
580 close(drm_fd);
581 } else {
582 av_log(&vaapi_log, AV_LOG_VERBOSE, "Opened VA display via "
583 "DRM device %s.\n", device);
584 }
585 }
586 }
587 #endif
588
589 if (!display) {
590 av_log(&vaapi_log, AV_LOG_ERROR, "No VA display found for "
591 "device %s.\n", device);
592 return AVERROR(EINVAL);
593 }
594
595 vas = vaInitialize(display, &major, &minor);
596 if (vas != VA_STATUS_SUCCESS) {
597 av_log(&vaapi_log, AV_LOG_ERROR, "Failed to initialise VAAPI "
598 "connection: %d (%s).\n", vas, vaErrorStr(vas));
599 return AVERROR(EIO);
600 }
601 av_log(&vaapi_log, AV_LOG_VERBOSE, "Initialised VAAPI connection: "
602 "version %d.%d\n", major, minor);
603
604 hw_device_ctx = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VAAPI);
605 if (!hw_device_ctx) {
606 av_log(&vaapi_log, AV_LOG_ERROR, "Failed to create VAAPI "
607 "hardware context.\n");
608 vaTerminate(display);
609 return AVERROR(ENOMEM);
610 }
611
612 hwdev = (AVHWDeviceContext*)hw_device_ctx->data;
613 hwdev->free = &vaapi_device_uninit;
614
615 hwctx = hwdev->hwctx;
616 hwctx->display = display;
617
618 err = av_hwdevice_ctx_init(hw_device_ctx);
619 if (err < 0) {
620 av_log(&vaapi_log, AV_LOG_ERROR, "Failed to initialise VAAPI "
621 "hardware context: %d\n", err);
622 return err;
623 }
624
625 return 0;
626 }