hwcontext_d3d11va: fix crash on frames_init failure
[libav.git] / libavutil / hwcontext_d3d11va.c
CommitLineData
fff90422 1/*
2 * This file is part of Libav.
3 *
4 * Libav is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * Libav is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with Libav; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19#include <windows.h>
20
fd1ffa1f
MS
21// Include thread.h before redefining _WIN32_WINNT, to get
22// the right implementation for AVOnce
23#include "thread.h"
24
fff90422 25#if !defined(_WIN32_WINNT) || _WIN32_WINNT < 0x0600
26#undef _WIN32_WINNT
27#define _WIN32_WINNT 0x0600
28#endif
29#define COBJMACROS
30
31#include <initguid.h>
32#include <d3d11.h>
33#include <dxgi1_2.h>
34
35#include "avassert.h"
36#include "common.h"
37#include "hwcontext.h"
38#include "hwcontext_d3d11va.h"
39#include "hwcontext_internal.h"
40#include "imgutils.h"
41#include "pixdesc.h"
42#include "pixfmt.h"
43
44typedef HRESULT(WINAPI *PFN_CREATE_DXGI_FACTORY)(REFIID riid, void **ppFactory);
45
fd1ffa1f
MS
46static AVOnce functions_loaded = AV_ONCE_INIT;
47
48static PFN_CREATE_DXGI_FACTORY mCreateDXGIFactory;
49static PFN_D3D11_CREATE_DEVICE mD3D11CreateDevice;
50
51static av_cold void load_functions(void)
52{
53#if HAVE_LOADLIBRARY
54 // We let these "leak" - this is fine, as unloading has no great benefit, and
55 // Windows will mark a DLL as loaded forever if its internal refcount overflows
56 // from too many LoadLibrary calls.
57 HANDLE d3dlib, dxgilib;
58
59 d3dlib = LoadLibrary("d3d11.dll");
60 dxgilib = LoadLibrary("dxgi.dll");
61 if (!d3dlib || !dxgilib)
62 return;
63
64 mD3D11CreateDevice = (PFN_D3D11_CREATE_DEVICE) GetProcAddress(d3dlib, "D3D11CreateDevice");
65 mCreateDXGIFactory = (PFN_CREATE_DXGI_FACTORY) GetProcAddress(dxgilib, "CreateDXGIFactory");
66#else
67 // In UWP (which lacks LoadLibrary), CreateDXGIFactory isn't available,
68 // only CreateDXGIFactory1
69 mD3D11CreateDevice = (PFN_D3D11_CREATE_DEVICE) D3D11CreateDevice;
70 mCreateDXGIFactory = (PFN_CREATE_DXGI_FACTORY) CreateDXGIFactory1;
71#endif
72}
73
fff90422 74typedef struct D3D11VAFramesContext {
75 int nb_surfaces_used;
76
77 DXGI_FORMAT format;
78
79 ID3D11Texture2D *staging_texture;
80} D3D11VAFramesContext;
81
82static const struct {
83 DXGI_FORMAT d3d_format;
84 enum AVPixelFormat pix_fmt;
85} supported_formats[] = {
86 { DXGI_FORMAT_NV12, AV_PIX_FMT_NV12 },
87 { DXGI_FORMAT_P010, AV_PIX_FMT_P010 },
88};
89
90static void d3d11va_default_lock(void *ctx)
91{
92 WaitForSingleObjectEx(ctx, INFINITE, FALSE);
93}
94
95static void d3d11va_default_unlock(void *ctx)
96{
97 ReleaseMutex(ctx);
98}
99
100static void d3d11va_frames_uninit(AVHWFramesContext *ctx)
101{
102 AVD3D11VAFramesContext *frames_hwctx = ctx->hwctx;
103 D3D11VAFramesContext *s = ctx->internal->priv;
104
105 if (frames_hwctx->texture)
106 ID3D11Texture2D_Release(frames_hwctx->texture);
086321c6 107 frames_hwctx->texture = NULL;
fff90422 108
109 if (s->staging_texture)
110 ID3D11Texture2D_Release(s->staging_texture);
086321c6 111 s->staging_texture = NULL;
fff90422 112}
113
114static void free_texture(void *opaque, uint8_t *data)
115{
116 ID3D11Texture2D_Release((ID3D11Texture2D *)opaque);
117}
118
119static AVBufferRef *wrap_texture_buf(ID3D11Texture2D *tex, int index)
120{
121 AVBufferRef *buf;
122 AVD3D11FrameDescriptor *desc = av_mallocz(sizeof(*desc));
123 if (!desc) {
124 ID3D11Texture2D_Release(tex);
125 return NULL;
126 }
127
128 desc->texture = tex;
129 desc->index = index;
130
131 buf = av_buffer_create((uint8_t *)desc, sizeof(desc), free_texture, tex, 0);
132 if (!buf) {
133 ID3D11Texture2D_Release(tex);
134 av_free(desc);
135 return NULL;
136 }
137
138 return buf;
139}
140
141static AVBufferRef *d3d11va_alloc_single(AVHWFramesContext *ctx)
142{
143 D3D11VAFramesContext *s = ctx->internal->priv;
144 AVD3D11VAFramesContext *hwctx = ctx->hwctx;
145 AVD3D11VADeviceContext *device_hwctx = ctx->device_ctx->hwctx;
146 HRESULT hr;
147 ID3D11Texture2D *tex;
148 D3D11_TEXTURE2D_DESC texDesc = {
149 .Width = ctx->width,
150 .Height = ctx->height,
151 .MipLevels = 1,
152 .Format = s->format,
153 .SampleDesc = { .Count = 1 },
154 .ArraySize = 1,
155 .Usage = D3D11_USAGE_DEFAULT,
156 .BindFlags = hwctx->BindFlags,
157 .MiscFlags = hwctx->MiscFlags,
158 };
159
160 hr = ID3D11Device_CreateTexture2D(device_hwctx->device, &texDesc, NULL, &tex);
161 if (FAILED(hr)) {
162 av_log(ctx, AV_LOG_ERROR, "Could not create the texture (%lx)\n", (long)hr);
163 return NULL;
164 }
165
166 return wrap_texture_buf(tex, 0);
167}
168
169static AVBufferRef *d3d11va_pool_alloc(void *opaque, int size)
170{
171 AVHWFramesContext *ctx = (AVHWFramesContext*)opaque;
172 D3D11VAFramesContext *s = ctx->internal->priv;
173 AVD3D11VAFramesContext *hwctx = ctx->hwctx;
174 D3D11_TEXTURE2D_DESC texDesc;
175
176 if (!hwctx->texture)
177 return d3d11va_alloc_single(ctx);
178
179 ID3D11Texture2D_GetDesc(hwctx->texture, &texDesc);
180
181 if (s->nb_surfaces_used >= texDesc.ArraySize) {
182 av_log(ctx, AV_LOG_ERROR, "Static surface pool size exceeded.\n");
183 return NULL;
184 }
185
186 ID3D11Texture2D_AddRef(hwctx->texture);
187 return wrap_texture_buf(hwctx->texture, s->nb_surfaces_used++);
188}
189
190static int d3d11va_frames_init(AVHWFramesContext *ctx)
191{
192 AVD3D11VAFramesContext *hwctx = ctx->hwctx;
193 AVD3D11VADeviceContext *device_hwctx = ctx->device_ctx->hwctx;
194 D3D11VAFramesContext *s = ctx->internal->priv;
195
196 int i;
197 HRESULT hr;
198 D3D11_TEXTURE2D_DESC texDesc;
199
200 for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) {
201 if (ctx->sw_format == supported_formats[i].pix_fmt) {
202 s->format = supported_formats[i].d3d_format;
203 break;
204 }
205 }
206 if (i == FF_ARRAY_ELEMS(supported_formats)) {
207 av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n",
208 av_get_pix_fmt_name(ctx->sw_format));
209 return AVERROR(EINVAL);
210 }
211
212 texDesc = (D3D11_TEXTURE2D_DESC){
213 .Width = ctx->width,
214 .Height = ctx->height,
215 .MipLevels = 1,
216 .Format = s->format,
217 .SampleDesc = { .Count = 1 },
218 .ArraySize = ctx->initial_pool_size,
219 .Usage = D3D11_USAGE_DEFAULT,
220 .BindFlags = hwctx->BindFlags,
221 .MiscFlags = hwctx->MiscFlags,
222 };
223
224 if (hwctx->texture) {
225 D3D11_TEXTURE2D_DESC texDesc2;
226 ID3D11Texture2D_GetDesc(hwctx->texture, &texDesc2);
227
228 if (texDesc.Width != texDesc2.Width ||
229 texDesc.Height != texDesc2.Height ||
230 texDesc.Format != texDesc2.Format) {
231 av_log(ctx, AV_LOG_ERROR, "User-provided texture has mismatching parameters\n");
232 return AVERROR(EINVAL);
233 }
234 } else if (texDesc.ArraySize > 0) {
235 hr = ID3D11Device_CreateTexture2D(device_hwctx->device, &texDesc, NULL, &hwctx->texture);
236 if (FAILED(hr)) {
237 av_log(ctx, AV_LOG_ERROR, "Could not create the texture (%lx)\n", (long)hr);
238 return AVERROR_UNKNOWN;
239 }
240 }
241
242 texDesc.ArraySize = 1;
243 texDesc.Usage = D3D11_USAGE_STAGING;
244 texDesc.BindFlags = 0;
245 texDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ | D3D11_CPU_ACCESS_WRITE;
246 texDesc.MiscFlags = 0;
247 hr = ID3D11Device_CreateTexture2D(device_hwctx->device, &texDesc, NULL, &s->staging_texture);
248 if (FAILED(hr)) {
249 av_log(ctx, AV_LOG_ERROR, "Could not create the staging texture (%lx)\n", (long)hr);
250 return AVERROR_UNKNOWN;
251 }
252
253 ctx->internal->pool_internal = av_buffer_pool_init2(sizeof(AVD3D11FrameDescriptor),
254 ctx, d3d11va_pool_alloc, NULL);
255 if (!ctx->internal->pool_internal)
256 return AVERROR(ENOMEM);
257
258 return 0;
259}
260
261static int d3d11va_get_buffer(AVHWFramesContext *ctx, AVFrame *frame)
262{
263 AVD3D11FrameDescriptor *desc;
264
265 frame->buf[0] = av_buffer_pool_get(ctx->pool);
266 if (!frame->buf[0])
267 return AVERROR(ENOMEM);
268
269 desc = (AVD3D11FrameDescriptor *)frame->buf[0]->data;
270
271 frame->data[0] = (uint8_t *)desc->texture;
272 frame->data[1] = (uint8_t *)desc->index;
273 frame->format = AV_PIX_FMT_D3D11;
274 frame->width = ctx->width;
275 frame->height = ctx->height;
276
277 return 0;
278}
279
280static int d3d11va_transfer_get_formats(AVHWFramesContext *ctx,
281 enum AVHWFrameTransferDirection dir,
282 enum AVPixelFormat **formats)
283{
284 enum AVPixelFormat *fmts;
285
286 fmts = av_malloc_array(2, sizeof(*fmts));
287 if (!fmts)
288 return AVERROR(ENOMEM);
289
290 fmts[0] = ctx->sw_format;
291 fmts[1] = AV_PIX_FMT_NONE;
292
293 *formats = fmts;
294
295 return 0;
296}
297
298static void fill_texture_ptrs(uint8_t *data[4], int linesize[4],
299 AVHWFramesContext *ctx,
300 D3D11_TEXTURE2D_DESC *desc,
301 D3D11_MAPPED_SUBRESOURCE *map)
302{
303 int i;
304
305 for (i = 0; i < 4; i++)
306 linesize[i] = map->RowPitch;
307
308 av_image_fill_pointers(data, ctx->sw_format, desc->Height,
309 (uint8_t*)map->pData, linesize);
310}
311
312static int d3d11va_transfer_data(AVHWFramesContext *ctx, AVFrame *dst,
313 const AVFrame *src)
314{
315 AVD3D11VADeviceContext *device_hwctx = ctx->device_ctx->hwctx;
316 D3D11VAFramesContext *s = ctx->internal->priv;
317 int download = src->format == AV_PIX_FMT_D3D11;
318 const AVFrame *frame = download ? src : dst;
319 const AVFrame *other = download ? dst : src;
320 // (The interface types are compatible.)
321 ID3D11Resource *texture = (ID3D11Resource *)(ID3D11Texture2D *)frame->data[0];
322 int index = (intptr_t)frame->data[1];
323 ID3D11Resource *staging = (ID3D11Resource *)s->staging_texture;
324 int w = FFMIN(dst->width, src->width);
325 int h = FFMIN(dst->height, src->height);
326 uint8_t *map_data[4];
327 int map_linesize[4];
328 D3D11_TEXTURE2D_DESC desc;
329 D3D11_MAPPED_SUBRESOURCE map;
330 HRESULT hr;
331
332 if (frame->hw_frames_ctx->data != (uint8_t *)ctx || other->format != ctx->sw_format)
333 return AVERROR(EINVAL);
334
335 device_hwctx->lock(device_hwctx->lock_ctx);
336
337 ID3D11Texture2D_GetDesc(s->staging_texture, &desc);
338
339 if (download) {
340 ID3D11DeviceContext_CopySubresourceRegion(device_hwctx->device_context,
341 staging, 0, 0, 0, 0,
342 texture, index, NULL);
343
344 hr = ID3D11DeviceContext_Map(device_hwctx->device_context,
345 staging, 0, D3D11_MAP_READ, 0, &map);
346 if (FAILED(hr))
347 goto map_failed;
348
349 fill_texture_ptrs(map_data, map_linesize, ctx, &desc, &map);
350
351 av_image_copy(dst->data, dst->linesize, map_data, map_linesize,
352 ctx->sw_format, w, h);
353
354 ID3D11DeviceContext_Unmap(device_hwctx->device_context, staging, 0);
355 } else {
356 hr = ID3D11DeviceContext_Map(device_hwctx->device_context,
357 staging, 0, D3D11_MAP_WRITE, 0, &map);
358 if (FAILED(hr))
359 goto map_failed;
360
361 fill_texture_ptrs(map_data, map_linesize, ctx, &desc, &map);
362
363 av_image_copy(map_data, map_linesize, src->data, src->linesize,
364 ctx->sw_format, w, h);
365
366 ID3D11DeviceContext_Unmap(device_hwctx->device_context, staging, 0);
367
368 ID3D11DeviceContext_CopySubresourceRegion(device_hwctx->device_context,
369 texture, index, 0, 0, 0,
370 staging, 0, NULL);
371 }
372
373 device_hwctx->unlock(device_hwctx->lock_ctx);
374 return 0;
375
376map_failed:
377 av_log(ctx, AV_LOG_ERROR, "Unable to lock D3D11VA surface (%lx)\n", (long)hr);
378 device_hwctx->unlock(device_hwctx->lock_ctx);
379 return AVERROR_UNKNOWN;
380}
381
382static int d3d11va_device_init(AVHWDeviceContext *hwdev)
383{
384 AVD3D11VADeviceContext *device_hwctx = hwdev->hwctx;
385 HRESULT hr;
386
387 if (!device_hwctx->lock) {
388 device_hwctx->lock_ctx = CreateMutex(NULL, 0, NULL);
389 if (device_hwctx->lock_ctx == INVALID_HANDLE_VALUE) {
390 av_log(NULL, AV_LOG_ERROR, "Failed to create a mutex\n");
391 return AVERROR(EINVAL);
392 }
393 device_hwctx->lock = d3d11va_default_lock;
394 device_hwctx->unlock = d3d11va_default_unlock;
395 }
396
397 if (!device_hwctx->device_context) {
398 ID3D11Device_GetImmediateContext(device_hwctx->device, &device_hwctx->device_context);
399 if (!device_hwctx->device_context)
400 return AVERROR_UNKNOWN;
401 }
402
403 if (!device_hwctx->video_device) {
404 hr = ID3D11DeviceContext_QueryInterface(device_hwctx->device, &IID_ID3D11VideoDevice,
405 (void **)&device_hwctx->video_device);
406 if (FAILED(hr))
407 return AVERROR_UNKNOWN;
408 }
409
410 if (!device_hwctx->video_context) {
411 hr = ID3D11DeviceContext_QueryInterface(device_hwctx->device_context, &IID_ID3D11VideoContext,
412 (void **)&device_hwctx->video_context);
413 if (FAILED(hr))
414 return AVERROR_UNKNOWN;
415 }
416
417 return 0;
418}
419
420static void d3d11va_device_uninit(AVHWDeviceContext *hwdev)
421{
422 AVD3D11VADeviceContext *device_hwctx = hwdev->hwctx;
423
424 if (device_hwctx->device)
425 ID3D11Device_Release(device_hwctx->device);
426
427 if (device_hwctx->device_context)
428 ID3D11DeviceContext_Release(device_hwctx->device_context);
429
430 if (device_hwctx->video_device)
431 ID3D11VideoDevice_Release(device_hwctx->video_device);
432
433 if (device_hwctx->video_context)
434 ID3D11VideoContext_Release(device_hwctx->video_context);
435
436 if (device_hwctx->lock == d3d11va_default_lock)
437 CloseHandle(device_hwctx->lock_ctx);
438}
439
440static int d3d11va_device_create(AVHWDeviceContext *ctx, const char *device,
441 AVDictionary *opts, int flags)
442{
443 AVD3D11VADeviceContext *device_hwctx = ctx->hwctx;
fff90422 444
445 HRESULT hr;
fff90422 446 IDXGIAdapter *pAdapter = NULL;
447 ID3D10Multithread *pMultithread;
448 UINT creationFlags = D3D11_CREATE_DEVICE_VIDEO_SUPPORT;
fd1ffa1f 449 int ret;
fff90422 450
fd1ffa1f
MS
451 if ((ret = ff_thread_once(&functions_loaded, load_functions)) != 0)
452 return AVERROR_UNKNOWN;
453 if (!mD3D11CreateDevice || !mCreateDXGIFactory) {
454 av_log(ctx, AV_LOG_ERROR, "Failed to load D3D11 library or its functions\n");
fff90422 455 return AVERROR_UNKNOWN;
456 }
457
fd1ffa1f
MS
458 if (device) {
459 IDXGIFactory2 *pDXGIFactory;
460 hr = mCreateDXGIFactory(&IID_IDXGIFactory2, (void **)&pDXGIFactory);
461 if (SUCCEEDED(hr)) {
462 int adapter = atoi(device);
463 if (FAILED(IDXGIFactory2_EnumAdapters(pDXGIFactory, adapter, &pAdapter)))
464 pAdapter = NULL;
465 IDXGIFactory2_Release(pDXGIFactory);
466 }
fff90422 467 }
468
fd1ffa1f 469 hr = mD3D11CreateDevice(pAdapter, pAdapter ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE, NULL, creationFlags, NULL, 0,
fff90422 470 D3D11_SDK_VERSION, &device_hwctx->device, NULL, NULL);
471 if (pAdapter)
472 IDXGIAdapter_Release(pAdapter);
473 if (FAILED(hr)) {
474 av_log(ctx, AV_LOG_ERROR, "Failed to create Direct3D device (%lx)\n", (long)hr);
475 return AVERROR_UNKNOWN;
476 }
477
478 hr = ID3D11Device_QueryInterface(device_hwctx->device, &IID_ID3D10Multithread, (void **)&pMultithread);
479 if (SUCCEEDED(hr)) {
480 ID3D10Multithread_SetMultithreadProtected(pMultithread, TRUE);
481 ID3D10Multithread_Release(pMultithread);
482 }
483
484 return 0;
485}
486
487const HWContextType ff_hwcontext_type_d3d11va = {
488 .type = AV_HWDEVICE_TYPE_D3D11VA,
489 .name = "D3D11VA",
490
491 .device_hwctx_size = sizeof(AVD3D11VADeviceContext),
492 .frames_hwctx_size = sizeof(AVD3D11VAFramesContext),
493 .frames_priv_size = sizeof(D3D11VAFramesContext),
494
495 .device_create = d3d11va_device_create,
496 .device_init = d3d11va_device_init,
497 .device_uninit = d3d11va_device_uninit,
498 .frames_init = d3d11va_frames_init,
499 .frames_uninit = d3d11va_frames_uninit,
500 .frames_get_buffer = d3d11va_get_buffer,
501 .transfer_get_formats = d3d11va_transfer_get_formats,
502 .transfer_data_to = d3d11va_transfer_data,
503 .transfer_data_from = d3d11va_transfer_data,
504
505 .pix_fmts = (const enum AVPixelFormat[]){ AV_PIX_FMT_D3D11, AV_PIX_FMT_NONE },
506};