avconv: add support for VDPAU decoding
[libav.git] / avconv_vdpau.c
CommitLineData
7671dd7c
AK
1/*
2 * This file is part of Libav.
3 *
4 * Libav is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * Libav is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with Libav; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19#include <stdint.h>
20
21#include <vdpau/vdpau.h>
22#include <vdpau/vdpau_x11.h>
23
24#include <X11/Xlib.h>
25
26#include "avconv.h"
27
28#include "libavcodec/vdpau.h"
29
30#include "libavutil/avassert.h"
31#include "libavutil/buffer.h"
32#include "libavutil/frame.h"
33#include "libavutil/pixfmt.h"
34
35typedef struct VDPAUContext {
36 Display *dpy;
37
38 VdpDevice device;
39 VdpDecoder decoder;
40 VdpGetProcAddress *get_proc_address;
41
42 VdpGetErrorString *get_error_string;
43 VdpGetInformationString *get_information_string;
44 VdpDeviceDestroy *device_destroy;
45 VdpDecoderCreate *decoder_create;
46 VdpDecoderDestroy *decoder_destroy;
47 VdpDecoderRender *decoder_render;
48 VdpVideoSurfaceCreate *video_surface_create;
49 VdpVideoSurfaceDestroy *video_surface_destroy;
50 VdpVideoSurfaceGetBitsYCbCr *video_surface_get_bits;
51 VdpVideoSurfaceGetParameters *video_surface_get_parameters;
52 VdpVideoSurfaceQueryGetPutBitsYCbCrCapabilities *video_surface_query;
53
54 AVFrame *tmp_frame;
55
56 enum AVPixelFormat pix_fmt;
57 VdpYCbCrFormat vdpau_format;
58} VDPAUContext;
59
60static void vdpau_uninit(AVCodecContext *s)
61{
62 InputStream *ist = s->opaque;
63 VDPAUContext *ctx = ist->hwaccel_ctx;
64
65 ist->hwaccel_uninit = NULL;
66 ist->hwaccel_get_buffer = NULL;
67 ist->hwaccel_retrieve_data = NULL;
68
69 if (ctx->decoder_destroy)
70 ctx->decoder_destroy(ctx->decoder);
71
72 if (ctx->device_destroy)
73 ctx->device_destroy(ctx->device);
74
75 if (ctx->dpy)
76 XCloseDisplay(ctx->dpy);
77
78 av_frame_free(&ctx->tmp_frame);
79
80 av_freep(&ist->hwaccel_ctx);
81 av_freep(&s->hwaccel_context);
82}
83
84static void vdpau_release_buffer(void *opaque, uint8_t *data)
85{
86 VdpVideoSurface surface = *(VdpVideoSurface*)data;
87 VDPAUContext *ctx = opaque;
88
89 ctx->video_surface_destroy(surface);
90 av_freep(&data);
91}
92
93static int vdpau_get_buffer(AVCodecContext *s, AVFrame *frame, int flags)
94{
95 InputStream *ist = s->opaque;
96 VDPAUContext *ctx = ist->hwaccel_ctx;
97 VdpVideoSurface *surface;
98 VdpStatus err;
99
100 av_assert0(frame->format == AV_PIX_FMT_VDPAU);
101
102 surface = av_malloc(sizeof(*surface));
103 if (!surface)
104 return AVERROR(ENOMEM);
105
106 frame->buf[0] = av_buffer_create((uint8_t*)surface, sizeof(*surface),
107 vdpau_release_buffer, ctx,
108 AV_BUFFER_FLAG_READONLY);
109 if (!frame->buf[0]) {
110 av_freep(&surface);
111 return AVERROR(ENOMEM);
112 }
113
114 // properly we should keep a pool of surfaces instead of creating
115 // them anew for each frame, but since we don't care about speed
116 // much in this code, we don't bother
117 err = ctx->video_surface_create(ctx->device, VDP_CHROMA_TYPE_420,
118 frame->width, frame->height, surface);
119 if (err != VDP_STATUS_OK) {
120 av_log(NULL, AV_LOG_ERROR, "Error allocating a VDPAU video surface: %s\n",
121 ctx->get_error_string(err));
122 av_buffer_unref(&frame->buf[0]);
123 return AVERROR_UNKNOWN;
124 }
125
126 frame->data[3] = (uint8_t*)(uintptr_t)*surface;
127
128 return 0;
129}
130
131static int vdpau_retrieve_data(AVCodecContext *s, AVFrame *frame)
132{
133 VdpVideoSurface surface = (VdpVideoSurface)(uintptr_t)frame->data[3];
134 InputStream *ist = s->opaque;
135 VDPAUContext *ctx = ist->hwaccel_ctx;
136 VdpStatus err;
137 int ret, chroma_type;
138
139 err = ctx->video_surface_get_parameters(surface, &chroma_type,
140 &ctx->tmp_frame->width,
141 &ctx->tmp_frame->height);
142 if (err != VDP_STATUS_OK) {
143 av_log(NULL, AV_LOG_ERROR, "Error getting surface parameters: %s\n",
144 ctx->get_error_string(err));
145 return AVERROR_UNKNOWN;
146 }
147 ctx->tmp_frame->format = ctx->pix_fmt;
148
149 ret = av_frame_get_buffer(ctx->tmp_frame, 32);
150 if (ret < 0)
151 return ret;
152
153 ctx->tmp_frame->width = frame->width;
154 ctx->tmp_frame->height = frame->height;
155
156 err = ctx->video_surface_get_bits(surface, ctx->vdpau_format,
157 (void * const *)ctx->tmp_frame->data,
158 ctx->tmp_frame->linesize);
159 if (err != VDP_STATUS_OK) {
160 av_log(NULL, AV_LOG_ERROR, "Error retrieving frame data from VDPAU: %s\n",
161 ctx->get_error_string(err));
162 ret = AVERROR_UNKNOWN;
163 goto fail;
164 }
165
166 if (ctx->vdpau_format == VDP_YCBCR_FORMAT_YV12)
167 FFSWAP(uint8_t*, ctx->tmp_frame->data[1], ctx->tmp_frame->data[2]);
168
169 ret = av_frame_copy_props(ctx->tmp_frame, frame);
170 if (ret < 0)
171 goto fail;
172
173 av_frame_unref(frame);
174 av_frame_move_ref(frame, ctx->tmp_frame);
175 return 0;
176
177fail:
178 av_frame_unref(ctx->tmp_frame);
179 return ret;
180}
181
182static const int vdpau_formats[][2] = {
183 { VDP_YCBCR_FORMAT_YV12, AV_PIX_FMT_YUV420P },
184 { VDP_YCBCR_FORMAT_NV12, AV_PIX_FMT_NV12 },
185 { VDP_YCBCR_FORMAT_YUYV, AV_PIX_FMT_YUYV422 },
186 { VDP_YCBCR_FORMAT_UYVY, AV_PIX_FMT_UYVY422 },
187};
188
189static int vdpau_alloc(AVCodecContext *s)
190{
191 InputStream *ist = s->opaque;
192 int loglevel = (ist->hwaccel_id == HWACCEL_AUTO) ? AV_LOG_VERBOSE : AV_LOG_ERROR;
193 AVVDPAUContext *vdpau_ctx;
194 VDPAUContext *ctx;
195 const char *display, *vendor;
196 VdpStatus err;
197 int i;
198
199 ctx = av_mallocz(sizeof(*ctx));
200 if (!ctx)
201 return AVERROR(ENOMEM);
202
203 ist->hwaccel_ctx = ctx;
204 ist->hwaccel_uninit = vdpau_uninit;
205 ist->hwaccel_get_buffer = vdpau_get_buffer;
206 ist->hwaccel_retrieve_data = vdpau_retrieve_data;
207
208 ctx->tmp_frame = av_frame_alloc();
209 if (!ctx->tmp_frame)
210 goto fail;
211
212 ctx->dpy = XOpenDisplay(ist->hwaccel_device);
213 if (!ctx->dpy) {
214 av_log(NULL, loglevel, "Cannot open the X11 display %s.\n",
215 XDisplayName(ist->hwaccel_device));
216 goto fail;
217 }
218 display = XDisplayString(ctx->dpy);
219
220 err = vdp_device_create_x11(ctx->dpy, XDefaultScreen(ctx->dpy), &ctx->device,
221 &ctx->get_proc_address);
222 if (err != VDP_STATUS_OK) {
223 av_log(NULL, loglevel, "VDPAU device creation on X11 display %s failed.\n",
224 display);
225 goto fail;
226 }
227
228#define GET_CALLBACK(id, result) \
229do { \
230 void *tmp; \
231 err = ctx->get_proc_address(ctx->device, id, &tmp); \
232 if (err != VDP_STATUS_OK) { \
233 av_log(NULL, loglevel, "Error getting the " #id " callback.\n"); \
234 goto fail; \
235 } \
236 ctx->result = tmp; \
237} while (0)
238
239 GET_CALLBACK(VDP_FUNC_ID_GET_ERROR_STRING, get_error_string);
240 GET_CALLBACK(VDP_FUNC_ID_GET_INFORMATION_STRING, get_information_string);
241 GET_CALLBACK(VDP_FUNC_ID_DEVICE_DESTROY, device_destroy);
242 GET_CALLBACK(VDP_FUNC_ID_DECODER_CREATE, decoder_create);
243 GET_CALLBACK(VDP_FUNC_ID_DECODER_DESTROY, decoder_destroy);
244 GET_CALLBACK(VDP_FUNC_ID_DECODER_RENDER, decoder_render);
245 GET_CALLBACK(VDP_FUNC_ID_VIDEO_SURFACE_CREATE, video_surface_create);
246 GET_CALLBACK(VDP_FUNC_ID_VIDEO_SURFACE_DESTROY, video_surface_destroy);
247 GET_CALLBACK(VDP_FUNC_ID_VIDEO_SURFACE_GET_BITS_Y_CB_CR, video_surface_get_bits);
248 GET_CALLBACK(VDP_FUNC_ID_VIDEO_SURFACE_GET_PARAMETERS, video_surface_get_parameters);
249 GET_CALLBACK(VDP_FUNC_ID_VIDEO_SURFACE_QUERY_GET_PUT_BITS_Y_CB_CR_CAPABILITIES,
250 video_surface_query);
251
252 for (i = 0; i < FF_ARRAY_ELEMS(vdpau_formats); i++) {
253 VdpBool supported;
254 err = ctx->video_surface_query(ctx->device, VDP_CHROMA_TYPE_420,
255 vdpau_formats[i][0], &supported);
256 if (err != VDP_STATUS_OK) {
257 av_log(NULL, loglevel,
258 "Error querying VDPAU surface capabilities: %s\n",
259 ctx->get_error_string(err));
260 goto fail;
261 }
262 if (supported)
263 break;
264 }
265 if (i == FF_ARRAY_ELEMS(vdpau_formats)) {
266 av_log(NULL, loglevel,
267 "No supported VDPAU format for retrieving the data.\n");
268 return AVERROR(EINVAL);
269 }
270 ctx->vdpau_format = vdpau_formats[i][0];
271 ctx->pix_fmt = vdpau_formats[i][1];
272
273 vdpau_ctx = av_vdpau_alloc_context();
274 if (!vdpau_ctx)
275 goto fail;
276 vdpau_ctx->render = ctx->decoder_render;
277
278 s->hwaccel_context = vdpau_ctx;
279
280 ctx->get_information_string(&vendor);
281 av_log(NULL, AV_LOG_VERBOSE, "Using VDPAU -- %s -- on X11 display %s, "
282 "to decode input stream #%d:%d.\n", vendor,
283 display, ist->file_index, ist->st->index);
284
285 return 0;
286
287fail:
288 av_log(NULL, loglevel, "VDPAU init failed for stream #%d:%d.\n",
289 ist->file_index, ist->st->index);
290 vdpau_uninit(s);
291 return AVERROR(EINVAL);
292}
293
294int vdpau_init(AVCodecContext *s)
295{
296 InputStream *ist = s->opaque;
297 int loglevel = (ist->hwaccel_id == HWACCEL_AUTO) ? AV_LOG_VERBOSE : AV_LOG_ERROR;
298 AVVDPAUContext *vdpau_ctx;
299 VDPAUContext *ctx;
300 VdpStatus err;
301 int profile, ret;
302
303 if (!ist->hwaccel_ctx) {
304 ret = vdpau_alloc(s);
305 if (ret < 0)
306 return ret;
307 }
308 ctx = ist->hwaccel_ctx;
309 vdpau_ctx = s->hwaccel_context;
310
311 ret = av_vdpau_get_profile(s, &profile);
312 if (ret < 0) {
313 av_log(NULL, loglevel, "No known VDPAU decoder profile for this stream.\n");
314 return AVERROR(EINVAL);
315 }
316
317 if (ctx->decoder)
318 ctx->decoder_destroy(ctx->decoder);
319
320 err = ctx->decoder_create(ctx->device, profile,
321 s->coded_width, s->coded_height,
322 16, &ctx->decoder);
323 if (err != VDP_STATUS_OK) {
324 av_log(NULL, loglevel, "Error creating the VDPAU decoder: %s\n",
325 ctx->get_error_string(err));
326 return AVERROR_UNKNOWN;
327 }
328
329 vdpau_ctx->decoder = ctx->decoder;
330
331 ist->hwaccel_get_buffer = vdpau_get_buffer;
332 ist->hwaccel_retrieve_data = vdpau_retrieve_data;
333
334 return 0;
335}