avconv_vdpau: update to new VDPAU interface
[libav.git] / avconv_vdpau.c
1 /*
2 * This file is part of Libav.
3 *
4 * Libav is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * Libav is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with Libav; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19 #include <stdint.h>
20
21 #include <vdpau/vdpau.h>
22 #include <vdpau/vdpau_x11.h>
23
24 #include <X11/Xlib.h>
25
26 #include "avconv.h"
27
28 #include "libavcodec/vdpau.h"
29
30 #include "libavutil/avassert.h"
31 #include "libavutil/buffer.h"
32 #include "libavutil/frame.h"
33 #include "libavutil/pixfmt.h"
34
35 typedef struct VDPAUContext {
36 Display *dpy;
37
38 VdpDevice device;
39 VdpDecoder decoder;
40 VdpGetProcAddress *get_proc_address;
41
42 VdpGetErrorString *get_error_string;
43 VdpGetInformationString *get_information_string;
44 VdpDeviceDestroy *device_destroy;
45 VdpVideoSurfaceCreate *video_surface_create;
46 VdpVideoSurfaceDestroy *video_surface_destroy;
47 VdpVideoSurfaceGetBitsYCbCr *video_surface_get_bits;
48 VdpVideoSurfaceGetParameters *video_surface_get_parameters;
49 VdpVideoSurfaceQueryGetPutBitsYCbCrCapabilities *video_surface_query;
50
51 AVFrame *tmp_frame;
52
53 enum AVPixelFormat pix_fmt;
54 VdpYCbCrFormat vdpau_format;
55 } VDPAUContext;
56
57 static void vdpau_uninit(AVCodecContext *s)
58 {
59 InputStream *ist = s->opaque;
60 VDPAUContext *ctx = ist->hwaccel_ctx;
61
62 ist->hwaccel_uninit = NULL;
63 ist->hwaccel_get_buffer = NULL;
64 ist->hwaccel_retrieve_data = NULL;
65
66 if (ctx->device_destroy)
67 ctx->device_destroy(ctx->device);
68
69 if (ctx->dpy)
70 XCloseDisplay(ctx->dpy);
71
72 av_frame_free(&ctx->tmp_frame);
73
74 av_freep(&ist->hwaccel_ctx);
75 av_freep(&s->hwaccel_context);
76 }
77
78 static void vdpau_release_buffer(void *opaque, uint8_t *data)
79 {
80 VdpVideoSurface surface = *(VdpVideoSurface*)data;
81 VDPAUContext *ctx = opaque;
82
83 ctx->video_surface_destroy(surface);
84 av_freep(&data);
85 }
86
87 static int vdpau_get_buffer(AVCodecContext *s, AVFrame *frame, int flags)
88 {
89 InputStream *ist = s->opaque;
90 VDPAUContext *ctx = ist->hwaccel_ctx;
91 VdpVideoSurface *surface;
92 VdpStatus err;
93
94 av_assert0(frame->format == AV_PIX_FMT_VDPAU);
95
96 surface = av_malloc(sizeof(*surface));
97 if (!surface)
98 return AVERROR(ENOMEM);
99
100 frame->buf[0] = av_buffer_create((uint8_t*)surface, sizeof(*surface),
101 vdpau_release_buffer, ctx,
102 AV_BUFFER_FLAG_READONLY);
103 if (!frame->buf[0]) {
104 av_freep(&surface);
105 return AVERROR(ENOMEM);
106 }
107
108 // properly we should keep a pool of surfaces instead of creating
109 // them anew for each frame, but since we don't care about speed
110 // much in this code, we don't bother
111 err = ctx->video_surface_create(ctx->device, VDP_CHROMA_TYPE_420,
112 frame->width, frame->height, surface);
113 if (err != VDP_STATUS_OK) {
114 av_log(NULL, AV_LOG_ERROR, "Error allocating a VDPAU video surface: %s\n",
115 ctx->get_error_string(err));
116 av_buffer_unref(&frame->buf[0]);
117 return AVERROR_UNKNOWN;
118 }
119
120 frame->data[3] = (uint8_t*)(uintptr_t)*surface;
121
122 return 0;
123 }
124
125 static int vdpau_retrieve_data(AVCodecContext *s, AVFrame *frame)
126 {
127 VdpVideoSurface surface = (VdpVideoSurface)(uintptr_t)frame->data[3];
128 InputStream *ist = s->opaque;
129 VDPAUContext *ctx = ist->hwaccel_ctx;
130 VdpStatus err;
131 int ret, chroma_type;
132
133 err = ctx->video_surface_get_parameters(surface, &chroma_type,
134 &ctx->tmp_frame->width,
135 &ctx->tmp_frame->height);
136 if (err != VDP_STATUS_OK) {
137 av_log(NULL, AV_LOG_ERROR, "Error getting surface parameters: %s\n",
138 ctx->get_error_string(err));
139 return AVERROR_UNKNOWN;
140 }
141 ctx->tmp_frame->format = ctx->pix_fmt;
142
143 ret = av_frame_get_buffer(ctx->tmp_frame, 32);
144 if (ret < 0)
145 return ret;
146
147 ctx->tmp_frame->width = frame->width;
148 ctx->tmp_frame->height = frame->height;
149
150 err = ctx->video_surface_get_bits(surface, ctx->vdpau_format,
151 (void * const *)ctx->tmp_frame->data,
152 ctx->tmp_frame->linesize);
153 if (err != VDP_STATUS_OK) {
154 av_log(NULL, AV_LOG_ERROR, "Error retrieving frame data from VDPAU: %s\n",
155 ctx->get_error_string(err));
156 ret = AVERROR_UNKNOWN;
157 goto fail;
158 }
159
160 if (ctx->vdpau_format == VDP_YCBCR_FORMAT_YV12)
161 FFSWAP(uint8_t*, ctx->tmp_frame->data[1], ctx->tmp_frame->data[2]);
162
163 ret = av_frame_copy_props(ctx->tmp_frame, frame);
164 if (ret < 0)
165 goto fail;
166
167 av_frame_unref(frame);
168 av_frame_move_ref(frame, ctx->tmp_frame);
169 return 0;
170
171 fail:
172 av_frame_unref(ctx->tmp_frame);
173 return ret;
174 }
175
176 static const int vdpau_formats[][2] = {
177 { VDP_YCBCR_FORMAT_YV12, AV_PIX_FMT_YUV420P },
178 { VDP_YCBCR_FORMAT_NV12, AV_PIX_FMT_NV12 },
179 { VDP_YCBCR_FORMAT_YUYV, AV_PIX_FMT_YUYV422 },
180 { VDP_YCBCR_FORMAT_UYVY, AV_PIX_FMT_UYVY422 },
181 };
182
183 static int vdpau_alloc(AVCodecContext *s)
184 {
185 InputStream *ist = s->opaque;
186 int loglevel = (ist->hwaccel_id == HWACCEL_AUTO) ? AV_LOG_VERBOSE : AV_LOG_ERROR;
187 VDPAUContext *ctx;
188 const char *display, *vendor;
189 VdpStatus err;
190 int i;
191
192 ctx = av_mallocz(sizeof(*ctx));
193 if (!ctx)
194 return AVERROR(ENOMEM);
195
196 ist->hwaccel_ctx = ctx;
197 ist->hwaccel_uninit = vdpau_uninit;
198 ist->hwaccel_get_buffer = vdpau_get_buffer;
199 ist->hwaccel_retrieve_data = vdpau_retrieve_data;
200
201 ctx->tmp_frame = av_frame_alloc();
202 if (!ctx->tmp_frame)
203 goto fail;
204
205 ctx->dpy = XOpenDisplay(ist->hwaccel_device);
206 if (!ctx->dpy) {
207 av_log(NULL, loglevel, "Cannot open the X11 display %s.\n",
208 XDisplayName(ist->hwaccel_device));
209 goto fail;
210 }
211 display = XDisplayString(ctx->dpy);
212
213 err = vdp_device_create_x11(ctx->dpy, XDefaultScreen(ctx->dpy), &ctx->device,
214 &ctx->get_proc_address);
215 if (err != VDP_STATUS_OK) {
216 av_log(NULL, loglevel, "VDPAU device creation on X11 display %s failed.\n",
217 display);
218 goto fail;
219 }
220
221 #define GET_CALLBACK(id, result) \
222 do { \
223 void *tmp; \
224 err = ctx->get_proc_address(ctx->device, id, &tmp); \
225 if (err != VDP_STATUS_OK) { \
226 av_log(NULL, loglevel, "Error getting the " #id " callback.\n"); \
227 goto fail; \
228 } \
229 ctx->result = tmp; \
230 } while (0)
231
232 GET_CALLBACK(VDP_FUNC_ID_GET_ERROR_STRING, get_error_string);
233 GET_CALLBACK(VDP_FUNC_ID_GET_INFORMATION_STRING, get_information_string);
234 GET_CALLBACK(VDP_FUNC_ID_DEVICE_DESTROY, device_destroy);
235 GET_CALLBACK(VDP_FUNC_ID_VIDEO_SURFACE_CREATE, video_surface_create);
236 GET_CALLBACK(VDP_FUNC_ID_VIDEO_SURFACE_DESTROY, video_surface_destroy);
237 GET_CALLBACK(VDP_FUNC_ID_VIDEO_SURFACE_GET_BITS_Y_CB_CR, video_surface_get_bits);
238 GET_CALLBACK(VDP_FUNC_ID_VIDEO_SURFACE_GET_PARAMETERS, video_surface_get_parameters);
239 GET_CALLBACK(VDP_FUNC_ID_VIDEO_SURFACE_QUERY_GET_PUT_BITS_Y_CB_CR_CAPABILITIES,
240 video_surface_query);
241
242 for (i = 0; i < FF_ARRAY_ELEMS(vdpau_formats); i++) {
243 VdpBool supported;
244 err = ctx->video_surface_query(ctx->device, VDP_CHROMA_TYPE_420,
245 vdpau_formats[i][0], &supported);
246 if (err != VDP_STATUS_OK) {
247 av_log(NULL, loglevel,
248 "Error querying VDPAU surface capabilities: %s\n",
249 ctx->get_error_string(err));
250 goto fail;
251 }
252 if (supported)
253 break;
254 }
255 if (i == FF_ARRAY_ELEMS(vdpau_formats)) {
256 av_log(NULL, loglevel,
257 "No supported VDPAU format for retrieving the data.\n");
258 return AVERROR(EINVAL);
259 }
260 ctx->vdpau_format = vdpau_formats[i][0];
261 ctx->pix_fmt = vdpau_formats[i][1];
262
263 if (av_vdpau_bind_context(s, ctx->device, ctx->get_proc_address, 0))
264 goto fail;
265
266 ctx->get_information_string(&vendor);
267 av_log(NULL, AV_LOG_VERBOSE, "Using VDPAU -- %s -- on X11 display %s, "
268 "to decode input stream #%d:%d.\n", vendor,
269 display, ist->file_index, ist->st->index);
270
271 return 0;
272
273 fail:
274 av_log(NULL, loglevel, "VDPAU init failed for stream #%d:%d.\n",
275 ist->file_index, ist->st->index);
276 vdpau_uninit(s);
277 return AVERROR(EINVAL);
278 }
279
280 int vdpau_init(AVCodecContext *s)
281 {
282 InputStream *ist = s->opaque;
283
284 if (!ist->hwaccel_ctx) {
285 int ret = vdpau_alloc(s);
286 if (ret < 0)
287 return ret;
288 }
289
290 ist->hwaccel_get_buffer = vdpau_get_buffer;
291 ist->hwaccel_retrieve_data = vdpau_retrieve_data;
292
293 return 0;
294 }