739962a38213e41e399838a09b5d5f6428afd929
[libav.git] / libavcodec / vmnc.c
1 /*
2 * VMware Screen Codec (VMnc) decoder
3 * Copyright (c) 2006 Konstantin Shishkov
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 *
19 */
20
21 /**
22 * @file vmnc.c
23 * VMware Screen Codec (VMnc) decoder
24 * As Alex Beregszaszi discovered, this is effectively RFB data dump
25 */
26
27 #include <stdio.h>
28 #include <stdlib.h>
29
30 #include "common.h"
31 #include "avcodec.h"
32
33 enum EncTypes {
34 MAGIC_WMVd = 0x574D5664,
35 MAGIC_WMVe,
36 MAGIC_WMVf,
37 MAGIC_WMVg,
38 MAGIC_WMVh,
39 MAGIC_WMVi,
40 MAGIC_WMVj
41 };
42
43 enum HexTile_Flags {
44 HT_RAW = 1, // tile is raw
45 HT_BKG = 2, // background color is present
46 HT_FG = 4, // foreground color is present
47 HT_SUB = 8, // subrects are present
48 HT_CLR = 16 // each subrect has own color
49 };
50
51 /*
52 * Decoder context
53 */
54 typedef struct VmncContext {
55 AVCodecContext *avctx;
56 AVFrame pic;
57
58 int bpp;
59 int bpp2;
60 int bigendian;
61 uint8_t pal[768];
62 int width, height;
63
64 /* cursor data */
65 int cur_w, cur_h;
66 int cur_x, cur_y;
67 int cur_hx, cur_hy;
68 uint8_t* curbits, *curmask;
69 uint8_t* screendta;
70 } VmncContext;
71
72 /* read pixel value from stream */
73 static always_inline int vmnc_get_pixel(uint8_t* buf, int bpp, int be) {
74 switch(bpp * 2 + be) {
75 case 2:
76 case 3: return *buf;
77 case 4: return LE_16(buf);
78 case 5: return BE_16(buf);
79 case 8: return LE_32(buf);
80 case 9: return BE_32(buf);
81 default: return 0;
82 }
83 }
84
85 static void load_cursor(VmncContext *c, uint8_t *src)
86 {
87 int i, j, p;
88 const int bpp = c->bpp2;
89 uint8_t *dst8 = c->curbits;
90 uint16_t *dst16 = (uint16_t*)c->curbits;
91 uint32_t *dst32 = (uint32_t*)c->curbits;
92
93 for(j = 0; j < c->cur_h; j++) {
94 for(i = 0; i < c->cur_w; i++) {
95 p = vmnc_get_pixel(src, bpp, c->bigendian);
96 src += bpp;
97 if(bpp == 1) *dst8++ = p;
98 if(bpp == 2) *dst16++ = p;
99 if(bpp == 4) *dst32++ = p;
100 }
101 }
102 dst8 = c->curmask;
103 dst16 = (uint16_t*)c->curmask;
104 dst32 = (uint32_t*)c->curmask;
105 for(j = 0; j < c->cur_h; j++) {
106 for(i = 0; i < c->cur_w; i++) {
107 p = vmnc_get_pixel(src, bpp, c->bigendian);
108 src += bpp;
109 if(bpp == 1) *dst8++ = p;
110 if(bpp == 2) *dst16++ = p;
111 if(bpp == 4) *dst32++ = p;
112 }
113 }
114 }
115
116 static void put_cursor(uint8_t *dst, int stride, VmncContext *c, int dx, int dy)
117 {
118 int i, j, t;
119 int w, h, x, y;
120 w = c->cur_w;
121 if(c->width < c->cur_x + c->cur_w) w = c->width - c->cur_x;
122 h = c->cur_h;
123 if(c->height < c->cur_y + c->cur_h) h = c->height - c->cur_y;
124 x = c->cur_x;
125 y = c->cur_y;
126 if(x < 0) {
127 w += x;
128 x = 0;
129 }
130 if(y < 0) {
131 h += y;
132 y = 0;
133 }
134
135 if((w < 1) || (h < 1)) return;
136 dst += x * c->bpp2 + y * stride;
137
138 if(c->bpp2 == 1) {
139 uint8_t* cd = c->curbits, *msk = c->curmask;
140 for(j = 0; j < h; j++) {
141 for(i = 0; i < w; i++)
142 dst[i] = (dst[i] & cd[i]) ^ msk[i];
143 msk += c->cur_w;
144 cd += c->cur_w;
145 dst += stride;
146 }
147 } else if(c->bpp2 == 2) {
148 uint16_t* cd = (uint16_t*)c->curbits, *msk = (uint16_t*)c->curmask;
149 uint16_t* dst2;
150 for(j = 0; j < h; j++) {
151 dst2 = (uint16_t*)dst;
152 for(i = 0; i < w; i++)
153 dst2[i] = (dst2[i] & cd[i]) ^ msk[i];
154 msk += c->cur_w;
155 cd += c->cur_w;
156 dst += stride;
157 }
158 } else if(c->bpp2 == 4) {
159 uint32_t* cd = (uint32_t*)c->curbits, *msk = (uint32_t*)c->curmask;
160 uint32_t* dst2;
161 for(j = 0; j < h; j++) {
162 dst2 = (uint32_t*)dst;
163 for(i = 0; i < w; i++)
164 dst2[i] = (dst2[i] & cd[i]) ^ msk[i];
165 msk += c->cur_w;
166 cd += c->cur_w;
167 dst += stride;
168 }
169 }
170 }
171
172 /* fill rectangle with given colour */
173 static always_inline void paint_rect(uint8_t *dst, int dx, int dy, int w, int h, int color, int bpp, int stride)
174 {
175 int i, j;
176 dst += dx * bpp + dy * stride;
177 if(bpp == 1){
178 for(j = 0; j < h; j++) {
179 memset(dst, color, w);
180 dst += stride;
181 }
182 }else if(bpp == 2){
183 uint16_t* dst2;
184 for(j = 0; j < h; j++) {
185 dst2 = (uint16_t*)dst;
186 for(i = 0; i < w; i++) {
187 *dst2++ = color;
188 }
189 dst += stride;
190 }
191 }else if(bpp == 4){
192 uint32_t* dst2;
193 for(j = 0; j < h; j++) {
194 dst2 = (uint32_t*)dst;
195 for(i = 0; i < w; i++) {
196 dst2[i] = color;
197 }
198 dst += stride;
199 }
200 }
201 }
202
203 static always_inline void paint_raw(uint8_t *dst, int w, int h, uint8_t* src, int bpp, int be, int stride)
204 {
205 int i, j, p;
206 for(j = 0; j < h; j++) {
207 for(i = 0; i < w; i++) {
208 p = vmnc_get_pixel(src, bpp, be);
209 src += bpp;
210 switch(bpp){
211 case 1:
212 dst[i] = p;
213 break;
214 case 2:
215 ((uint16_t*)dst)[i] = p;
216 break;
217 case 4:
218 ((uint32_t*)dst)[i] = p;
219 break;
220 }
221 }
222 dst += stride;
223 }
224 }
225
226 static int decode_hextile(VmncContext *c, uint8_t* dst, uint8_t* src, int ssize, int w, int h, int stride)
227 {
228 int i, j, k;
229 int bg = 0, fg = 0, rects, color, flags, xy, wh;
230 const int bpp = c->bpp2;
231 uint8_t *dst2;
232 int bw = 16, bh = 16;
233 uint8_t *ssrc=src;
234
235 for(j = 0; j < h; j += 16) {
236 dst2 = dst;
237 bw = 16;
238 if(j + 16 > h) bh = h - j;
239 for(i = 0; i < w; i += 16, dst2 += 16 * bpp) {
240 if(src - ssrc >= ssize) {
241 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
242 return -1;
243 }
244 if(i + 16 > w) bw = w - i;
245 flags = *src++;
246 if(flags & HT_RAW) {
247 if(src - ssrc > ssize - bw * bh * bpp) {
248 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
249 return -1;
250 }
251 paint_raw(dst2, bw, bh, src, bpp, c->bigendian, stride);
252 src += bw * bh * bpp;
253 } else {
254 if(flags & HT_BKG) {
255 bg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
256 }
257 if(flags & HT_FG) {
258 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
259 }
260 rects = 0;
261 if(flags & HT_SUB)
262 rects = *src++;
263 color = !!(flags & HT_CLR);
264
265 paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride);
266
267 if(src - ssrc > ssize - rects * (color * bpp + 2)) {
268 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
269 return -1;
270 }
271 for(k = 0; k < rects; k++) {
272 if(color) {
273 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
274 }
275 xy = *src++;
276 wh = *src++;
277 paint_rect(dst2, xy >> 4, xy & 0xF, (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride);
278 }
279 }
280 }
281 dst += stride * 16;
282 }
283 return src - ssrc;
284 }
285
286 static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8_t *buf, int buf_size)
287 {
288 VmncContext * const c = (VmncContext *)avctx->priv_data;
289 uint8_t *outptr;
290 uint8_t *src = buf;
291 int dx, dy, w, h, depth, enc, chunks, res, size_left;
292
293 c->pic.reference = 1;
294 c->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE;
295 if(avctx->reget_buffer(avctx, &c->pic) < 0){
296 av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
297 return -1;
298 }
299
300 c->pic.key_frame = 0;
301 c->pic.pict_type = FF_P_TYPE;
302
303 //restore screen after cursor
304 if(c->screendta) {
305 int i;
306 w = c->cur_w;
307 if(c->width < c->cur_x + w) w = c->width - c->cur_x;
308 h = c->cur_h;
309 if(c->height < c->cur_y + h) h = c->height - c->cur_y;
310 dx = c->cur_x;
311 if(dx < 0) {
312 w += dx;
313 dx = 0;
314 }
315 dy = c->cur_y;
316 if(dy < 0) {
317 h += dy;
318 dy = 0;
319 }
320 if((w > 0) && (h > 0)) {
321 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
322 for(i = 0; i < h; i++) {
323 memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2, w * c->bpp2);
324 outptr += c->pic.linesize[0];
325 }
326 }
327 }
328 src += 2;
329 chunks = BE_16(src); src += 2;
330 while(chunks--) {
331 dx = BE_16(src); src += 2;
332 dy = BE_16(src); src += 2;
333 w = BE_16(src); src += 2;
334 h = BE_16(src); src += 2;
335 enc = BE_32(src); src += 4;
336 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
337 size_left = buf_size - (src - buf);
338 switch(enc) {
339 case MAGIC_WMVd: // cursor
340 if(size_left < 2 + w * h * c->bpp2 * 2) {
341 av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", 2 + w * h * c->bpp2 * 2, size_left);
342 return -1;
343 }
344 src += 2;
345 c->cur_w = w;
346 c->cur_h = h;
347 c->cur_hx = dx;
348 c->cur_hy = dy;
349 if((c->cur_hx > c->cur_w) || (c->cur_hy > c->cur_h)) {
350 av_log(avctx, AV_LOG_ERROR, "Cursor hot spot is not in image: %ix%i of %ix%i cursor size\n", c->cur_hx, c->cur_hy, c->cur_w, c->cur_h);
351 c->cur_hx = c->cur_hy = 0;
352 }
353 c->curbits = av_realloc(c->curbits, c->cur_w * c->cur_h * c->bpp2);
354 c->curmask = av_realloc(c->curmask, c->cur_w * c->cur_h * c->bpp2);
355 c->screendta = av_realloc(c->screendta, c->cur_w * c->cur_h * c->bpp2);
356 load_cursor(c, src);
357 src += w * h * c->bpp2 * 2;
358 break;
359 case MAGIC_WMVe: // unknown
360 src += 2;
361 break;
362 case MAGIC_WMVf: // update cursor position
363 c->cur_x = dx - c->cur_hx;
364 c->cur_y = dy - c->cur_hy;
365 break;
366 case MAGIC_WMVg: // unknown
367 src += 10;
368 break;
369 case MAGIC_WMVh: // unknown
370 src += 4;
371 break;
372 case MAGIC_WMVi: // ServerInitialization struct
373 c->pic.key_frame = 1;
374 c->pic.pict_type = FF_I_TYPE;
375 depth = *src++;
376 if(depth != c->bpp) {
377 av_log(avctx, AV_LOG_INFO, "Depth mismatch. Container %i bpp, Frame data: %i bpp\n", c->bpp, depth);
378 }
379 src++;
380 c->bigendian = *src++;
381 if(c->bigendian & (~1)) {
382 av_log(avctx, AV_LOG_INFO, "Invalid header: bigendian flag = %i\n", c->bigendian);
383 return -1;
384 }
385 //skip the rest of pixel format data
386 src += 13;
387 break;
388 case MAGIC_WMVj: // unknown
389 src += 2;
390 break;
391 case 0x00000000: // raw rectangle data
392 if((dx + w > c->width) || (dy + h > c->height)) {
393 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height);
394 return -1;
395 }
396 if(size_left < w * h * c->bpp2) {
397 av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", w * h * c->bpp2, size_left);
398 return -1;
399 }
400 paint_raw(outptr, w, h, src, c->bpp2, c->bigendian, c->pic.linesize[0]);
401 src += w * h * c->bpp2;
402 break;
403 case 0x00000005: // HexTile encoded rectangle
404 if((dx + w > c->width) || (dy + h > c->height)) {
405 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height);
406 return -1;
407 }
408 res = decode_hextile(c, outptr, src, size_left, w, h, c->pic.linesize[0]);
409 if(res < 0)
410 return -1;
411 src += res;
412 break;
413 default:
414 av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc);
415 chunks = 0; // leave chunks decoding loop
416 }
417 }
418 if(c->screendta){
419 int i;
420 //save screen data before painting cursor
421 w = c->cur_w;
422 if(c->width < c->cur_x + w) w = c->width - c->cur_x;
423 h = c->cur_h;
424 if(c->height < c->cur_y + h) h = c->height - c->cur_y;
425 dx = c->cur_x;
426 if(dx < 0) {
427 w += dx;
428 dx = 0;
429 }
430 dy = c->cur_y;
431 if(dy < 0) {
432 h += dy;
433 dy = 0;
434 }
435 if((w > 0) && (h > 0)) {
436 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
437 for(i = 0; i < h; i++) {
438 memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr, w * c->bpp2);
439 outptr += c->pic.linesize[0];
440 }
441 outptr = c->pic.data[0];
442 put_cursor(outptr, c->pic.linesize[0], c, c->cur_x, c->cur_y);
443 }
444 }
445 *data_size = sizeof(AVFrame);
446 *(AVFrame*)data = c->pic;
447
448 /* always report that the buffer was completely consumed */
449 return buf_size;
450 }
451
452
453
454 /*
455 *
456 * Init VMnc decoder
457 *
458 */
459 static int decode_init(AVCodecContext *avctx)
460 {
461 VmncContext * const c = (VmncContext *)avctx->priv_data;
462
463 c->avctx = avctx;
464 avctx->has_b_frames = 0;
465
466 c->pic.data[0] = NULL;
467 c->width = avctx->width;
468 c->height = avctx->height;
469
470 if (avcodec_check_dimensions(avctx, avctx->width, avctx->height) < 0) {
471 return 1;
472 }
473 c->bpp = avctx->bits_per_sample;
474 c->bpp2 = c->bpp/8;
475
476 switch(c->bpp){
477 case 8:
478 avctx->pix_fmt = PIX_FMT_PAL8;
479 break;
480 case 16:
481 avctx->pix_fmt = PIX_FMT_RGB555;
482 break;
483 case 32:
484 avctx->pix_fmt = PIX_FMT_RGB32;
485 break;
486 default:
487 av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp);
488 }
489
490 return 0;
491 }
492
493
494
495 /*
496 *
497 * Uninit VMnc decoder
498 *
499 */
500 static int decode_end(AVCodecContext *avctx)
501 {
502 VmncContext * const c = (VmncContext *)avctx->priv_data;
503
504 if (c->pic.data[0])
505 avctx->release_buffer(avctx, &c->pic);
506
507 av_free(c->curbits);
508 av_free(c->curmask);
509 av_free(c->screendta);
510 return 0;
511 }
512
513 AVCodec vmnc_decoder = {
514 "VMware video",
515 CODEC_TYPE_VIDEO,
516 CODEC_ID_VMNC,
517 sizeof(VmncContext),
518 decode_init,
519 NULL,
520 decode_end,
521 decode_frame
522 };
523