utvideodec: Fix decoding odd sizes with interlaced video with some formats
authorPaul B Mahol <onemda@gmail.com>
Fri, 21 Apr 2017 10:27:17 +0000 (12:27 +0200)
committerLuca Barbato <lu_zero@gentoo.org>
Mon, 24 Apr 2017 22:38:07 +0000 (00:38 +0200)
Signed-off-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: Luca Barbato <lu_zero@gentoo.org>
libavcodec/utvideodec.c

index 910d64b..0f58c83 100644 (file)
@@ -228,6 +228,16 @@ fail:
     return AVERROR_INVALIDDATA;
 }
 
+static int compute_cmask(int plane_no, int interlaced, int pix_fmt)
+{
+    const int is_luma = (pix_fmt == AV_PIX_FMT_YUV420P) && !plane_no;
+
+    if (interlaced)
+        return ~(1 + 2 * is_luma);
+
+    return ~is_luma;
+}
+
 static int decode_plane(UtvideoContext *c, int plane_no,
                         uint8_t *dst, int step, ptrdiff_t stride,
                         int width, int height,
@@ -238,7 +248,7 @@ static int decode_plane(UtvideoContext *c, int plane_no,
     VLC vlc;
     BitstreamContext bc;
     int prev, fsym;
-    const int cmask = ~(!plane_no && c->avctx->pix_fmt == AV_PIX_FMT_YUV420P);
+    const int cmask = compute_cmask(plane_no, c->interlaced, c->avctx->pix_fmt);
 
     if (build_huff(src, &vlc, &fsym)) {
         av_log(c->avctx, AV_LOG_ERROR, "Cannot build Huffman codes\n");