jpeg2000: Proper cleanup on failure in decode_frame()
[libav.git] / libavcodec / jpeg2000dec.c
index 9befa8b..15dfc2b 100644 (file)
@@ -84,8 +84,7 @@ typedef struct Jpeg2000DecoderContext {
     Jpeg2000Tile    *tile;
 
     /*options parameters*/
-    int16_t         lowres;
-    int16_t         reduction_factor;
+    int             reduction_factor;
 } Jpeg2000DecoderContext;
 
 /* get_bits functions for JPEG2000 packet bitstream
@@ -236,37 +235,35 @@ static int get_siz(Jpeg2000DecoderContext *s)
     s->avctx->height = ff_jpeg2000_ceildivpow2(s->height - s->image_offset_y,
                                                s->reduction_factor);
 
-    switch (s->avctx->profile) {
-    case FF_PROFILE_JPEG2000_DCINEMA_2K:
-    case FF_PROFILE_JPEG2000_DCINEMA_4K:
-        /* XYZ color-space for digital cinema profiles */
-        s->avctx->pix_fmt = AV_PIX_FMT_XYZ12;
+    switch (s->ncomponents) {
+    case 1:
+        if (s->precision > 8)
+            s->avctx->pix_fmt = AV_PIX_FMT_GRAY16;
+        else
+            s->avctx->pix_fmt = AV_PIX_FMT_GRAY8;
         break;
-    default:
-        /* For other profiles selects color-space according number of
-         * components and bit depth precision. */
-        switch (s->ncomponents) {
-        case 1:
-            if (s->precision > 8)
-                s->avctx->pix_fmt = AV_PIX_FMT_GRAY16;
-            else
-                s->avctx->pix_fmt = AV_PIX_FMT_GRAY8;
+    case 3:
+        switch (s->avctx->profile) {
+        case FF_PROFILE_JPEG2000_DCINEMA_2K:
+        case FF_PROFILE_JPEG2000_DCINEMA_4K:
+            /* XYZ color-space for digital cinema profiles */
+            s->avctx->pix_fmt = AV_PIX_FMT_XYZ12;
             break;
-        case 3:
+        default:
             if (s->precision > 8)
                 s->avctx->pix_fmt = AV_PIX_FMT_RGB48;
             else
                 s->avctx->pix_fmt = AV_PIX_FMT_RGB24;
             break;
-        case 4:
-            s->avctx->pix_fmt = AV_PIX_FMT_BGRA;
-            break;
-        default:
-            /* pixel format can not be identified */
-            s->avctx->pix_fmt = AV_PIX_FMT_NONE;
-            break;
         }
         break;
+    case 4:
+        s->avctx->pix_fmt = AV_PIX_FMT_RGBA;
+        break;
+    default:
+        /* pixel format can not be identified */
+        s->avctx->pix_fmt = AV_PIX_FMT_NONE;
+        break;
     }
     return 0;
 }
@@ -576,8 +573,6 @@ static int init_tile(Jpeg2000DecoderContext *s, int tileno)
     int tilex = tileno % s->numXtiles;
     int tiley = tileno / s->numXtiles;
     Jpeg2000Tile *tile = s->tile + tileno;
-    Jpeg2000CodingStyle *codsty;
-    Jpeg2000QuantStyle  *qntsty;
 
     if (!tile->comp)
         return AVERROR(ENOMEM);
@@ -585,26 +580,24 @@ static int init_tile(Jpeg2000DecoderContext *s, int tileno)
     /* copy codsty, qnsty to tile. TODO: Is it the best way?
      * codsty, qnsty is an array of 4 structs Jpeg2000CodingStyle
      * and Jpeg2000QuantStyle */
-    memcpy(tile->codsty, s->codsty, s->ncomponents * sizeof(*codsty));
-    memcpy(tile->qntsty, s->qntsty, s->ncomponents * sizeof(*qntsty));
+    memcpy(tile->codsty, s->codsty, s->ncomponents * sizeof(*tile->codsty));
+    memcpy(tile->qntsty, s->qntsty, s->ncomponents * sizeof(*tile->qntsty));
 
     for (compno = 0; compno < s->ncomponents; compno++) {
         Jpeg2000Component *comp = tile->comp + compno;
+        Jpeg2000CodingStyle *codsty = tile->codsty + compno;
+        Jpeg2000QuantStyle  *qntsty = tile->qntsty + compno;
         int ret; // global bandno
-        codsty = tile->codsty + compno;
-        qntsty = tile->qntsty + compno;
 
         comp->coord_o[0][0] = FFMAX(tilex       * s->tile_width  + s->tile_offset_x, s->image_offset_x);
         comp->coord_o[0][1] = FFMIN((tilex + 1) * s->tile_width  + s->tile_offset_x, s->width);
         comp->coord_o[1][0] = FFMAX(tiley       * s->tile_height + s->tile_offset_y, s->image_offset_y);
         comp->coord_o[1][1] = FFMIN((tiley + 1) * s->tile_height + s->tile_offset_y, s->height);
 
-        // FIXME: add a dcinema profile check ?
-        // value is guaranteed by profile (orig=0, 1 tile)
-        comp->coord[0][0] = 0;
-        comp->coord[0][1] = s->avctx->width;
-        comp->coord[1][0] = 0;
-        comp->coord[1][1] = s->avctx->height;
+        comp->coord[0][0] = ff_jpeg2000_ceildivpow2(comp->coord_o[0][0], s->reduction_factor);
+        comp->coord[0][1] = ff_jpeg2000_ceildivpow2(comp->coord_o[0][1], s->reduction_factor);
+        comp->coord[1][0] = ff_jpeg2000_ceildivpow2(comp->coord_o[1][0], s->reduction_factor);
+        comp->coord[1][1] = ff_jpeg2000_ceildivpow2(comp->coord_o[1][1], s->reduction_factor);
 
         if (ret = ff_jpeg2000_init_component(comp, codsty, qntsty,
                                              s->cbps[compno], s->cdx[compno],
@@ -947,12 +940,13 @@ static int decode_cblk(Jpeg2000DecoderContext *s, Jpeg2000CodingStyle *codsty,
     int passno = cblk->npasses, pass_t = 2, bpno = cblk->nonzerobits - 1, y;
 
     for (y = 0; y < height; y++)
-        memset(t1->data[y], 0, width * sizeof(width));
+        memset(t1->data[y], 0, width * sizeof(**t1->data));
+
     /* If code-block contains no compressed data: nothing to do. */
     if (!cblk->length)
         return 0;
     for (y = 0; y < height + 2; y++)
-        memset(t1->flags[y], 0, (width + 2) * sizeof(width));
+        memset(t1->flags[y], 0, (width + 2) * sizeof(**t1->flags));
 
     ff_mqc_initdec(&t1->mqc, cblk->data);
     cblk->data[cblk->length]     = 0xff;
@@ -993,11 +987,11 @@ static void dequantization_float(int x, int y, Jpeg2000Cblk *cblk,
                                  Jpeg2000T1Context *t1, Jpeg2000Band *band)
 {
     int i, j, idx;
-    float *datap = &comp->data[(comp->coord[0][1] - comp->coord[0][0]) * y + x];
+    float *datap = &comp->f_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x];
     for (j = 0; j < (cblk->coord[1][1] - cblk->coord[1][0]); ++j)
         for (i = 0; i < (cblk->coord[0][1] - cblk->coord[0][0]); ++i) {
             idx        = (comp->coord[0][1] - comp->coord[0][0]) * j + i;
-            datap[idx] = (float)(t1->data[j][i]) * ((float)band->stepsize);
+            datap[idx] = (float)(t1->data[j][i]) * band->f_stepsize;
         }
 }
 
@@ -1007,13 +1001,12 @@ static void dequantization_int(int x, int y, Jpeg2000Cblk *cblk,
                                Jpeg2000T1Context *t1, Jpeg2000Band *band)
 {
     int i, j, idx;
-    int32_t *datap =
-        (int32_t *) &comp->data[(comp->coord[0][1] - comp->coord[0][0]) * y + x];
+    int32_t *datap = &comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x];
     for (j = 0; j < (cblk->coord[1][1] - cblk->coord[1][0]); ++j)
         for (i = 0; i < (cblk->coord[0][1] - cblk->coord[0][0]); ++i) {
             idx        = (comp->coord[0][1] - comp->coord[0][0]) * j + i;
             datap[idx] =
-                ((int32_t)(t1->data[j][i]) * ((int32_t)band->stepsize) + (1 << 15)) >> 16;
+                ((int32_t)(t1->data[j][i]) * band->i_stepsize + (1 << 15)) >> 16;
         }
 }
 
@@ -1040,9 +1033,9 @@ static void mct_decode(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)
 
     for (i = 0; i < 3; i++)
         if (tile->codsty[0].transform == FF_DWT97)
-            srcf[i] = tile->comp[i].data;
+            srcf[i] = tile->comp[i].f_data;
         else
-            src[i] = (int32_t *)tile->comp[i].data;
+            src [i] = tile->comp[i].i_data;
 
     for (i = 0; i < 2; i++)
         csize *= tile->comp[0].coord[i][1] - tile->comp[0].coord[i][0];
@@ -1122,30 +1115,28 @@ static int jpeg2000_decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile,
                         x = cblk->coord[0][0];
                         y = cblk->coord[1][0];
 
-                        if (s->avctx->flags & CODEC_FLAG_BITEXACT)
-                            dequantization_int(x, y, cblk, comp, &t1, band);
-                        else
+                        if (codsty->transform == FF_DWT97)
                             dequantization_float(x, y, cblk, comp, &t1, band);
+                        else
+                            dequantization_int(x, y, cblk, comp, &t1, band);
                    } /* end cblk */
                 } /*end prec */
             } /* end band */
         } /* end reslevel */
 
         /* inverse DWT */
-        ff_dwt_decode(&comp->dwt, comp->data);
+        ff_dwt_decode(&comp->dwt, codsty->transform == FF_DWT97 ? (void*)comp->f_data : (void*)comp->i_data);
     } /*end comp */
 
     /* inverse MCT transformation */
     if (tile->codsty[0].mct)
         mct_decode(s, tile);
 
-    if (s->avctx->pix_fmt == AV_PIX_FMT_BGRA) // RGBA -> BGRA
-        FFSWAP(float *, tile->comp[0].data, tile->comp[2].data);
-
     if (s->precision <= 8) {
         for (compno = 0; compno < s->ncomponents; compno++) {
             Jpeg2000Component *comp = tile->comp + compno;
-            int32_t *datap = (int32_t *)comp->data;
+            float *datap = comp->f_data;
+            int32_t *i_datap = comp->i_data;
             y    = tile->comp[compno].coord[1][0] - s->image_offset_y;
             line = picture->data[0] + y * picture->linesize[0];
             for (; y < tile->comp[compno].coord[1][1] - s->image_offset_y; y += s->cdy[compno]) {
@@ -1155,12 +1146,16 @@ static int jpeg2000_decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile,
                 dst = line + x * s->ncomponents + compno;
 
                 for (; x < tile->comp[compno].coord[0][1] - s->image_offset_x; x += s->cdx[compno]) {
-                    *datap += 1 << (s->cbps[compno] - 1);
-                    if (*datap < 0)
-                        *datap = 0;
-                    else if (*datap >= (1 << s->cbps[compno]))
-                        *datap = (1 << s->cbps[compno]) - 1;
-                    *dst = *datap++;
+                     int val;
+                    /* DC level shift and clip see ISO 15444-1:2002 G.1.2 */
+                    if (tile->codsty->transform == FF_DWT97)
+                        val = lrintf(*datap) + (1 << (s->cbps[compno] - 1));
+                    else
+                        val = *i_datap + (1 << (s->cbps[compno] - 1));
+                    val = av_clip(val, 0, (1 << s->cbps[compno]) - 1);
+                    *dst = val << (8 - s->cbps[compno]);
+                    datap++;
+                    i_datap++;
                     dst += s->ncomponents;
                 }
                 line += picture->linesize[0];
@@ -1169,8 +1164,8 @@ static int jpeg2000_decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile,
     } else {
         for (compno = 0; compno < s->ncomponents; compno++) {
             Jpeg2000Component *comp = tile->comp + compno;
-            float *datap = comp->data;
-            int32_t *i_datap = (int32_t *) comp->data;
+            float *datap = comp->f_data;
+            int32_t *i_datap = comp->i_data;
             uint16_t *linel;
 
             y     = tile->comp[compno].coord[1][0] - s->image_offset_y;
@@ -1180,15 +1175,15 @@ static int jpeg2000_decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile,
                 x   = tile->comp[compno].coord[0][0] - s->image_offset_x;
                 dst = linel + (x * s->ncomponents + compno);
                 for (; x < s->avctx->width; x += s->cdx[compno]) {
-                    int16_t val;
+                    int val;
                     /* DC level shift and clip see ISO 15444-1:2002 G.1.2 */
-                    if (s->avctx->flags & CODEC_FLAG_BITEXACT)
-                        val = *i_datap + (1 << (s->cbps[compno] - 1));
-                    else
+                    if (tile->codsty->transform == FF_DWT97)
                         val = lrintf(*datap) + (1 << (s->cbps[compno] - 1));
+                    else
+                        val = *i_datap + (1 << (s->cbps[compno] - 1));
                     val = av_clip(val, 0, (1 << s->cbps[compno]) - 1);
                     /* align 12 bit values in little-endian mode */
-                    *dst = val << 4;
+                    *dst = val << (16 - s->cbps[compno]);
                     datap++;
                     i_datap++;
                     dst += s->ncomponents;
@@ -1351,11 +1346,10 @@ static int jpeg2000_decode_frame(AVCodecContext *avctx, void *data,
     bytestream2_init(&s->g, avpkt->data, avpkt->size);
     s->curtileno = 0; // TODO: only one tile in DCI JP2K. to implement for more tiles
 
-    // reduction factor, i.e number of resolution levels to skip
-    s->reduction_factor = s->lowres;
-
-    if (bytestream2_get_bytes_left(&s->g) < 2)
-        return AVERROR_INVALIDDATA;
+    if (bytestream2_get_bytes_left(&s->g) < 2) {
+        ret = AVERROR_INVALIDDATA;
+        goto end;
+    }
 
     // check if the image is in jp2 format
     if (bytestream2_get_bytes_left(&s->g) >= 12 &&
@@ -1365,17 +1359,17 @@ static int jpeg2000_decode_frame(AVCodecContext *avctx, void *data,
         if (!jp2_find_codestream(s)) {
             av_log(avctx, AV_LOG_ERROR,
                    "Could not find Jpeg2000 codestream atom.\n");
-            return AVERROR_INVALIDDATA;
+            ret = AVERROR_INVALIDDATA;
+            goto end;
         }
     } else {
         bytestream2_seek(&s->g, 0, SEEK_SET);
-        if (bytestream2_peek_be16(&s->g) != JPEG2000_SOC)
-            bytestream2_skip(&s->g, 8);
     }
 
     if (bytestream2_get_be16u(&s->g) != JPEG2000_SOC) {
         av_log(avctx, AV_LOG_ERROR, "SOC marker not present\n");
-        return AVERROR_INVALIDDATA;
+        ret = AVERROR_INVALIDDATA;
+        goto end;
     }
     if (ret = jpeg2000_read_main_headers(s))
         goto end;
@@ -1394,6 +1388,8 @@ static int jpeg2000_decode_frame(AVCodecContext *avctx, void *data,
         if (ret = jpeg2000_decode_tile(s, s->tile + tileno, picture))
             goto end;
 
+    jpeg2000_dec_cleanup(s);
+
     *got_frame = 1;
 
     return bytestream2_tell(&s->g);
@@ -1413,7 +1409,7 @@ static void jpeg2000_init_static_data(AVCodec *codec)
 
 static const AVOption options[] = {
     { "lowres",  "Lower the decoding resolution by a power of two",
-        OFFSET(lowres), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, JPEG2000_MAX_RESLEVELS - 1, VD },
+        OFFSET(reduction_factor), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, JPEG2000_MAX_RESLEVELS - 1, VD },
     { NULL },
 };
 
@@ -1443,8 +1439,5 @@ AVCodec ff_jpeg2000_decoder = {
     .init_static_data = jpeg2000_init_static_data,
     .decode           = jpeg2000_decode_frame,
     .priv_class       = &class,
-    .pix_fmts         = (enum AVPixelFormat[]) { AV_PIX_FMT_XYZ12,
-                                                 AV_PIX_FMT_GRAY8,
-                                                 -1 },
     .profiles         = NULL_IF_CONFIG_SMALL(profiles)
 };