cook shouldn't try to enforce the bitstream reader type
[libav.git] / libavcodec / cook.c
index 516a138..88ee071 100644 (file)
@@ -15,7 +15,7 @@
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  *
  */
 
@@ -45,7 +45,6 @@
 #include <stddef.h>
 #include <stdio.h>
 
-#define ALT_BITSTREAM_READER
 #include "avcodec.h"
 #include "bitstream.h"
 #include "dsputil.h"
@@ -90,8 +89,8 @@ typedef struct {
     int                 samples_per_channel;
     int                 samples_per_frame;
     int                 subbands;
-    int                 numvector_bits;
-    int                 numvector_size;                //1 << numvector_bits;
+    int                 log2_numvector_size;
+    int                 numvector_size;                //1 << log2_numvector_size;
     int                 js_subband_start;
     int                 total_subbands;
     int                 num_vectors;
@@ -113,10 +112,11 @@ typedef struct {
     /* gain buffers */
     COOKgain*           gain_now_ptr;
     COOKgain*           gain_previous_ptr;
-    COOKgain            gain_copy;
     COOKgain            gain_current;
     COOKgain            gain_now;
     COOKgain            gain_previous;
+    COOKgain            gain_channel1[2];
+    COOKgain            gain_channel2[2];
 
     /* VLC data */
     int                 js_vlc_bits;
@@ -131,11 +131,6 @@ typedef struct {
     float               rootpow2tab[127];
 
     /* data buffers */
-    uint8_t*            frame_reorder_buffer;
-    int*                frame_reorder_index;
-    int                 frame_reorder_counter;
-    int                 frame_reorder_complete;
-    int                 frame_reorder_index_size;
 
     uint8_t*            decoded_bytes_buffer;
     float               mono_mdct_output[2048] __attribute__((aligned(16)));
@@ -143,6 +138,7 @@ typedef struct {
     float               mono_previous_buffer1[1024];
     float               mono_previous_buffer2[1024];
     float*              decode_buf_ptr[4];
+    float*              decode_buf_ptr2[2];
     float               decode_buffer_1[1024];
     float               decode_buffer_2[1024];
     float               decode_buffer_3[1024];
@@ -188,8 +184,8 @@ static void init_pow2table(COOKContext *q){
     int i;
     q->pow2tab[63] = 1.0;
     for (i=1 ; i<64 ; i++){
-        q->pow2tab[63+i]=(float)pow(2.0,(double)i);
-        q->pow2tab[63-i]=1.0/(float)pow(2.0,(double)i);
+        q->pow2tab[63+i]=(float)((uint64_t)1<<i);
+        q->pow2tab[63-i]=1.0/(float)((uint64_t)1<<i);
     }
 }
 
@@ -198,8 +194,8 @@ static void init_rootpow2table(COOKContext *q){
     int i;
     q->rootpow2tab[63] = 1.0;
     for (i=1 ; i<64 ; i++){
-        q->rootpow2tab[63+i]=sqrt((float)powf(2.0,(float)i));
-        q->rootpow2tab[63-i]=sqrt(1.0/(float)powf(2.0,(float)i));
+        q->rootpow2tab[63+i]=sqrt((float)((uint64_t)1<<i));
+        q->rootpow2tab[63-i]=sqrt(1.0/(float)((uint64_t)1<<i));
     }
 }
 
@@ -211,10 +207,6 @@ static void init_gain_table(COOKContext *q) {
         q->gain_table[i] = pow((double)q->pow2tab[i+52] ,
                                (1.0/(double)q->gain_size_factor));
     }
-    memset(&q->gain_copy, 0, sizeof(COOKgain));
-    memset(&q->gain_current, 0, sizeof(COOKgain));
-    memset(&q->gain_now, 0, sizeof(COOKgain));
-    memset(&q->gain_previous, 0, sizeof(COOKgain));
 }
 
 
@@ -325,8 +317,6 @@ static int cook_decode_close(AVCodecContext *avctx)
     av_free(q->mlt_precos);
     av_free(q->mlt_presin);
     av_free(q->mlt_postcos);
-    av_free(q->frame_reorder_index);
-    av_free(q->frame_reorder_buffer);
     av_free(q->decoded_bytes_buffer);
 
     /* Free the transform. */
@@ -707,7 +697,7 @@ static void mono_decode(COOKContext *q, float* mlt_buffer) {
     memset(&category_index, 0, 128*sizeof(int));
 
     decode_envelope(q, quant_index_table);
-    q->num_vectors = get_bits(&q->gb,q->numvector_bits);
+    q->num_vectors = get_bits(&q->gb,q->log2_numvector_size);
     dequant_envelope(q, quant_index_table, quant_value_table);
     categorize(q, quant_index_table, category, category_index);
     expand_category(q, category, category_index);
@@ -915,7 +905,7 @@ static void joint_decode(COOKContext *q, float* mlt_buffer1,
                          float* mlt_buffer2) {
     int i,j;
     int decouple_tab[SUBBAND_SIZE];
-    float decode_buffer[2048];  //Only 1060 might be needed.
+    float decode_buffer[1060];
     int idx, cpl_tmp,tmp_idx;
     float f1,f2;
     float* cplscale;
@@ -940,20 +930,18 @@ static void joint_decode(COOKContext *q, float* mlt_buffer1,
     /* When we reach js_subband_start (the higher frequencies)
        the coefficients are stored in a coupling scheme. */
     idx = (1 << q->js_vlc_bits) - 1;
-    if (q->js_subband_start < q->subbands) {
-        for (i=0 ; i<q->subbands ; i++) {
-            cpl_tmp = cplband[i + q->js_subband_start];
-            idx -=decouple_tab[cpl_tmp];
-            cplscale = (float*)cplscales[q->js_vlc_bits-2];  //choose decoupler table
-            f1 = cplscale[decouple_tab[cpl_tmp]];
-            f2 = cplscale[idx-1];
-            for (j=0 ; j<SUBBAND_SIZE ; j++) {
-                tmp_idx = ((2*q->js_subband_start + i)*20)+j;
-                mlt_buffer1[20*(i+q->js_subband_start) + j] = f1 * decode_buffer[tmp_idx];
-                mlt_buffer2[20*(i+q->js_subband_start) + j] = f2 * decode_buffer[tmp_idx];
-            }
-            idx = (1 << q->js_vlc_bits) - 1;
+    for (i=q->js_subband_start ; i<q->subbands ; i++) {
+        cpl_tmp = cplband[i];
+        idx -=decouple_tab[cpl_tmp];
+        cplscale = (float*)cplscales[q->js_vlc_bits-2];  //choose decoupler table
+        f1 = cplscale[decouple_tab[cpl_tmp]];
+        f2 = cplscale[idx-1];
+        for (j=0 ; j<SUBBAND_SIZE ; j++) {
+            tmp_idx = ((q->js_subband_start + i)*20)+j;
+            mlt_buffer1[20*i + j] = f1 * decode_buffer[tmp_idx];
+            mlt_buffer2[20*i + j] = f2 * decode_buffer[tmp_idx];
         }
+        idx = (1 << q->js_vlc_bits) - 1;
     }
 }
 
@@ -965,7 +953,6 @@ static void joint_decode(COOKContext *q, float* mlt_buffer1,
  * @param inbuffer          pointer to the inbuffer
  * @param sub_packet_size   subpacket size
  * @param outbuffer         pointer to the outbuffer
- * @param pos               the subpacket number in the frame
  */
 
 
@@ -984,7 +971,6 @@ static int decode_subpacket(COOKContext *q, uint8_t *inbuffer,
     decode_bytes(inbuffer, q->decoded_bytes_buffer, sub_packet_size);
     init_get_bits(&q->gb, q->decoded_bytes_buffer, sub_packet_size*8);
     decode_gain_info(&q->gb, &q->gain_current);
-    memcpy(&q->gain_copy, &q->gain_current ,sizeof(COOKgain));  //This copy does not seem to be used. FIXME
 
     if(q->nb_channels==2 && q->joint_stereo==1){
         joint_decode(q, q->decode_buf_ptr[0], q->decode_buf_ptr[2]);
@@ -1026,24 +1012,22 @@ static int decode_subpacket(COOKContext *q, uint8_t *inbuffer,
 
     } else if (q->nb_channels==2 && q->joint_stereo==0) {
             /* channel 0 */
-            mono_decode(q, q->decode_buf_ptr[0]);
+            mono_decode(q, q->decode_buf_ptr2[0]);
 
-            tmp_ptr = q->decode_buf_ptr[0];
-            q->decode_buf_ptr[0] = q->decode_buf_ptr[1];
-            q->decode_buf_ptr[1] = q->decode_buf_ptr[2];
-            q->decode_buf_ptr[2] = q->decode_buf_ptr[3];
-            q->decode_buf_ptr[3] = tmp_ptr;
+            tmp_ptr = q->decode_buf_ptr2[0];
+            q->decode_buf_ptr2[0] = q->decode_buf_ptr2[1];
+            q->decode_buf_ptr2[1] = tmp_ptr;
 
-            q->gain_now_ptr = &q->gain_now;
-            q->gain_previous_ptr = &q->gain_previous;
+            memcpy(&q->gain_channel1[0], &q->gain_current ,sizeof(COOKgain));
+            q->gain_now_ptr = &q->gain_channel1[0];
+            q->gain_previous_ptr = &q->gain_channel1[1];
 
-            cook_imlt(q, q->decode_buf_ptr[0], q->mono_mdct_output,q->mlt_tmp);
+            cook_imlt(q, q->decode_buf_ptr2[0], q->mono_mdct_output,q->mlt_tmp);
             gain_compensate(q, q->mono_mdct_output, q->gain_now_ptr,
-                            q->gain_previous_ptr, q->previous_buffer_ptr[0]);
-            /* Swap out the previous buffer. */
-            tmp_ptr = q->previous_buffer_ptr[0];
-            q->previous_buffer_ptr[0] = q->previous_buffer_ptr[1];
-            q->previous_buffer_ptr[1] = tmp_ptr;
+                            q->gain_previous_ptr, q->mono_previous_buffer1);
+
+            memcpy(&q->gain_channel1[1], &q->gain_channel1[0],sizeof(COOKgain));
+
 
             for (j=0 ; j<q->samples_per_frame ; j++){
                 value = lrintf(q->mono_mdct_output[j]);
@@ -1055,25 +1039,28 @@ static int decode_subpacket(COOKContext *q, uint8_t *inbuffer,
             /* channel 1 */
             //av_log(NULL,AV_LOG_ERROR,"bits = %d\n",get_bits_count(&q->gb));
             init_get_bits(&q->gb, q->decoded_bytes_buffer, sub_packet_size*8+q->bits_per_subpacket);
-            decode_gain_info(&q->gb, &q->gain_current);
-            //memcpy(&q->gain_copy, &q->gain_current ,sizeof(COOKgain));
+
+            q->gain_now_ptr = &q->gain_channel2[0];
+            q->gain_previous_ptr = &q->gain_channel2[1];
+
+            decode_gain_info(&q->gb, &q->gain_channel2[0]);
             mono_decode(q, q->decode_buf_ptr[0]);
-            tmp_ptr = q->decode_buf_ptr[0];
-            q->decode_buf_ptr[1] = q->decode_buf_ptr[2];
-            q->decode_buf_ptr[2] = q->decode_buf_ptr[3];
-            q->decode_buf_ptr[3] = tmp_ptr;
 
-            q->gain_now_ptr = &q->gain_now;
-            q->gain_previous_ptr = &q->gain_previous;
+            tmp_ptr = q->decode_buf_ptr[0];
+            q->decode_buf_ptr[0] = q->decode_buf_ptr[1];
+            q->decode_buf_ptr[1] = tmp_ptr;
 
             cook_imlt(q, q->decode_buf_ptr[0], q->mono_mdct_output,q->mlt_tmp);
-            gain_compensate(q, q->mono_mdct_output, q->gain_now_ptr, q->gain_previous_ptr, q->previous_buffer_ptr[0]);
+            gain_compensate(q, q->mono_mdct_output, q->gain_now_ptr,
+                            q->gain_previous_ptr, q->mono_previous_buffer2);
 
             /* Swap out the previous buffer. */
             tmp_ptr = q->previous_buffer_ptr[0];
             q->previous_buffer_ptr[0] = q->previous_buffer_ptr[1];
             q->previous_buffer_ptr[1] = tmp_ptr;
 
+            memcpy(&q->gain_channel2[1], &q->gain_channel2[0] ,sizeof(COOKgain));
+
             for (j=0 ; j<q->samples_per_frame ; j++){
                 value = lrintf(q->mono_mdct_output[j]);
                 if(value < -32768) value = -32768;
@@ -1081,11 +1068,6 @@ static int decode_subpacket(COOKContext *q, uint8_t *inbuffer,
                 outbuffer[2*j] = value;
             }
 
-
-            /* Swap out the previous buffer. */
-            memcpy(&q->gain_now, &q->gain_previous, sizeof(COOKgain));
-            memcpy(&q->gain_previous, &q->gain_current, sizeof(COOKgain));
-
     } else {
         mono_decode(q, q->decode_buf_ptr[0]);
 
@@ -1135,6 +1117,7 @@ static int cook_decode_frame(AVCodecContext *avctx,
 
     return avctx->block_align;
 }
+
 #ifdef COOKDEBUG
 static void dump_cook_context(COOKContext *q, COOKextradata *e)
 {
@@ -1156,13 +1139,12 @@ static void dump_cook_context(COOKContext *q, COOKextradata *e)
     PRINT("random_state",q->random_state);
     PRINT("mlt_size",q->mlt_size);
     PRINT("js_subband_start",q->js_subband_start);
-    PRINT("numvector_bits",q->numvector_bits);
+    PRINT("log2_numvector_size",q->log2_numvector_size);
     PRINT("numvector_size",q->numvector_size);
     PRINT("total_subbands",q->total_subbands);
-    PRINT("frame_reorder_counter",q->frame_reorder_counter);
-    PRINT("frame_reorder_index_size",q->frame_reorder_index_size);
 }
 #endif
+
 /**
  * Cook initialization
  *
@@ -1209,7 +1191,7 @@ static int cook_decode_init(AVCodecContext *avctx)
 
     /* Initialize default data states. */
     q->js_subband_start = 0;
-    q->numvector_bits = 5;
+    q->log2_numvector_size = 5;
     q->total_subbands = q->subbands;
 
     /* Initialize version-dependent variables */
@@ -1225,9 +1207,7 @@ static int cook_decode_init(AVCodecContext *avctx)
         case MONO_COOK2:
             if (q->nb_channels != 1) {
                 q->joint_stereo = 0;
-                av_log(NULL,AV_LOG_ERROR,"Non-joint-stereo files are decoded with wrong gain at the moment!\n");
                 q->bits_per_subpacket = q->bits_per_subpacket/2;
-
             }
             av_log(NULL,AV_LOG_DEBUG,"MONO_COOK2\n");
             break;
@@ -1244,10 +1224,10 @@ static int cook_decode_init(AVCodecContext *avctx)
                 q->js_vlc_bits = e->js_vlc_bits;
             }
             if (q->samples_per_channel > 256) {
-                q->numvector_bits++;   // q->numvector_bits  = 6
+                q->log2_numvector_size  = 6;
             }
             if (q->samples_per_channel > 512) {
-                q->numvector_bits++;   // q->numvector_bits  = 7
+                q->log2_numvector_size  = 7;
             }
             break;
         case MC_COOK:
@@ -1262,7 +1242,7 @@ static int cook_decode_init(AVCodecContext *avctx)
 
     /* Initialize variable relations */
     q->mlt_size = q->samples_per_channel;
-    q->numvector_size = (1 << q->numvector_bits);
+    q->numvector_size = (1 << q->log2_numvector_size);
 
     /* Generate tables */
     init_rootpow2table(q);
@@ -1272,6 +1252,10 @@ static int cook_decode_init(AVCodecContext *avctx)
     if (init_cook_vlc_tables(q) != 0)
         return -1;
 
+
+    if(avctx->block_align >= UINT_MAX/2)
+        return -1;
+
     /* Pad the databuffer with FF_INPUT_BUFFER_PADDING_SIZE,
        this is for the bitstreamreader. */
     if ((q->decoded_bytes_buffer = av_mallocz((avctx->block_align+(4-avctx->block_align%4) + FF_INPUT_BUFFER_PADDING_SIZE)*sizeof(uint8_t)))  == NULL)
@@ -1282,19 +1266,34 @@ static int cook_decode_init(AVCodecContext *avctx)
     q->decode_buf_ptr[2] = q->decode_buffer_3;
     q->decode_buf_ptr[3] = q->decode_buffer_4;
 
+    q->decode_buf_ptr2[0] = q->decode_buffer_3;
+    q->decode_buf_ptr2[1] = q->decode_buffer_4;
+
     q->previous_buffer_ptr[0] = q->mono_previous_buffer1;
     q->previous_buffer_ptr[1] = q->mono_previous_buffer2;
 
-    memset(q->decode_buffer_1,0,1024*sizeof(float));
-    memset(q->decode_buffer_2,0,1024*sizeof(float));
-    memset(q->decode_buffer_3,0,1024*sizeof(float));
-    memset(q->decode_buffer_4,0,1024*sizeof(float));
-
     /* Initialize transform. */
     if ( init_cook_mlt(q) == 0 )
         return -1;
 
-    //dump_cook_context(q,e);
+    /* Try to catch some obviously faulty streams, othervise it might be exploitable */
+    if (q->total_subbands > 53) {
+        av_log(NULL,AV_LOG_ERROR,"total_subbands > 53, report sample!\n");
+        return -1;
+    }
+    if (q->subbands > 50) {
+        av_log(NULL,AV_LOG_ERROR,"subbands > 50, report sample!\n");
+        return -1;
+    }
+    if ((q->samples_per_channel == 256) || (q->samples_per_channel == 512) || (q->samples_per_channel == 1024)) {
+    } else {
+        av_log(NULL,AV_LOG_ERROR,"unknown amount of samples_per_channel = %d, report sample!\n",q->samples_per_channel);
+        return -1;
+    }
+
+#ifdef COOKDEBUG
+    dump_cook_context(q,e);
+#endif
     return 0;
 }