cook shouldn't try to enforce the bitstream reader type
[libav.git] / libavcodec / cook.c
index e1e3356..88ee071 100644 (file)
@@ -15,7 +15,7 @@
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  *
  */
 
@@ -45,7 +45,6 @@
 #include <stddef.h>
 #include <stdio.h>
 
-#define ALT_BITSTREAM_READER
 #include "avcodec.h"
 #include "bitstream.h"
 #include "dsputil.h"
@@ -90,8 +89,8 @@ typedef struct {
     int                 samples_per_channel;
     int                 samples_per_frame;
     int                 subbands;
-    int                 numvector_bits;
-    int                 numvector_size;                //1 << numvector_bits;
+    int                 log2_numvector_size;
+    int                 numvector_size;                //1 << log2_numvector_size;
     int                 js_subband_start;
     int                 total_subbands;
     int                 num_vectors;
@@ -113,10 +112,11 @@ typedef struct {
     /* gain buffers */
     COOKgain*           gain_now_ptr;
     COOKgain*           gain_previous_ptr;
-    COOKgain            gain_copy;
     COOKgain            gain_current;
     COOKgain            gain_now;
     COOKgain            gain_previous;
+    COOKgain            gain_channel1[2];
+    COOKgain            gain_channel2[2];
 
     /* VLC data */
     int                 js_vlc_bits;
@@ -138,6 +138,7 @@ typedef struct {
     float               mono_previous_buffer1[1024];
     float               mono_previous_buffer2[1024];
     float*              decode_buf_ptr[4];
+    float*              decode_buf_ptr2[2];
     float               decode_buffer_1[1024];
     float               decode_buffer_2[1024];
     float               decode_buffer_3[1024];
@@ -183,8 +184,8 @@ static void init_pow2table(COOKContext *q){
     int i;
     q->pow2tab[63] = 1.0;
     for (i=1 ; i<64 ; i++){
-        q->pow2tab[63+i]=(float)pow(2.0,(double)i);
-        q->pow2tab[63-i]=1.0/(float)pow(2.0,(double)i);
+        q->pow2tab[63+i]=(float)((uint64_t)1<<i);
+        q->pow2tab[63-i]=1.0/(float)((uint64_t)1<<i);
     }
 }
 
@@ -193,8 +194,8 @@ static void init_rootpow2table(COOKContext *q){
     int i;
     q->rootpow2tab[63] = 1.0;
     for (i=1 ; i<64 ; i++){
-        q->rootpow2tab[63+i]=sqrt((float)powf(2.0,(float)i));
-        q->rootpow2tab[63-i]=sqrt(1.0/(float)powf(2.0,(float)i));
+        q->rootpow2tab[63+i]=sqrt((float)((uint64_t)1<<i));
+        q->rootpow2tab[63-i]=sqrt(1.0/(float)((uint64_t)1<<i));
     }
 }
 
@@ -206,10 +207,6 @@ static void init_gain_table(COOKContext *q) {
         q->gain_table[i] = pow((double)q->pow2tab[i+52] ,
                                (1.0/(double)q->gain_size_factor));
     }
-    memset(&q->gain_copy, 0, sizeof(COOKgain));
-    memset(&q->gain_current, 0, sizeof(COOKgain));
-    memset(&q->gain_now, 0, sizeof(COOKgain));
-    memset(&q->gain_previous, 0, sizeof(COOKgain));
 }
 
 
@@ -700,7 +697,7 @@ static void mono_decode(COOKContext *q, float* mlt_buffer) {
     memset(&category_index, 0, 128*sizeof(int));
 
     decode_envelope(q, quant_index_table);
-    q->num_vectors = get_bits(&q->gb,q->numvector_bits);
+    q->num_vectors = get_bits(&q->gb,q->log2_numvector_size);
     dequant_envelope(q, quant_index_table, quant_value_table);
     categorize(q, quant_index_table, category, category_index);
     expand_category(q, category, category_index);
@@ -956,7 +953,6 @@ static void joint_decode(COOKContext *q, float* mlt_buffer1,
  * @param inbuffer          pointer to the inbuffer
  * @param sub_packet_size   subpacket size
  * @param outbuffer         pointer to the outbuffer
- * @param pos               the subpacket number in the frame
  */
 
 
@@ -975,7 +971,6 @@ static int decode_subpacket(COOKContext *q, uint8_t *inbuffer,
     decode_bytes(inbuffer, q->decoded_bytes_buffer, sub_packet_size);
     init_get_bits(&q->gb, q->decoded_bytes_buffer, sub_packet_size*8);
     decode_gain_info(&q->gb, &q->gain_current);
-    memcpy(&q->gain_copy, &q->gain_current ,sizeof(COOKgain));  //This copy does not seem to be used. FIXME
 
     if(q->nb_channels==2 && q->joint_stereo==1){
         joint_decode(q, q->decode_buf_ptr[0], q->decode_buf_ptr[2]);
@@ -1017,24 +1012,22 @@ static int decode_subpacket(COOKContext *q, uint8_t *inbuffer,
 
     } else if (q->nb_channels==2 && q->joint_stereo==0) {
             /* channel 0 */
-            mono_decode(q, q->decode_buf_ptr[0]);
+            mono_decode(q, q->decode_buf_ptr2[0]);
 
-            tmp_ptr = q->decode_buf_ptr[0];
-            q->decode_buf_ptr[0] = q->decode_buf_ptr[1];
-            q->decode_buf_ptr[1] = q->decode_buf_ptr[2];
-            q->decode_buf_ptr[2] = q->decode_buf_ptr[3];
-            q->decode_buf_ptr[3] = tmp_ptr;
+            tmp_ptr = q->decode_buf_ptr2[0];
+            q->decode_buf_ptr2[0] = q->decode_buf_ptr2[1];
+            q->decode_buf_ptr2[1] = tmp_ptr;
 
-            q->gain_now_ptr = &q->gain_now;
-            q->gain_previous_ptr = &q->gain_previous;
+            memcpy(&q->gain_channel1[0], &q->gain_current ,sizeof(COOKgain));
+            q->gain_now_ptr = &q->gain_channel1[0];
+            q->gain_previous_ptr = &q->gain_channel1[1];
 
-            cook_imlt(q, q->decode_buf_ptr[0], q->mono_mdct_output,q->mlt_tmp);
+            cook_imlt(q, q->decode_buf_ptr2[0], q->mono_mdct_output,q->mlt_tmp);
             gain_compensate(q, q->mono_mdct_output, q->gain_now_ptr,
-                            q->gain_previous_ptr, q->previous_buffer_ptr[0]);
-            /* Swap out the previous buffer. */
-            tmp_ptr = q->previous_buffer_ptr[0];
-            q->previous_buffer_ptr[0] = q->previous_buffer_ptr[1];
-            q->previous_buffer_ptr[1] = tmp_ptr;
+                            q->gain_previous_ptr, q->mono_previous_buffer1);
+
+            memcpy(&q->gain_channel1[1], &q->gain_channel1[0],sizeof(COOKgain));
+
 
             for (j=0 ; j<q->samples_per_frame ; j++){
                 value = lrintf(q->mono_mdct_output[j]);
@@ -1046,25 +1039,28 @@ static int decode_subpacket(COOKContext *q, uint8_t *inbuffer,
             /* channel 1 */
             //av_log(NULL,AV_LOG_ERROR,"bits = %d\n",get_bits_count(&q->gb));
             init_get_bits(&q->gb, q->decoded_bytes_buffer, sub_packet_size*8+q->bits_per_subpacket);
-            decode_gain_info(&q->gb, &q->gain_current);
-            //memcpy(&q->gain_copy, &q->gain_current ,sizeof(COOKgain));
+
+            q->gain_now_ptr = &q->gain_channel2[0];
+            q->gain_previous_ptr = &q->gain_channel2[1];
+
+            decode_gain_info(&q->gb, &q->gain_channel2[0]);
             mono_decode(q, q->decode_buf_ptr[0]);
-            tmp_ptr = q->decode_buf_ptr[0];
-            q->decode_buf_ptr[1] = q->decode_buf_ptr[2];
-            q->decode_buf_ptr[2] = q->decode_buf_ptr[3];
-            q->decode_buf_ptr[3] = tmp_ptr;
 
-            q->gain_now_ptr = &q->gain_now;
-            q->gain_previous_ptr = &q->gain_previous;
+            tmp_ptr = q->decode_buf_ptr[0];
+            q->decode_buf_ptr[0] = q->decode_buf_ptr[1];
+            q->decode_buf_ptr[1] = tmp_ptr;
 
             cook_imlt(q, q->decode_buf_ptr[0], q->mono_mdct_output,q->mlt_tmp);
-            gain_compensate(q, q->mono_mdct_output, q->gain_now_ptr, q->gain_previous_ptr, q->previous_buffer_ptr[0]);
+            gain_compensate(q, q->mono_mdct_output, q->gain_now_ptr,
+                            q->gain_previous_ptr, q->mono_previous_buffer2);
 
             /* Swap out the previous buffer. */
             tmp_ptr = q->previous_buffer_ptr[0];
             q->previous_buffer_ptr[0] = q->previous_buffer_ptr[1];
             q->previous_buffer_ptr[1] = tmp_ptr;
 
+            memcpy(&q->gain_channel2[1], &q->gain_channel2[0] ,sizeof(COOKgain));
+
             for (j=0 ; j<q->samples_per_frame ; j++){
                 value = lrintf(q->mono_mdct_output[j]);
                 if(value < -32768) value = -32768;
@@ -1072,11 +1068,6 @@ static int decode_subpacket(COOKContext *q, uint8_t *inbuffer,
                 outbuffer[2*j] = value;
             }
 
-
-            /* Swap out the previous buffer. */
-            memcpy(&q->gain_now, &q->gain_previous, sizeof(COOKgain));
-            memcpy(&q->gain_previous, &q->gain_current, sizeof(COOKgain));
-
     } else {
         mono_decode(q, q->decode_buf_ptr[0]);
 
@@ -1126,6 +1117,7 @@ static int cook_decode_frame(AVCodecContext *avctx,
 
     return avctx->block_align;
 }
+
 #ifdef COOKDEBUG
 static void dump_cook_context(COOKContext *q, COOKextradata *e)
 {
@@ -1147,11 +1139,12 @@ static void dump_cook_context(COOKContext *q, COOKextradata *e)
     PRINT("random_state",q->random_state);
     PRINT("mlt_size",q->mlt_size);
     PRINT("js_subband_start",q->js_subband_start);
-    PRINT("numvector_bits",q->numvector_bits);
+    PRINT("log2_numvector_size",q->log2_numvector_size);
     PRINT("numvector_size",q->numvector_size);
     PRINT("total_subbands",q->total_subbands);
 }
 #endif
+
 /**
  * Cook initialization
  *
@@ -1198,7 +1191,7 @@ static int cook_decode_init(AVCodecContext *avctx)
 
     /* Initialize default data states. */
     q->js_subband_start = 0;
-    q->numvector_bits = 5;
+    q->log2_numvector_size = 5;
     q->total_subbands = q->subbands;
 
     /* Initialize version-dependent variables */
@@ -1214,9 +1207,7 @@ static int cook_decode_init(AVCodecContext *avctx)
         case MONO_COOK2:
             if (q->nb_channels != 1) {
                 q->joint_stereo = 0;
-                av_log(NULL,AV_LOG_ERROR,"Non-joint-stereo files are decoded with wrong gain at the moment!\n");
                 q->bits_per_subpacket = q->bits_per_subpacket/2;
-
             }
             av_log(NULL,AV_LOG_DEBUG,"MONO_COOK2\n");
             break;
@@ -1233,10 +1224,10 @@ static int cook_decode_init(AVCodecContext *avctx)
                 q->js_vlc_bits = e->js_vlc_bits;
             }
             if (q->samples_per_channel > 256) {
-                q->numvector_bits++;   // q->numvector_bits  = 6
+                q->log2_numvector_size  = 6;
             }
             if (q->samples_per_channel > 512) {
-                q->numvector_bits++;   // q->numvector_bits  = 7
+                q->log2_numvector_size  = 7;
             }
             break;
         case MC_COOK:
@@ -1251,7 +1242,7 @@ static int cook_decode_init(AVCodecContext *avctx)
 
     /* Initialize variable relations */
     q->mlt_size = q->samples_per_channel;
-    q->numvector_size = (1 << q->numvector_bits);
+    q->numvector_size = (1 << q->log2_numvector_size);
 
     /* Generate tables */
     init_rootpow2table(q);
@@ -1261,6 +1252,10 @@ static int cook_decode_init(AVCodecContext *avctx)
     if (init_cook_vlc_tables(q) != 0)
         return -1;
 
+
+    if(avctx->block_align >= UINT_MAX/2)
+        return -1;
+
     /* Pad the databuffer with FF_INPUT_BUFFER_PADDING_SIZE,
        this is for the bitstreamreader. */
     if ((q->decoded_bytes_buffer = av_mallocz((avctx->block_align+(4-avctx->block_align%4) + FF_INPUT_BUFFER_PADDING_SIZE)*sizeof(uint8_t)))  == NULL)
@@ -1271,17 +1266,31 @@ static int cook_decode_init(AVCodecContext *avctx)
     q->decode_buf_ptr[2] = q->decode_buffer_3;
     q->decode_buf_ptr[3] = q->decode_buffer_4;
 
+    q->decode_buf_ptr2[0] = q->decode_buffer_3;
+    q->decode_buf_ptr2[1] = q->decode_buffer_4;
+
     q->previous_buffer_ptr[0] = q->mono_previous_buffer1;
     q->previous_buffer_ptr[1] = q->mono_previous_buffer2;
 
-    memset(q->decode_buffer_1,0,1024*sizeof(float));
-    memset(q->decode_buffer_2,0,1024*sizeof(float));
-    memset(q->decode_buffer_3,0,1024*sizeof(float));
-    memset(q->decode_buffer_4,0,1024*sizeof(float));
-
     /* Initialize transform. */
     if ( init_cook_mlt(q) == 0 )
         return -1;
+
+    /* Try to catch some obviously faulty streams, othervise it might be exploitable */
+    if (q->total_subbands > 53) {
+        av_log(NULL,AV_LOG_ERROR,"total_subbands > 53, report sample!\n");
+        return -1;
+    }
+    if (q->subbands > 50) {
+        av_log(NULL,AV_LOG_ERROR,"subbands > 50, report sample!\n");
+        return -1;
+    }
+    if ((q->samples_per_channel == 256) || (q->samples_per_channel == 512) || (q->samples_per_channel == 1024)) {
+    } else {
+        av_log(NULL,AV_LOG_ERROR,"unknown amount of samples_per_channel = %d, report sample!\n",q->samples_per_channel);
+        return -1;
+    }
+
 #ifdef COOKDEBUG
     dump_cook_context(q,e);
 #endif