libtremor: merge upstream revision 17525 'Commit additional hardening to setup packet decode.'

git-svn-id: svn://svn.rockbox.org/rockbox/trunk@28762 a1c6a512-1295-4272-9138-f99709370657

apps/codecs/libtremor/codebook.c

@@ -101,6 +101,7 @@ int vorbis_staticbook_unpack(oggpack_buffer *opb,static_codebook *s){
     s->q_delta=oggpack_read(opb,32);
     s->q_quant=oggpack_read(opb,4)+1;
     s->q_sequencep=oggpack_read(opb,1);
+    if(s->q_sequencep==-1)goto _eofout;
 
     {
       int quantvals=0;

apps/codecs/libtremor/floor1.c

@@ -80,6 +80,7 @@ static vorbis_info_floor *floor1_unpack (vorbis_info *vi,oggpack_buffer *opb){
   info->partitions=oggpack_read(opb,5); /* only 0 to 31 legal */
   for(j=0;j<info->partitions;j++){
     info->partitionclass[j]=oggpack_read(opb,4); /* only 0 to 15 legal */
+    if(info->partitionclass[j]<0)goto err_out;
     if(maxclass<info->partitionclass[j])maxclass=info->partitionclass[j];
   }
 
@@ -102,6 +103,7 @@ static vorbis_info_floor *floor1_unpack (vorbis_info *vi,oggpack_buffer *opb){
   /* read the post list */
   info->mult=oggpack_read(opb,2)+1;     /* only 1,2,3,4 legal now */ 
   rangebits=oggpack_read(opb,4);
+  if(rangebits<0)goto err_out;
 
   for(j=0,k=0;j<info->partitions;j++){
     count+=info->class_dim[info->partitionclass[j]]; 

apps/codecs/libtremor/info.c

@@ -166,7 +166,7 @@ static int _vorbis_unpack_books(vorbis_info *vi,oggpack_buffer *opb){
 
   /* codebooks */
   ci->books=oggpack_read(opb,8)+1;
-  /*ci->book_param=_ogg_calloc(ci->books,sizeof(*ci->book_param));*/
+  if(ci->books<=0)goto err_out;
   for(i=0;i<ci->books;i++){
     ci->book_param[i]=(static_codebook *)_ogg_calloc(1,sizeof(*ci->book_param[i]));
     if(vorbis_staticbook_unpack(opb,ci->book_param[i]))goto err_out;
@@ -174,8 +174,7 @@ static int _vorbis_unpack_books(vorbis_info *vi,oggpack_buffer *opb){
 
   /* time backend settings */
   ci->times=oggpack_read(opb,6)+1;
-  /*ci->time_type=_ogg_malloc(ci->times*sizeof(*ci->time_type));*/
+  if(ci->times<=0)goto err_out;
-  /*ci->time_param=_ogg_calloc(ci->times,sizeof(void *));*/
   for(i=0;i<ci->times;i++){
     ci->time_type[i]=oggpack_read(opb,16);
     if(ci->time_type[i]<0 || ci->time_type[i]>=VI_TIMEB)goto err_out;
@@ -186,8 +185,7 @@ static int _vorbis_unpack_books(vorbis_info *vi,oggpack_buffer *opb){
 
   /* floor backend settings */
   ci->floors=oggpack_read(opb,6)+1;
-  /*ci->floor_type=_ogg_malloc(ci->floors*sizeof(*ci->floor_type));*/
+  if(ci->floors<=0)goto err_out;
-  /*ci->floor_param=_ogg_calloc(ci->floors,sizeof(void *));*/
   for(i=0;i<ci->floors;i++){
     ci->floor_type[i]=oggpack_read(opb,16);
     if(ci->floor_type[i]<0 || ci->floor_type[i]>=VI_FLOORB)goto err_out;
@@ -197,8 +195,7 @@ static int _vorbis_unpack_books(vorbis_info *vi,oggpack_buffer *opb){
 
   /* residue backend settings */
   ci->residues=oggpack_read(opb,6)+1;
-  /*ci->residue_type=_ogg_malloc(ci->residues*sizeof(*ci->residue_type));*/
+  if(ci->residues<=0)goto err_out;
-  /*ci->residue_param=_ogg_calloc(ci->residues,sizeof(void *));*/
   for(i=0;i<ci->residues;i++){
     ci->residue_type[i]=oggpack_read(opb,16);
     if(ci->residue_type[i]<0 || ci->residue_type[i]>=VI_RESB)goto err_out;
@@ -208,8 +205,7 @@ static int _vorbis_unpack_books(vorbis_info *vi,oggpack_buffer *opb){
 
   /* map backend settings */
   ci->maps=oggpack_read(opb,6)+1;
-  /*ci->map_type=_ogg_malloc(ci->maps*sizeof(*ci->map_type));*/
+  if(ci->maps<=0)goto err_out;
-  /*ci->map_param=_ogg_calloc(ci->maps,sizeof(void *));*/
   for(i=0;i<ci->maps;i++){
     ci->map_type[i]=oggpack_read(opb,16);
     if(ci->map_type[i]<0 || ci->map_type[i]>=VI_MAPB)goto err_out;
@@ -219,7 +215,7 @@ static int _vorbis_unpack_books(vorbis_info *vi,oggpack_buffer *opb){
   
   /* mode settings */
   ci->modes=oggpack_read(opb,6)+1;
-  /*vi->mode_param=_ogg_calloc(vi->modes,sizeof(void *));*/
+  if(ci->modes<=0)goto err_out;
   for(i=0;i<ci->modes;i++){
     ci->mode_param[i]=(vorbis_info_mode *)_ogg_calloc(1,sizeof(*ci->mode_param[i]));
     ci->mode_param[i]->blockflag=oggpack_read(opb,1);
@@ -230,6 +226,7 @@ static int _vorbis_unpack_books(vorbis_info *vi,oggpack_buffer *opb){
     if(ci->mode_param[i]->windowtype>=VI_WINDOWB)goto err_out;
     if(ci->mode_param[i]->transformtype>=VI_WINDOWB)goto err_out;
     if(ci->mode_param[i]->mapping>=ci->maps)goto err_out;
+    if(ci->mode_param[i]->mapping<0)goto err_out;
   }
   
   if(oggpack_read(opb,1)!=1)goto err_out; /* top level EOP check */

apps/codecs/libtremor/mapping0.c

@@ -128,19 +128,24 @@ static int ilog(unsigned int v){
 
 /* also responsible for range checking */
 static vorbis_info_mapping *mapping0_unpack(vorbis_info *vi,oggpack_buffer *opb){
-  int i;
+  int i,b;
   vorbis_info_mapping0 *info=(vorbis_info_mapping0 *)_ogg_calloc(1,sizeof(*info));
   codec_setup_info     *ci=(codec_setup_info *)vi->codec_setup;
   memset(info,0,sizeof(*info));
 
-  if(oggpack_read(opb,1))
+  b=oggpack_read(opb,1);
+  if(b<0)goto err_out;
+  if(b){
     info->submaps=oggpack_read(opb,4)+1;
-  else
+    if(info->submaps<=0)goto err_out;
+  }else
     info->submaps=1;
 
-  if(oggpack_read(opb,1)){
+  b=oggpack_read(opb,1);
+  if(b<0)goto err_out;
+  if(b){
     info->coupling_steps=oggpack_read(opb,8)+1;
-
+    if(info->coupling_steps<=0)goto err_out;
     for(i=0;i<info->coupling_steps;i++){
       int testM=info->coupling_mag[i]=oggpack_read(opb,ilog(vi->channels));
       int testA=info->coupling_ang[i]=oggpack_read(opb,ilog(vi->channels));
@@ -154,21 +159,22 @@ static vorbis_info_mapping *mapping0_unpack(vorbis_info *vi,oggpack_buffer *opb)
 
   }
 
-  if(oggpack_read(opb,2)>0)goto err_out; /* 2,3:reserved */
+  if(oggpack_read(opb,2)!=0)goto err_out; /* 2,3:reserved */
     
   if(info->submaps>1){
     for(i=0;i<vi->channels;i++){
       info->chmuxlist[i]=oggpack_read(opb,4);
-      if(info->chmuxlist[i]>=info->submaps)goto err_out;
+      if(info->chmuxlist[i]>=info->submaps || info->chmuxlist[i]<0)goto err_out;
     }
   }
   for(i=0;i<info->submaps;i++){
     int temp=oggpack_read(opb,8);
     if(temp>=ci->times)goto err_out;
     info->floorsubmap[i]=oggpack_read(opb,8);
-    if(info->floorsubmap[i]>=ci->floors)goto err_out;
+    if(info->floorsubmap[i]>=ci->floors || info->floorsubmap[i]<0)goto err_out;
     info->residuesubmap[i]=oggpack_read(opb,8);
-    if(info->residuesubmap[i]>=ci->residues)goto err_out;
+    if(info->residuesubmap[i]>=ci->residues || info->residuesubmap[i]<0)
+      goto err_out;
   }
 
   return info;