libtremor: merge upstream revision 17520 and 17521 tweaking some of the recently introduced chaecks to not reject files made with buggy beta encoders.

git-svn-id: svn://svn.rockbox.org/rockbox/trunk@28759 a1c6a512-1295-4272-9138-f99709370657

2 files changed, 8 insertions, 3 deletions

diff --git a/apps/codecs/libtremor/backends.h b/apps/codecs/libtremor/backends.h
index 29ab935993..0f85c8ec32 100644
--- a/apps/codecs/libtremor/backends.h
+++ b/apps/codecs/libtremor/backends.h
@@ -92,6 +92,7 @@ typedef struct vorbis_info_residue0{
   /* first stage (lossless partitioning) */
   int    grouping;         /* group n vectors per partition */
   int    partitions;       /* possible codebooks for a partition */
+  int    partvals;         /* partitions ^ groupbook dim */
   int    groupbook;        /* huffbook for partitioning */
   int    secondstages[64]; /* expanded out to pointers in lookup */
   int    booklist[256];    /* list of second stage books */
diff --git a/apps/codecs/libtremor/res012.c b/apps/codecs/libtremor/res012.c
index 9abe75aed2..a60cf7fa1e 100644
--- a/apps/codecs/libtremor/res012.c
+++ b/apps/codecs/libtremor/res012.c
@@ -114,6 +114,10 @@ static vorbis_info_residue *res0_unpack(vorbis_info *vi,oggpack_buffer *opb){
 
   /* verify the phrasebook is not specifying an impossible or
      inconsistent partitioning scheme. */
+  /* modify the phrasebook ranging check from r16327; an early beta
+     encoder had a bug where it used an oversized phrasebook by
+     accident.  These files should continue to be playable, but don't
+     allow an exploit */
   {
     int entries = ci->book_param[info->groupbook]->entries;
     int dim = ci->book_param[info->groupbook]->dim;
@@ -123,7 +127,7 @@ static vorbis_info_residue *res0_unpack(vorbis_info *vi,oggpack_buffer *opb){
       if(partvals > entries) goto errout;
       dim--;
     }
-    if(partvals != entries) goto errout;
+    info->partvals = partvals;
   }
 
   return(info);
@@ -220,7 +224,7 @@ static int _01inverse(vorbis_block *vb,vorbis_look_residue *vl,
           /* fetch the partition word for each channel */
           for(j=0;j<ch;j++){
             int temp=vorbis_book_decode(look->phrasebook,&vb->opb);
-            if(temp==-1)goto eopbreak;
+	    if(temp==-1 || temp>=info->partvals)goto eopbreak;
             partword[j][l]=look->decodemap[temp];
             if(partword[j][l]==NULL)goto errout;
           }
@@ -305,7 +309,7 @@ int res2_inverse(vorbis_block *vb,vorbis_look_residue *vl,
         if(s==0){
           /* fetch the partition word */
           int temp=vorbis_book_decode(look->phrasebook,&vb->opb);
-          if(temp==-1)goto eopbreak;
+	  if(temp==-1 || temp>info->partvals)goto eopbreak;
           partword[l]=look->decodemap[temp];
           if(partword[l]==NULL)goto errout;
         }