rockbox: fix iriver firmware validation for larger bootloaders

It turns out the code for the firmware size validation was incorrectly
using the total length of the firmware file to check if it was too big
for the flash rom which caused it to incorrectly flag larger bootloaders
as too big even though they would still fit.

As it happens this situation can only really arise after mkboot has been
run on the decoded firmware image. Because mkboot writes the actual binary
size to the ESTFBINR header we will be using that to check if the firmware
image is too large for the flash rom.

Now because this information is embedded in the decrypted / encrypted
region we have to collect it as the region is processed so the validation
is also moved to after the data processing loop.

Change-Id: I8bfead73812fe4e59f08fbbe8956790604dcb4e2

tools/fwpatcher/iriver.c

@@ -101,7 +101,8 @@ int iriver_decode(TCHAR *infile_name, TCHAR *outfile_name, unsigned int modify,
     FILE * outfile = NULL;
     int i = -1;
     unsigned char headerdata[512];
-    unsigned long dwLength1, dwLength2, dwLength3, fp = 0;
+    unsigned int dwLength1, dwLength2, dwLength3, fp = 0;
+    unsigned int minsize, maxsize, sizes[2];
     unsigned char blockdata[16+16];
     unsigned char out[16];
     unsigned char newmunge;
@@ -138,10 +139,7 @@ int iriver_decode(TCHAR *infile_name, TCHAR *outfile_name, unsigned int modify,
     dwLength3 = headerdata[8] | (headerdata[9]<<8) |
         (headerdata[10]<<16) | (headerdata[11]<<24);
 
-    if( dwLength1 < firmware_minsize[ i ] ||
+    if( dwLength2 > dwLength1 ||
-        dwLength1 > firmware_maxsize[ i ] ||
-        dwLength2 < firmware_minsize[ i ] ||
-        dwLength2 > dwLength1 ||
         dwLength3 > dwLength1 ||
         dwLength2>>9 != dwLength3 ||
         dwLength2+dwLength3+512 != dwLength1 )
@@ -151,6 +149,10 @@ int iriver_decode(TCHAR *infile_name, TCHAR *outfile_name, unsigned int modify,
 		goto error;
     };
 
+    minsize = firmware_minsize[i];
+    maxsize = firmware_maxsize[i];
+    sizes[0] = sizes[1] = 0;
+
     pChecksums = ppChecksums = (unsigned char *)( malloc( dwLength3 ) );
 
     if( modify )
@@ -177,6 +179,12 @@ int iriver_decode(TCHAR *infile_name, TCHAR *outfile_name, unsigned int modify,
             ck += out[i];
         }
 
+        if (fp <= 32)
+            sizes[fp / 16 - 1] = (out[0] << 24) |
+                                 (out[1] << 16) |
+                                 (out[2] <<  8) |
+                                 (out[3] <<  0);
+
         if( fp > ESTF_SIZE || stripmode != STRIP_HEADER_CHECKSUM_ESTF )
         {
             fwrite( out+4, 1, 12, outfile );
@@ -204,6 +212,16 @@ int iriver_decode(TCHAR *infile_name, TCHAR *outfile_name, unsigned int modify,
             s+=16;
     };
 
+    if( sizes[0] < minsize ||
+        sizes[1] < minsize ||
+        sizes[0] > maxsize ||
+        sizes[1] > maxsize )
+    {
+        fprintf( stderr, "This doesn't look like a valid encrypted "
+                 "iHP firmware - reason: ESTFBINR 'length' data\n" );
+        goto error;
+    };
+
     if( fp != dwLength2 )
     {
         fprintf( stderr, "This doesn't look like a valid encrypted "
@@ -269,7 +287,8 @@ int iriver_encode(TCHAR *infile_name, TCHAR *outfile_name, unsigned int modify )
     FILE * outfile = NULL;
     int i = -1;
     unsigned char headerdata[512];
-    unsigned long dwLength1, dwLength2, dwLength3, fp = 0;
+    unsigned int dwLength1, dwLength2, dwLength3, fp = 0;
+    unsigned int minsize, maxsize, sizes[2];
     unsigned char blockdata[16+16];
     unsigned char out[16];
     unsigned char newmunge;
@@ -313,10 +332,7 @@ int iriver_encode(TCHAR *infile_name, TCHAR *outfile_name, unsigned int modify )
     dwLength3 = headerdata[8] | (headerdata[9]<<8) |
         (headerdata[10]<<16) | (headerdata[11]<<24);
 
-    if( dwLength1 < firmware_minsize[i] ||
+    if( dwLength2 > dwLength1 ||
-        dwLength1 > firmware_maxsize[i] ||
-        dwLength2 < firmware_minsize[i] ||
-        dwLength2 > dwLength1 ||
         dwLength3 > dwLength1 ||
         dwLength2+dwLength3+512 != dwLength1 )
     {
@@ -325,6 +341,10 @@ int iriver_encode(TCHAR *infile_name, TCHAR *outfile_name, unsigned int modify )
         goto error;
     };
 
+    minsize = firmware_minsize[i];
+    maxsize = firmware_maxsize[i];
+    sizes[0] = sizes[1] = 0;
+
     pChecksums = ppChecksums = (unsigned char *)( malloc( dwLength3 ) );
 
     fwrite( headerdata, 512, 1, outfile );
@@ -335,6 +355,13 @@ int iriver_encode(TCHAR *infile_name, TCHAR *outfile_name, unsigned int modify )
            ( lenread = fread( blockdata+16, 1, 16, infile ) ) == 16 )
     {
         fp += 16;
+
+        if (fp <= 32)
+            sizes[fp / 16 - 1] = (blockdata[28] << 24) |
+                                 (blockdata[29] << 16) |
+                                 (blockdata[30] <<  8) |
+                                 (blockdata[31] <<  0);
+
         for( i=0; i<16; ++i )
         {
             newmunge = blockdata[16+((12+i)&0xf)] ^ blockdata[i];
@@ -355,6 +382,16 @@ int iriver_encode(TCHAR *infile_name, TCHAR *outfile_name, unsigned int modify )
             s+=16;
     };
 
+    if( sizes[0] < minsize ||
+        sizes[1] < minsize ||
+        sizes[0] > maxsize ||
+        sizes[1] > maxsize )
+    {
+        fprintf( stderr, "This doesn't look like a valid decoded iHP"
+                 " firmware - reason: ESTFBINR 'length' data\n" );
+        goto error;
+    };
+
     if( fp != dwLength2 )
     {
         fprintf( stderr, "This doesn't look like a valid decoded "

tools/iriver.c

@@ -103,7 +103,8 @@ int iriver_decode(const char *infile_name, const char *outfile_name, BOOL modify
     FILE * outfile = NULL;
     int i = -1;
     unsigned char headerdata[512];
-    unsigned long dwLength1, dwLength2, dwLength3, fp = 0;
+    unsigned int dwLength1, dwLength2, dwLength3, fp = 0;
+    unsigned int minsize, maxsize, sizes[2];
     unsigned char blockdata[16+16];
     unsigned char out[16];
     unsigned char newmunge;
@@ -143,10 +144,7 @@ int iriver_decode(const char *infile_name, const char *outfile_name, BOOL modify
     dwLength3 = headerdata[8] | (headerdata[9]<<8) |
         (headerdata[10]<<16) | (headerdata[11]<<24);
 
-    if( dwLength1 < firmware_minsize[ i ] ||
+    if( dwLength2 > dwLength1 ||
-        dwLength1 > firmware_maxsize[ i ] ||
-        dwLength2 < firmware_minsize[ i ] ||
-        dwLength2 > dwLength1 ||
         dwLength3 > dwLength1 ||
         dwLength2>>9 != dwLength3 ||
         dwLength2+dwLength3+512 != dwLength1 )
@@ -158,6 +156,10 @@ int iriver_decode(const char *infile_name, const char *outfile_name, BOOL modify
         return -3;
     };
 
+    minsize = firmware_minsize[i];
+    maxsize = firmware_maxsize[i];
+    sizes[0] = sizes[1] = 0;
+
     pChecksums = ppChecksums = (unsigned char *)( malloc( dwLength3 ) );
 
     if( modify )
@@ -184,6 +186,12 @@ int iriver_decode(const char *infile_name, const char *outfile_name, BOOL modify
             ck += out[i];
         }
 
+        if (fp <= 32)
+            sizes[fp / 16 - 1] = (out[0] << 24) |
+                                 (out[1] << 16) |
+                                 (out[2] <<  8) |
+                                 (out[3] <<  0);
+
         if( fp > ESTF_SIZE || stripmode != STRIP_HEADER_CHECKSUM_ESTF )
         {
             fwrite( out+4, 1, 12, outfile );
@@ -211,13 +219,25 @@ int iriver_decode(const char *infile_name, const char *outfile_name, BOOL modify
             s+=16;
     };
 
+    if( sizes[0] < minsize ||
+        sizes[1] < minsize ||
+        sizes[0] > maxsize ||
+        sizes[1] > maxsize )
+    {
+        fprintf( stderr, "This doesn't look like a valid encrypted "
+                 "iHP firmware - reason: ESTFBINR 'length' data\n" );
+        fclose(infile);
+        fclose(outfile);
+        return -4;
+    };
+
     if( fp != dwLength2 )
     {
         fprintf( stderr, "This doesn't look like a valid encrypted "
                  "iHP firmware - reason: 'length2' mismatch\n" );
         fclose(infile);
         fclose(outfile);
-        return -4;
+        return -5;
     };
 
     fp = 0;
@@ -234,7 +254,7 @@ int iriver_decode(const char *infile_name, const char *outfile_name, BOOL modify
                      "iHP firmware - reason: Checksum mismatch!" );
             fclose(infile);
             fclose(outfile);
-            return -5;
+            return -6;
         };
         ppChecksums += lenread;
     };
@@ -245,7 +265,7 @@ int iriver_decode(const char *infile_name, const char *outfile_name, BOOL modify
                  "iHP firmware - reason: 'length3' mismatch\n" );
         fclose(infile);
         fclose(outfile);
-        return -6;
+        return -7;
     };
 
 
@@ -276,7 +296,8 @@ int iriver_encode(const char *infile_name, const char *outfile_name, BOOL modify
     FILE * outfile = NULL;
     int i = -1;
     unsigned char headerdata[512];
-    unsigned long dwLength1, dwLength2, dwLength3, fp = 0;
+    unsigned int dwLength1, dwLength2, dwLength3, fp = 0;
+    unsigned int minsize, maxsize, sizes[2];
     unsigned char blockdata[16+16];
     unsigned char out[16];
     unsigned char newmunge;
@@ -321,10 +342,7 @@ int iriver_encode(const char *infile_name, const char *outfile_name, BOOL modify
     dwLength3 = headerdata[8] | (headerdata[9]<<8) |
         (headerdata[10]<<16) | (headerdata[11]<<24);
 
-    if( dwLength1 < firmware_minsize[i] ||
+    if( dwLength2 > dwLength1 ||
-        dwLength1 > firmware_maxsize[i] ||
-        dwLength2 < firmware_minsize[i] ||
-        dwLength2 > dwLength1 ||
         dwLength3 > dwLength1 ||
         dwLength2+dwLength3+512 != dwLength1 )
     {
@@ -335,6 +353,10 @@ int iriver_encode(const char *infile_name, const char *outfile_name, BOOL modify
         return -3;
     };
 
+    minsize = firmware_minsize[i];
+    maxsize = firmware_maxsize[i];
+    sizes[0] = sizes[1] = 0;
+
     pChecksums = ppChecksums = (unsigned char *)( malloc( dwLength3 ) );
 
     fwrite( headerdata, 512, 1, outfile );
@@ -345,6 +367,13 @@ int iriver_encode(const char *infile_name, const char *outfile_name, BOOL modify
            ( lenread = fread( blockdata+16, 1, 16, infile ) ) == 16 )
     {
         fp += 16;
+
+        if (fp <= 32)
+            sizes[fp / 16 - 1] = (blockdata[28] << 24) |
+                                 (blockdata[29] << 16) |
+                                 (blockdata[30] <<  8) |
+                                 (blockdata[31] <<  0);
+
         for( i=0; i<16; ++i )
         {
             newmunge = blockdata[16+((12+i)&0xf)] ^ blockdata[i];
@@ -365,13 +394,25 @@ int iriver_encode(const char *infile_name, const char *outfile_name, BOOL modify
             s+=16;
     };
 
+    if( sizes[0] < minsize ||
+        sizes[1] < minsize ||
+        sizes[0] > maxsize ||
+        sizes[1] > maxsize )
+    {
+        fprintf( stderr, "This doesn't look like a valid decoded iHP"
+                 " firmware - reason: ESTFBINR 'length' data\n" );
+        fclose(infile);
+        fclose(outfile);
+        return -4;
+    };
+
     if( fp != dwLength2 )
     {
         fprintf( stderr, "This doesn't look like a valid decoded "
                  "iHP firmware - reason: 'length1' mismatch\n" );
         fclose(infile);
         fclose(outfile);
-        return -4;
+        return -5;
     };
 
     /* write out remainder w/out applying descrambler */
@@ -392,7 +433,7 @@ int iriver_encode(const char *infile_name, const char *outfile_name, BOOL modify
                  "iHP firmware - reason: 'length2' mismatch\n" );
         fclose(infile);
         fclose(outfile);
-        return -5;
+        return -6;
     };
 
     fprintf( stderr, "File encoded successfully and checksum table built!\n" );