Fix FS#12981 JPEG decoding problem when entropy data starts with FF

This changes JPEG fill and invalid byte handling to be like
mozjpeg, and bases entropy data start on SOS marker location.

Thanks to Stefan Waldmann and Dean Tersigni for reporting.

Change-Id: I3c79cc6ac8d714fdc75c12b57ba427d611c99519
Chaange-Id: Ibc7c17d38d5be63642bdaf6adfd6acc2a6cf4450

apps/plugins/imageviewer/jpeg/jpeg_decoder.c

@@ -507,29 +507,27 @@ static void idct8x8(unsigned char* p_byte, int* inptr, int* quantptr, int skip_l
 /* Preprocess the JPEG JFIF file */
 int process_markers(unsigned char* p_src, long size, struct jpeg* p_jpeg)
 {
-    unsigned char* p_bytes = p_src;
+    unsigned char* p_end = p_src + size;
     int marker_size; /* variable length of marker segment */
     int i, j, n;
     int ret = 0; /* returned flags */
 
-    p_jpeg->p_entropy_end = p_src + size;
+    p_jpeg->p_entropy_end = p_end;
 
-    while (p_src < p_bytes + size)
+    while (p_src < p_end)
     {
         if (*p_src++ != 0xFF) /* no marker? */
         {
-            p_src--; /* it's image data, put it back */
+            continue; /* discard */
-            p_jpeg->p_entropy_data = p_src;
-            break; /* exit marker processing */
         }
 
         switch (*p_src++)
         {
-        case 0xFF: /* Fill byte */
+        case 0xFF: /* Previous FF was fill byte */
-            ret |= FILL_FF;
+            p_src--; /* This FF could be start of a marker */
-        case 0x00: /* Zero stuffed byte - entropy data */
-            p_src--; /* put it back */
             continue;
+        case 0x00: /* Zero stuffed byte - discard */
+            break;
 
         case 0xC0: /* SOF Huff  - Baseline DCT */
             {
@@ -657,6 +655,8 @@ int process_markers(unsigned char* p_src, long size, struct jpeg* p_jpeg)
                     p_jpeg->scanheader[i].AC_select = *p_src++ & 0x0F;
                 }
                 p_src += 3; /* skip spectral information */
+                p_jpeg->p_entropy_data = p_src;
+                p_end = p_src; /* exit while loop */
             }
             break;
 

apps/recorder/jpeg_common.h

@@ -139,7 +139,6 @@ struct bitstream
 #define HUFFTAB   0x0001 /* with huffman table */
 #define QUANTTAB  0x0002 /* with quantization table */
 #define APP0_JFIF 0x0004 /* with APP0 segment following JFIF standard */
-#define FILL_FF   0x0008 /* with 0xFF padding bytes at begin/end */
 #define SOF0      0x0010 /* with SOF0-Segment */
 #define DHT       0x0020 /* with Definition of huffman tables */
 #define SOS       0x0040 /* with Start-of-Scan segment */

apps/recorder/jpeg_load.c

@@ -956,25 +956,25 @@ static int process_markers(struct jpeg* p_jpeg)
     int marker_size; /* variable length of marker segment */
     int i, j, n;
     int ret = 0; /* returned flags */
+    bool done = false;
 
-    while ((c = e_getc(p_jpeg, -1)))
+    while (!done && (c = e_getc(p_jpeg, -1)))
     {
         if (c != 0xFF) /* no marker? */
         {
             JDEBUGF("Non-marker data\n");
-            jpeg_putc(p_jpeg);
+            continue; /* discard */
-            break; /* exit marker processing */
         }
 
         c = e_getc(p_jpeg, -1);
         JDEBUGF("marker value %X\n",c);
         switch (c)
         {
-        case 0xFF: /* Fill byte */
+        case 0xFF: /* Previous FF was fill byte */
-            ret |= FILL_FF;
+            jpeg_putc(p_jpeg); /* This FF could be start of a marker */
-        case 0x00: /* Zero stuffed byte - entropy data */
-            jpeg_putc(p_jpeg);
             continue;
+        case 0x00: /* Zero stuffed byte */
+            break; /* discard */
 
         case 0xC0: /* SOF Huff  - Baseline DCT */
             {
@@ -1132,6 +1132,7 @@ static int process_markers(struct jpeg* p_jpeg)
                 }
                 /* skip spectral information */
                 e_skip_bytes(p_jpeg, marker_size);
+                done = true;
             }
             break;