vp3dec: Check coefficient index in vp3_dequant()
authorReinhard Tartler <siretart@tauware.de>
Sun, 4 Dec 2011 09:10:33 +0000 (10:10 +0100)
committerReinhard Tartler <siretart@tauware.de>
Fri, 23 Dec 2011 14:56:01 +0000 (15:56 +0100)
Based on a patch by Michael Niedermayer <michaelni@gmx.at>

Fixes NGS00145, CVE-2011-4352

Found-by: Phillip Langlois
Signed-off-by: Reinhard Tartler <siretart@tauware.de>
(cherry picked from commit 8b94df0f2047e9728cb872adc9e64557b7a5152f)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>
libavcodec/vp3.c

index c117a64..890db5c 100644 (file)
@@ -1291,6 +1291,10 @@ static inline int vp3_dequant(Vp3DecodeContext *s, Vp3Fragment *frag,
         case 1: // zero run
             s->dct_tokens[plane][i]++;
             i += (token >> 2) & 0x7f;
+            if (i > 63) {
+                av_log(s->avctx, AV_LOG_ERROR, "Coefficient index overflow\n");
+                return i;
+            }
             block[perm[i]] = (token >> 9) * dequantizer[perm[i]];
             i++;
             break;
@@ -1493,7 +1497,10 @@ static void render_slice(Vp3DecodeContext *s, int slice)
                     /* invert DCT and place (or add) in final output */
 
                     if (s->all_fragments[i].coding_method == MODE_INTRA) {
-                        vp3_dequant(s, s->all_fragments + i, plane, 0, block);
+                        int index;
+                        index = vp3_dequant(s, s->all_fragments + i, plane, 0, block);
+                        if (index > 63)
+                            continue;
                         if(s->avctx->idct_algo!=FF_IDCT_VP3)
                             block[0] += 128<<3;
                         s->dsp.idct_put(
@@ -1501,7 +1508,10 @@ static void render_slice(Vp3DecodeContext *s, int slice)
                             stride,
                             block);
                     } else {
-                        if (vp3_dequant(s, s->all_fragments + i, plane, 1, block)) {
+                        int index = vp3_dequant(s, s->all_fragments + i, plane, 1, block);
+                        if (index > 63)
+                            continue;
+                        if (index > 0) {
                         s->dsp.idct_add(
                             output_plane + first_pixel,
                             stride,