huffyuvdec: Check init_vlc() return codes.
authorMichael Niedermayer <michaelni@gmx.at>
Tue, 29 Jan 2013 17:29:41 +0000 (18:29 +0100)
committerMichael Niedermayer <michaelni@gmx.at>
Sun, 22 Sep 2013 20:52:12 +0000 (22:52 +0200)
Prevents out of array writes

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit f67a0d115254461649470452058fa3c28c0df294)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 95ab8d33e1a680f30a5a9605175112008ab81afc)

Conflicts:

libavcodec/huffyuv.c
(cherry picked from commit 277def59fce10d91e3113e5c0f63e22bc4abfa88)

Conflicts:

libavcodec/huffyuv.c

libavcodec/huffyuv.c

index 574daac..071c490 100644 (file)
@@ -33,6 +33,7 @@
 #include "put_bits.h"
 #include "dsputil.h"
 #include "thread.h"
+#include "libavutil/avassert.h"
 
 #define VLC_BITS 11
 
@@ -315,6 +316,7 @@ static void generate_joint_tables(HYuvContext *s){
                     int len1 = s->len[p][u];
                     if(len1 > limit)
                         continue;
+                    av_assert0(i < (1 << VLC_BITS));
                     len[i] = len0 + len1;
                     bits[i] = (s->bits[0][y] << len1) + s->bits[p][u];
                     symbols[i] = (y<<8) + u;
@@ -348,6 +350,7 @@ static void generate_joint_tables(HYuvContext *s){
                     int len2 = s->len[2][r&255];
                     if(len2 > limit1)
                         continue;
+                    av_assert0(i < (1 << VLC_BITS));
                     len[i] = len0 + len1 + len2;
                     bits[i] = (code << len2) + s->bits[2][r&255];
                     if(s->decorrelate){
@@ -371,6 +374,7 @@ static void generate_joint_tables(HYuvContext *s){
 static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length){
     GetBitContext gb;
     int i;
+    int ret;
 
     init_get_bits(&gb, src, length*8);
 
@@ -381,7 +385,8 @@ static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length){
             return -1;
         }
         free_vlc(&s->vlc[i]);
-        init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, s->bits[i], 4, 4, 0);
+        if ((ret = init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, s->bits[i], 4, 4, 0)) < 0)
+            return ret;
     }
 
     generate_joint_tables(s);
@@ -393,6 +398,7 @@ static int read_old_huffman_tables(HYuvContext *s){
 #if 1
     GetBitContext gb;
     int i;
+    int ret;
 
     init_get_bits(&gb, classic_shift_luma, sizeof(classic_shift_luma)*8);
     if(read_len_table(s->len[0], &gb)<0)
@@ -413,7 +419,8 @@ static int read_old_huffman_tables(HYuvContext *s){
 
     for(i=0; i<3; i++){
         free_vlc(&s->vlc[i]);
-        init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, s->bits[i], 4, 4, 0);
+        if ((ret = init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1, s->bits[i], 4, 4, 0)) < 0)
+            return ret;
     }
 
     generate_joint_tables(s);