Merge commit 'd002fce24a7206e95f9492031f45adc35711e183'
authorMichael Niedermayer <michaelni@gmx.at>
Thu, 12 Sep 2013 09:55:08 +0000 (11:55 +0200)
committerMichael Niedermayer <michaelni@gmx.at>
Thu, 12 Sep 2013 09:55:08 +0000 (11:55 +0200)
* commit 'd002fce24a7206e95f9492031f45adc35711e183':
  smacker: Check malloc return values

Conflicts:
libavcodec/smacker.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>
1  2 
libavcodec/smacker.c

@@@ -204,41 -200,38 +204,48 @@@ static int smacker_decode_header_tree(S
      tmp2.bits = av_mallocz(256 * 4);
      tmp2.lengths = av_mallocz(256 * sizeof(int));
      tmp2.values = av_mallocz(256 * sizeof(int));
+     if (!tmp1.bits || !tmp1.lengths || !tmp1.values ||
+         !tmp2.bits || !tmp2.lengths || !tmp2.values) {
+         err = AVERROR(ENOMEM);
+         goto error;
+     }
  
      if(get_bits1(gb)) {
 -        smacker_decode_tree(gb, &tmp1, 0, 0);
 +        res = smacker_decode_tree(gb, &tmp1, 0, 0);
 +        if (res < 0)
 +            return res;
          skip_bits1(gb);
 -        res = init_vlc(&vlc[0], SMKTREE_BITS, tmp1.length,
 -                    tmp1.lengths, sizeof(int), sizeof(int),
 -                    tmp1.bits, sizeof(uint32_t), sizeof(uint32_t), INIT_VLC_LE);
 -        if(res < 0) {
 -            av_log(smk->avctx, AV_LOG_ERROR, "Cannot build VLC table\n");
 -            err = res;
 -            goto error;
 +        if(tmp1.current > 1) {
 +            res = init_vlc(&vlc[0], SMKTREE_BITS, tmp1.length,
 +                        tmp1.lengths, sizeof(int), sizeof(int),
 +                        tmp1.bits, sizeof(uint32_t), sizeof(uint32_t), INIT_VLC_LE);
 +            if(res < 0) {
 +                av_log(smk->avctx, AV_LOG_ERROR, "Cannot build VLC table\n");
-                 return AVERROR_INVALIDDATA;
++                err = res;
++                goto error;
 +            }
          }
 -    } else {
 +    }
 +    if (!vlc[0].table) {
          av_log(smk->avctx, AV_LOG_ERROR, "Skipping low bytes tree\n");
      }
      if(get_bits1(gb)){
 -        smacker_decode_tree(gb, &tmp2, 0, 0);
 +        res = smacker_decode_tree(gb, &tmp2, 0, 0);
 +        if (res < 0)
 +            return res;
          skip_bits1(gb);
 -        res = init_vlc(&vlc[1], SMKTREE_BITS, tmp2.length,
 -                    tmp2.lengths, sizeof(int), sizeof(int),
 -                    tmp2.bits, sizeof(uint32_t), sizeof(uint32_t), INIT_VLC_LE);
 -        if(res < 0) {
 -            av_log(smk->avctx, AV_LOG_ERROR, "Cannot build VLC table\n");
 -            err = res;
 -            goto error;
 +        if(tmp2.current > 1) {
 +            res = init_vlc(&vlc[1], SMKTREE_BITS, tmp2.length,
 +                        tmp2.lengths, sizeof(int), sizeof(int),
 +                        tmp2.bits, sizeof(uint32_t), sizeof(uint32_t), INIT_VLC_LE);
 +            if(res < 0) {
 +                av_log(smk->avctx, AV_LOG_ERROR, "Cannot build VLC table\n");
-                 return AVERROR_INVALIDDATA;
++                err = res;
++                goto error;
 +            }
          }
 -    } else {
 +    }
 +    if (!vlc[1].table) {
          av_log(smk->avctx, AV_LOG_ERROR, "Skipping high bytes tree\n");
      }