avcodec/mpegvideo_enc: Check for integer overflow in ff_mpv_reallocate_putbitbuffer()
authorMichael Niedermayer <michael@niedermayer.cc>
Thu, 21 Jan 2016 14:39:43 +0000 (15:39 +0100)
committerMichael Niedermayer <michael@niedermayer.cc>
Thu, 21 Jan 2016 14:56:11 +0000 (15:56 +0100)
Fixes assertion failure
Fixes: 6568d187979ce17878b6fe5fbbb89142/signal_sigabrt_7ffff6ae7cb7_7176_564bbc6741bdcf907f5c4e685c9a77a2.mpg

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
libavcodec/mpegvideo_enc.c

index 55550d5..af5b85f 100644 (file)
@@ -2845,6 +2845,11 @@ int ff_mpv_reallocate_putbitbuffer(MpegEncContext *s, size_t threshold, size_t s
         uint8_t *new_buffer = NULL;
         int new_buffer_size = 0;
 
+        if ((s->avctx->internal->byte_buffer_size + size_increase) >= INT_MAX/8) {
+            av_log(s->avctx, AV_LOG_ERROR, "Cannot reallocate putbit buffer\n");
+            return AVERROR(ENOMEM);
+        }
+
         av_fast_padded_malloc(&new_buffer, &new_buffer_size,
                               s->avctx->internal->byte_buffer_size + size_increase);
         if (!new_buffer)