pnmdec: make sure v is capped by maxval
authorAndreas Cadhalpun <Andreas.Cadhalpun@googlemail.com>
Wed, 9 Nov 2016 00:09:35 +0000 (01:09 +0100)
committerAndreas Cadhalpun <Andreas.Cadhalpun@googlemail.com>
Sat, 26 Nov 2016 23:46:35 +0000 (00:46 +0100)
Otherwise put_bits can be called with a value that doesn't fit in the
sample_len, causing an assertion failure.

Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun@googlemail.com>
(cherry picked from commit cdb5479c9ddc886f0b8661db585405ebab343e80)
Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun@googlemail.com>
libavcodec/pnmdec.c

index d4261a4..4e2045d 100644 (file)
@@ -43,7 +43,7 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
     int buf_size         = avpkt->size;
     PNMContext * const s = avctx->priv_data;
     AVFrame * const p    = data;
-    int i, j, n, linesize, h, upgrade = 0, is_mono = 0;
+    int i, j, k, n, linesize, h, upgrade = 0, is_mono = 0;
     unsigned char *ptr;
     int components, sample_len, ret;
 
@@ -143,10 +143,14 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
                         v = (*s->bytestream++)&1;
                     } else {
                         /* read a sequence of digits */
-                        do {
+                        for (k = 0; k < 5 && c <= 9; k += 1) {
                             v = 10*v + c;
                             c = (*s->bytestream++) - '0';
-                        } while (c <= 9);
+                        }
+                        if (v > s->maxval) {
+                            av_log(avctx, AV_LOG_ERROR, "value %d larger than maxval %d\n", v, s->maxval);
+                            return AVERROR_INVALIDDATA;
+                        }
                     }
                     if (sample_len == 16) {
                         ((uint16_t*)ptr)[j] = (((1<<sample_len)-1)*v + (s->maxval>>1))/s->maxval;