avcodec/error_resilience: Fix integer overflow in filter181()
authorMichael Niedermayer <michael@niedermayer.cc>
Sun, 22 Apr 2018 19:46:05 +0000 (21:46 +0200)
committerMichael Niedermayer <michael@niedermayer.cc>
Mon, 9 Jul 2018 23:52:55 +0000 (01:52 +0200)
Fixes: runtime error: signed integer overflow: 197710 * 10923 cannot be represented in type 'int'
Fixes: 7010/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MPEG4_fuzzer-5667127596941312

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 1c97035e3b1677d6f0c5b6161ebfeffcf7bb638d)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
libavcodec/error_resilience.c

index 5a75fad..6bea0bf 100644 (file)
@@ -108,7 +108,7 @@ static void filter181(int16_t *data, int width, int height, int stride)
             dc = -prev_dc +
                  data[x     + y * stride] * 8 -
                  data[x + 1 + y * stride];
-            dc = (dc * 10923 + 32768) >> 16;
+            dc = (av_clip(dc, INT_MIN/10923, INT_MAX/10923 - 32768) * 10923 + 32768) >> 16;
             prev_dc = data[x + y * stride];
             data[x + y * stride] = dc;
         }
@@ -124,7 +124,7 @@ static void filter181(int16_t *data, int width, int height, int stride)
             dc = -prev_dc +
                  data[x +  y      * stride] * 8 -
                  data[x + (y + 1) * stride];
-            dc = (dc * 10923 + 32768) >> 16;
+            dc = (av_clip(dc, INT_MIN/10923, INT_MAX/10923 - 32768) * 10923 + 32768) >> 16;
             prev_dc = data[x + y * stride];
             data[x + y * stride] = dc;
         }