avcodec/h264idct_template: fix multiple runtime error: signed integer overflow
authorMichael Niedermayer <michael@niedermayer.cc>
Wed, 15 Mar 2017 01:58:16 +0000 (02:58 +0100)
committerMichael Niedermayer <michael@niedermayer.cc>
Thu, 16 Mar 2017 15:16:05 +0000 (16:16 +0100)
Fixes: 857/clusterfuzz-testcase-5319093760557056

Benchmark changes from 335->333 (so if its not a random fluctuation then it would be faster)

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
libavcodec/h264idct_template.c

index c627160..229a9ac 100644 (file)
@@ -304,7 +304,7 @@ void FUNCC(ff_h264_chroma422_dc_dequant_idct)(int16_t *_block, int qmul){
 void FUNCC(ff_h264_chroma_dc_dequant_idct)(int16_t *_block, int qmul){
     const int stride= 16*2;
     const int xStride= 16;
-    int a,b,c,d,e;
+    SUINT a,b,c,d,e;
     dctcoef *block = (dctcoef*)_block;
 
     a= block[stride*0 + xStride*0];
@@ -317,8 +317,8 @@ void FUNCC(ff_h264_chroma_dc_dequant_idct)(int16_t *_block, int qmul){
     b= c-d;
     c= c+d;
 
-    block[stride*0 + xStride*0]= ((a+c)*qmul) >> 7;
-    block[stride*0 + xStride*1]= ((e+b)*qmul) >> 7;
-    block[stride*1 + xStride*0]= ((a-c)*qmul) >> 7;
-    block[stride*1 + xStride*1]= ((e-b)*qmul) >> 7;
+    block[stride*0 + xStride*0]= (int)((a+c)*qmul) >> 7;
+    block[stride*0 + xStride*1]= (int)((e+b)*qmul) >> 7;
+    block[stride*1 + xStride*0]= (int)((a-c)*qmul) >> 7;
+    block[stride*1 + xStride*1]= (int)((e-b)*qmul) >> 7;
 }