swscale/utils: Fix for runtime error: left shift of negative value -1
authorMichael Niedermayer <michael@niedermayer.cc>
Fri, 4 Dec 2015 20:44:05 +0000 (21:44 +0100)
committerMichael Niedermayer <michael@niedermayer.cc>
Mon, 14 Dec 2015 15:51:01 +0000 (16:51 +0100)
Fixes: c106b36fa36db8ff8f3ed0c82be7bea2/asan_heap-oob_32699f0_6321_467b9a1d7e03d7cfd310b7e65dc53bcc.mov

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 325b59368dae3c3f2f5cc39873002b4cf133ccbc)

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
libswscale/utils.c

index 63fb05c..93cbad7 100644 (file)
@@ -384,7 +384,7 @@ static av_cold int initFilter(int16_t **outFilter, int32_t **filterPos,
 
         xDstInSrc = ((dstPos*(int64_t)xInc)>>7) - ((srcPos*0x10000LL)>>7);
         for (i = 0; i < dstW; i++) {
-            int xx = (xDstInSrc - ((int64_t)(filterSize - 2) << 16)) / (1 << 17);
+            int xx = (xDstInSrc - (filterSize - 2) * (1LL<<16)) / (1 << 17);
             int j;
             (*filterPos)[i] = xx;
             for (j = 0; j < filterSize; j++) {