avcodec/takdec: Fix integer overflows in decode_subframe()
authorMichael Niedermayer <michael@niedermayer.cc>
Fri, 22 Sep 2017 18:45:26 +0000 (20:45 +0200)
committerMichael Niedermayer <michael@niedermayer.cc>
Mon, 25 Sep 2017 09:09:10 +0000 (11:09 +0200)
Fixes: runtime error: signed integer overflow: -1562477869 + -691460395 cannot be represented in type 'int'
Fixes: 3196/clusterfuzz-testcase-minimized-4528307146063872

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 3dabb9c69db114b1f30c30e0a2788cffc50bac40)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
libavcodec/takdec.c

index 0fedc78..886ca8a 100644 (file)
@@ -472,10 +472,10 @@ static int decode_subframe(TAKDecContext *s, int32_t *decoded,
                 v += (unsigned)s->adsp.scalarproduct_int16(&s->residues[i], s->filter,
                                                  filter_order & -16);
             for (j = filter_order & -16; j < filter_order; j += 4) {
-                v += s->residues[i + j + 3] * s->filter[j + 3] +
-                     s->residues[i + j + 2] * s->filter[j + 2] +
-                     s->residues[i + j + 1] * s->filter[j + 1] +
-                     s->residues[i + j    ] * s->filter[j    ];
+                v += s->residues[i + j + 3] * (unsigned)s->filter[j + 3] +
+                     s->residues[i + j + 2] * (unsigned)s->filter[j + 2] +
+                     s->residues[i + j + 1] * (unsigned)s->filter[j + 1] +
+                     s->residues[i + j    ] * (unsigned)s->filter[j    ];
             }
             v = (av_clip(v >> filter_quant, -8192, 8191) << dshift) - *decoded;
             *decoded++ = v;