resample: Fix overflow
authorMichael Niedermayer <michaelni@gmx.at>
Thu, 27 Oct 2011 12:31:53 +0000 (14:31 +0200)
committerMichael Niedermayer <michaelni@gmx.at>
Sun, 6 Nov 2011 18:49:12 +0000 (19:49 +0100)
Found-by: Jim Radford
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 6ae93d030476ddd7fa2ab4d9d2dd25df85725390)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
libavcodec/resample2.c

index ac9db73..0147819 100644 (file)
@@ -190,8 +190,10 @@ AVResampleContext *av_resample_init(int out_rate, int in_rate, int filter_size,
     memcpy(&c->filter_bank[c->filter_length*phase_count+1], c->filter_bank, (c->filter_length-1)*sizeof(FELEM));
     c->filter_bank[c->filter_length*phase_count]= c->filter_bank[c->filter_length - 1];
 
-    c->src_incr= out_rate;
-    c->ideal_dst_incr= c->dst_incr= in_rate * phase_count;
+    if(!av_reduce(&c->src_incr, &c->dst_incr, out_rate, in_rate * (int64_t)phase_count, INT32_MAX/2))
+        return NULL;
+    c->ideal_dst_incr= c->dst_incr;
+
     c->index= -phase_count*((c->filter_length-1)/2);
 
     return c;