avcodec/hevc_cabac: Fix integer overflow in ff_hevc_cu_qp_delta_abs()

[ffmpeg.git] / libavcodec / sbrdsp_fixed.c

diff --git a/libavcodec/sbrdsp_fixed.c b/libavcodec/sbrdsp_fixed.c
index 5b7b7a6..07ef121 100644 (file)
--- a/libavcodec/sbrdsp_fixed.c
+++ b/libavcodec/sbrdsp_fixed.c
@@ -38,9 +38,14 @@ static SoftFloat sbr_sum_square_c(int (*x)[2], int n)
     int i, nz, round;
 
     for (i = 0; i < n; i += 2) {
+        // Larger values are inavlid and could cause overflows of accu.
+        av_assert2(FFABS(x[i + 0][0]) >> 29 == 0);
         accu += (int64_t)x[i + 0][0] * x[i + 0][0];
+        av_assert2(FFABS(x[i + 0][1]) >> 29 == 0);
         accu += (int64_t)x[i + 0][1] * x[i + 0][1];
+        av_assert2(FFABS(x[i + 1][0]) >> 29 == 0);
         accu += (int64_t)x[i + 1][0] * x[i + 1][0];
+        av_assert2(FFABS(x[i + 1][1]) >> 29 == 0);
         accu += (int64_t)x[i + 1][1] * x[i + 1][1];
     }
 
@@ -131,19 +136,19 @@ static av_always_inline void autocorrelate(const int x[40][2], SoftFloat phi[3][
 
     if (lag) {
         for (i = 1; i < 38; i++) {
-            accu_re += (int64_t)x[i][0] * x[i+lag][0];
-            accu_re += (int64_t)x[i][1] * x[i+lag][1];
-            accu_im += (int64_t)x[i][0] * x[i+lag][1];
-            accu_im -= (int64_t)x[i][1] * x[i+lag][0];
+            accu_re += (uint64_t)x[i][0] * x[i+lag][0];
+            accu_re += (uint64_t)x[i][1] * x[i+lag][1];
+            accu_im += (uint64_t)x[i][0] * x[i+lag][1];
+            accu_im -= (uint64_t)x[i][1] * x[i+lag][0];
         }
 
         real_sum = accu_re;
         imag_sum = accu_im;
 
-        accu_re += (int64_t)x[ 0][0] * x[lag][0];
-        accu_re += (int64_t)x[ 0][1] * x[lag][1];
-        accu_im += (int64_t)x[ 0][0] * x[lag][1];
-        accu_im -= (int64_t)x[ 0][1] * x[lag][0];
+        accu_re += (uint64_t)x[ 0][0] * x[lag][0];
+        accu_re += (uint64_t)x[ 0][1] * x[lag][1];
+        accu_im += (uint64_t)x[ 0][0] * x[lag][1];
+        accu_im -= (uint64_t)x[ 0][1] * x[lag][0];
 
         phi[2-lag][1][0] = autocorr_calc(accu_re);
         phi[2-lag][1][1] = autocorr_calc(accu_im);
@@ -151,28 +156,28 @@ static av_always_inline void autocorrelate(const int x[40][2], SoftFloat phi[3][
         if (lag == 1) {
             accu_re = real_sum;
             accu_im = imag_sum;
-            accu_re += (int64_t)x[38][0] * x[39][0];
-            accu_re += (int64_t)x[38][1] * x[39][1];
-            accu_im += (int64_t)x[38][0] * x[39][1];
-            accu_im -= (int64_t)x[38][1] * x[39][0];
+            accu_re += (uint64_t)x[38][0] * x[39][0];
+            accu_re += (uint64_t)x[38][1] * x[39][1];
+            accu_im += (uint64_t)x[38][0] * x[39][1];
+            accu_im -= (uint64_t)x[38][1] * x[39][0];
 
             phi[0][0][0] = autocorr_calc(accu_re);
             phi[0][0][1] = autocorr_calc(accu_im);
         }
     } else {
         for (i = 1; i < 38; i++) {
-            accu_re += (int64_t)x[i][0] * x[i][0];
-            accu_re += (int64_t)x[i][1] * x[i][1];
+            accu_re += (uint64_t)x[i][0] * x[i][0];
+            accu_re += (uint64_t)x[i][1] * x[i][1];
         }
         real_sum = accu_re;
-        accu_re += (int64_t)x[ 0][0] * x[ 0][0];
-        accu_re += (int64_t)x[ 0][1] * x[ 0][1];
+        accu_re += (uint64_t)x[ 0][0] * x[ 0][0];
+        accu_re += (uint64_t)x[ 0][1] * x[ 0][1];
 
         phi[2][1][0] = autocorr_calc(accu_re);
 
         accu_re = real_sum;
-        accu_re += (int64_t)x[38][0] * x[38][0];
-        accu_re += (int64_t)x[38][1] * x[38][1];
+        accu_re += (uint64_t)x[38][0] * x[38][0];
+        accu_re += (uint64_t)x[38][1] * x[38][1];
 
         phi[1][0][0] = autocorr_calc(accu_re);
     }
@@ -224,20 +229,22 @@ static void sbr_hf_gen_c(int (*X_high)[2], const int (*X_low)[2],
 static void sbr_hf_g_filt_c(int (*Y)[2], const int (*X_high)[40][2],
                           const SoftFloat *g_filt, int m_max, intptr_t ixh)
 {
-    int m, r;
+    int m;
     int64_t accu;
 
     for (m = 0; m < m_max; m++) {
-        r = 1 << (22-g_filt[m].exp);
-        accu = (int64_t)X_high[m][ixh][0] * ((g_filt[m].mant + 0x40)>>7);
-        Y[m][0] = (int)((accu + r) >> (23-g_filt[m].exp));
+        if (22 - g_filt[m].exp < 61) {
+            int64_t r = 1LL << (22-g_filt[m].exp);
+            accu = (int64_t)X_high[m][ixh][0] * ((g_filt[m].mant + 0x40)>>7);
+            Y[m][0] = (int)((accu + r) >> (23-g_filt[m].exp));
 
-        accu = (int64_t)X_high[m][ixh][1] * ((g_filt[m].mant + 0x40)>>7);
-        Y[m][1] = (int)((accu + r) >> (23-g_filt[m].exp));
+            accu = (int64_t)X_high[m][ixh][1] * ((g_filt[m].mant + 0x40)>>7);
+            Y[m][1] = (int)((accu + r) >> (23-g_filt[m].exp));
+        }
     }
 }
 
-static av_always_inline void sbr_hf_apply_noise(int (*Y)[2],
+static av_always_inline int sbr_hf_apply_noise(int (*Y)[2],
                                                 const SoftFloat *s_m,
                                                 const SoftFloat *q_filt,
                                                 int noise,
@@ -248,14 +255,17 @@ static av_always_inline void sbr_hf_apply_noise(int (*Y)[2],
     int m;
 
     for (m = 0; m < m_max; m++) {
-        int y0 = Y[m][0];
-        int y1 = Y[m][1];
+        unsigned y0 = Y[m][0];
+        unsigned y1 = Y[m][1];
         noise = (noise + 1) & 0x1ff;
         if (s_m[m].mant) {
             int shift, round;
 
             shift = 22 - s_m[m].exp;
-            if (shift < 30) {
+            if (shift < 1) {
+                av_log(NULL, AV_LOG_ERROR, "Overflow in sbr_hf_apply_noise, shift=%d\n", shift);
+                return AVERROR(ERANGE);
+            } else if (shift < 30) {
                 round = 1 << (shift-1);
                 y0 += (s_m[m].mant * phi_sign0 + round) >> shift;
                 y1 += (s_m[m].mant * phi_sign1 + round) >> shift;
@@ -265,7 +275,10 @@ static av_always_inline void sbr_hf_apply_noise(int (*Y)[2],
             int64_t accu;
 
             shift = 22 - q_filt[m].exp;
-            if (shift < 30) {
+            if (shift < 1) {
+                av_log(NULL, AV_LOG_ERROR, "Overflow in sbr_hf_apply_noise, shift=%d\n", shift);
+                return AVERROR(ERANGE);
+            } else if (shift < 30) {
                 round = 1 << (shift-1);
 
                 accu = (int64_t)q_filt[m].mant * ff_sbr_noise_table_fixed[noise][0];
@@ -281,6 +294,7 @@ static av_always_inline void sbr_hf_apply_noise(int (*Y)[2],
         Y[m][1] = y1;
         phi_sign1 = -phi_sign1;
     }
+    return 0;
 }
 
 #include "sbrdsp_template.c"