vorbis: fix overflows in floor1[] vector and inverse db table index.

diff --git a/libavcodec/vorbis.c b/libavcodec/vorbis.c
index 0b26870..52ded8b 100644 (file)
--- a/libavcodec/vorbis.c
+++ b/libavcodec/vorbis.c
@@ -152,7 +152,7 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
     }
 }
 
     }
 }
 

-static inline void render_line_unrolled(intptr_t x, uint8_t y, int x1,
+static inline void render_line_unrolled(intptr_t x, int y, int x1,

                                         intptr_t sy, int ady, int adx,
                                         float *buf)
 {
                                         intptr_t sy, int ady, int adx,
                                         float *buf)
 {


@@ -164,30 +164,30 @@ static inline void render_line_unrolled(intptr_t x, uint8_t y, int x1,
         if (err >= 0) {
             err += ady - adx;
             y   += sy;
         if (err >= 0) {
             err += ady - adx;
             y   += sy;

-            buf[x++] = ff_vorbis_floor1_inverse_db_table[y];
+            buf[x++] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)];

         }
         }

-        buf[x] = ff_vorbis_floor1_inverse_db_table[y];
+        buf[x] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)];

     }
     if (x <= 0) {
         if (err + ady >= 0)
             y += sy;
     }
     if (x <= 0) {
         if (err + ady >= 0)
             y += sy;

-        buf[x] = ff_vorbis_floor1_inverse_db_table[y];
+        buf[x] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)];

     }
 }
 
     }
 }
 

-static void render_line(int x0, uint8_t y0, int x1, int y1, float *buf)
+static void render_line(int x0, int y0, int x1, int y1, float *buf)

 {
     int dy  = y1 - y0;
     int adx = x1 - x0;
     int ady = FFABS(dy);
     int sy  = dy < 0 ? -1 : 1;
 {
     int dy  = y1 - y0;
     int adx = x1 - x0;
     int ady = FFABS(dy);
     int sy  = dy < 0 ? -1 : 1;

-    buf[x0] = ff_vorbis_floor1_inverse_db_table[y0];
+    buf[x0] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y0)];

     if (ady*2 <= adx) { // optimized common case
         render_line_unrolled(x0, y0, x1, sy, ady, adx, buf);
     } else {
         int base  = dy / adx;
         int x     = x0;
     if (ady*2 <= adx) { // optimized common case
         render_line_unrolled(x0, y0, x1, sy, ady, adx, buf);
     } else {
         int base  = dy / adx;
         int x     = x0;

-        uint8_t y = y0;
+        int y     = y0;

         int err   = -adx;
         ady -= FFABS(base) * adx;
         while (++x < x1) {
         int err   = -adx;
         ady -= FFABS(base) * adx;
         while (++x < x1) {


@@ -197,7 +197,7 @@ static void render_line(int x0, uint8_t y0, int x1, int y1, float *buf)
                 err -= adx;
                 y   += sy;
             }
                 err -= adx;
                 y   += sy;
             }

-            buf[x] = ff_vorbis_floor1_inverse_db_table[y];
+            buf[x] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)];

         }
     }
 }
         }
     }
 }


@@ -206,8 +206,7 @@ void ff_vorbis_floor1_render_list(vorbis_floor1_entry * list, int values,
                                   uint16_t *y_list, int *flag,
                                   int multiplier, float *out, int samples)
 {
                                   uint16_t *y_list, int *flag,
                                   int multiplier, float *out, int samples)
 {

-    int lx, i;
-    uint8_t ly;
+    int lx, ly, i;

     lx = 0;
     ly = y_list[0] * multiplier;
     for (i = 1; i < values; i++) {
     lx = 0;
     ly = y_list[0] * multiplier;
     for (i = 1; i < values; i++) {

diff --git a/libavcodec/vorbisdec.c b/libavcodec/vorbisdec.c
index 2f1477f..aa4bdff 100644 (file)
--- a/libavcodec/vorbisdec.c
+++ b/libavcodec/vorbisdec.c
@@ -1244,20 +1244,20 @@ static int vorbis_floor1_decode(vorbis_context *vc,
             floor1_flag[i]               = 1;
             if (val >= room) {
                 if (highroom > lowroom) {
             floor1_flag[i]               = 1;
             if (val >= room) {
                 if (highroom > lowroom) {

-                    floor1_Y_final[i] = val - lowroom + predicted;
+                    floor1_Y_final[i] = av_clip_uint16(val - lowroom + predicted);

                 } else {
                 } else {

-                    floor1_Y_final[i] = predicted - val + highroom - 1;
+                    floor1_Y_final[i] = av_clip_uint16(predicted - val + highroom - 1);

                 }
             } else {
                 if (val & 1) {
                 }
             } else {
                 if (val & 1) {

-                    floor1_Y_final[i] = predicted - (val + 1) / 2;
+                    floor1_Y_final[i] = av_clip_uint16(predicted - (val + 1) / 2);

                 } else {
                 } else {

-                    floor1_Y_final[i] = predicted + val / 2;
+                    floor1_Y_final[i] = av_clip_uint16(predicted + val / 2);

                 }
             }
         } else {
             floor1_flag[i]    = 0;
                 }
             }
         } else {
             floor1_flag[i]    = 0;

-            floor1_Y_final[i] = predicted;
+            floor1_Y_final[i] = av_clip_uint16(predicted);

         }
 
         av_dlog(NULL, " Decoded floor(%d) = %u / val %u\n",
         }
 
         av_dlog(NULL, " Decoded floor(%d) = %u / val %u\n",