ff_emulated_edge_mc: fix integer anomalies, fix out of array reads
authorMichael Niedermayer <michaelni@gmx.at>
Fri, 16 Nov 2012 19:57:35 +0000 (20:57 +0100)
committerMichael Niedermayer <michaelni@gmx.at>
Fri, 16 Nov 2012 20:33:52 +0000 (21:33 +0100)
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
libavcodec/dsputil_template.c
libavcodec/x86/dsputil_mmx.c

index 0d6e8df..eacf59b 100644 (file)
@@ -131,10 +131,12 @@ void FUNC(ff_emulated_edge_mc)(uint8_t *buf, const uint8_t *src, int linesize, i
     int start_y, start_x, end_y, end_x;
 
     if(src_y>= h){
-        src+= (h-1-src_y)*linesize;
+        src-= src_y*linesize;
+        src+= (h-1)*linesize;
         src_y=h-1;
     }else if(src_y<=-block_h){
-        src+= (1-block_h-src_y)*linesize;
+        src-= src_y*linesize;
+        src+= (1-block_h)*linesize;
         src_y=1-block_h;
     }
     if(src_x>= w){
index ff338b0..2d9de06 100644 (file)
@@ -1878,10 +1878,12 @@ static av_always_inline void emulated_edge_mc(uint8_t *buf, const uint8_t *src,
     int start_y, start_x, end_y, end_x, src_y_add = 0;
 
     if (src_y >= h) {
-        src_y_add = h - 1 - src_y;
+        src -= src_y*linesize;
+        src_y_add = h - 1;
         src_y     = h - 1;
     } else if (src_y <= -block_h) {
-        src_y_add = 1 - block_h - src_y;
+        src -= src_y*linesize;
+        src_y_add = 1 - block_h;
         src_y     = 1 - block_h;
     }
     if (src_x >= w) {