diracdec: fix edge emulation check, fixes out of array reads
authorMichael Niedermayer <michaelni@gmx.at>
Sun, 11 Nov 2012 02:34:09 +0000 (03:34 +0100)
committerMichael Niedermayer <michaelni@gmx.at>
Sun, 11 Nov 2012 02:34:40 +0000 (03:34 +0100)
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
libavcodec/diracdec.c

index 13d9e47..a8f5938 100644 (file)
@@ -1404,8 +1404,8 @@ static int mc_subpel(DiracContext *s, DiracBlock *block, const uint8_t *src[5],
     }
 
     /* fixme: v/h _edge_pos */
-    if ((unsigned)x > p->width +EDGE_WIDTH/2 - p->xblen ||
-        (unsigned)y > p->height+EDGE_WIDTH/2 - p->yblen) {
+    if ((unsigned)x > FFMAX(p->width +EDGE_WIDTH/2 - p->xblen, 0) ||
+        (unsigned)y > FFMAX(p->height+EDGE_WIDTH/2 - p->yblen, 0)) {
         for (i = 0; i < nplanes; i++) {
             ff_emulated_edge_mc(s->edge_emu_buffer[i], src[i], p->stride,
                                 p->xblen, p->yblen, x, y,