avcodec/h264: Check mode before considering mixed mode intra prediction
authorMichael Niedermayer <michaelni@gmx.at>
Sat, 4 Oct 2014 12:51:46 +0000 (14:51 +0200)
committerMichael Niedermayer <michaelni@gmx.at>
Sat, 4 Oct 2014 21:52:29 +0000 (23:52 +0200)
Fixes out of array read
Fixes: asan_heap-oob_e476fc_2_asan_heap-oob_1333ec6_61_CAMACI3_Sony_C.jsv

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 9734a7a1de3043f012ad0f1ef11027d9488067e6)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
libavcodec/h264.c

index 69fb047..c4f4b05 100644 (file)
@@ -215,18 +215,18 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma)
 
     if ((h->left_samples_available & 0x8080) != 0x8080) {
         mode = left[mode];
-        if (is_chroma && (h->left_samples_available & 0x8080)) {
-            // mad cow disease mode, aka MBAFF + constrained_intra_pred
-            mode = ALZHEIMER_DC_L0T_PRED8x8 +
-                   (!(h->left_samples_available & 0x8000)) +
-                   2 * (mode == DC_128_PRED8x8);
-        }
         if (mode < 0) {
             av_log(h->avctx, AV_LOG_ERROR,
                    "left block unavailable for requested intra mode at %d %d\n",
                    h->mb_x, h->mb_y);
             return AVERROR_INVALIDDATA;
         }
+        if (is_chroma && (h->left_samples_available & 0x8080)) {
+            // mad cow disease mode, aka MBAFF + constrained_intra_pred
+            mode = ALZHEIMER_DC_L0T_PRED8x8 +
+                   (!(h->left_samples_available & 0x8000)) +
+                   2 * (mode == DC_128_PRED8x8);
+        }
     }
 
     return mode;