vp9: just disallow temporal or no-update segmentation on size-change.
authorRonald S. Bultje <rsbultje@gmail.com>
Sun, 9 Feb 2014 01:47:44 +0000 (20:47 -0500)
committerClément Bœsch <u@pkh.me>
Sun, 9 Feb 2014 17:07:14 +0000 (18:07 +0100)
The spec doesn't describe how it should be decoded so this is probably
the safest thing to do. Fixes valgrind errors on fuzzed11.ivf and fixes
valgrind errors on fuzzed10.ivf differently.

libavcodec/vp9.c

index 7080658..8099c76 100644 (file)
@@ -110,7 +110,6 @@ typedef struct VP9Context {
     uint8_t keyframe, last_keyframe;
     uint8_t invisible;
     uint8_t use_last_frame_mvs;
-    uint8_t use_last_frame_segmap;
     uint8_t errorres;
     uint8_t colorspace;
     uint8_t fullrange;
@@ -279,7 +278,7 @@ static int vp9_alloc_frame(AVCodecContext *ctx, VP9Frame *f)
 
     // retain segmentation map if it doesn't update
     if (s->segmentation.enabled && !s->segmentation.update_map &&
-        s->use_last_frame_segmap) {
+        !s->intraonly && !s->keyframe) {
         memcpy(f->segmentation_map, s->frames[LAST_FRAME].segmentation_map, sz);
     }
 
@@ -622,14 +621,19 @@ static int decode_frame_header(AVCodecContext *ctx,
             for (i = 0; i < 7; i++)
                 s->prob.seg[i] = get_bits1(&s->gb) ?
                                  get_bits(&s->gb, 8) : 255;
-            if ((s->segmentation.temporal = get_bits1(&s->gb)))
+            if ((s->segmentation.temporal = get_bits1(&s->gb))) {
                 for (i = 0; i < 3; i++)
                     s->prob.segpred[i] = get_bits1(&s->gb) ?
                                          get_bits(&s->gb, 8) : 255;
-        } else {
-            s->use_last_frame_segmap = !s->keyframe && !s->intraonly &&
-                s->frames[CUR_FRAME].tf.f->width == w &&
-                s->frames[CUR_FRAME].tf.f->height == h;
+            }
+        }
+        if ((!s->segmentation.update_map || s->segmentation.temporal) &&
+            (w != s->frames[CUR_FRAME].tf.f->width ||
+             h != s->frames[CUR_FRAME].tf.f->height)) {
+            av_log(ctx, AV_LOG_ERROR,
+                   "Reference segmap (temp=%d,update=%d) enabled on size-change!\n",
+                   s->segmentation.temporal, s->segmentation.update_map);
+            return AVERROR_INVALIDDATA;
         }
 
         if (get_bits1(&s->gb)) {
@@ -1284,8 +1288,7 @@ static void decode_mode(AVCodecContext *ctx)
     int h4 = FFMIN(s->rows - row, bwh_tab[1][b->bs][1]), y;
     int have_a = row > 0, have_l = col > s->tiling.tile_col_start;
 
-    if (!s->segmentation.enabled ||
-        (!s->segmentation.update_map && !s->use_last_frame_segmap)) {
+    if (!s->segmentation.enabled) {
         b->seg_id = 0;
     } else if (s->keyframe || s->intraonly) {
         b->seg_id = vp8_rac_get_tree(&s->c, vp9_segmentation_tree, s->prob.seg);