avcodec/qtrleenc: fix undefined behaviour
authorPaul B Mahol <onemda@gmail.com>
Wed, 25 Sep 2019 15:18:32 +0000 (17:18 +0200)
committerPaul B Mahol <onemda@gmail.com>
Sun, 29 Sep 2019 19:05:35 +0000 (21:05 +0200)
Fixes #7991.

libavcodec/qtrleenc.c

index cdd864b..6669c13 100644 (file)
@@ -259,9 +259,10 @@ static void qtrle_encode_line(QtrleEncContext *s, const AVFrame *p, int line, ui
         /* These bulk costs increase every iteration */
         lowest_bulk_cost += s->pixel_size;
         sec_lowest_bulk_cost += s->pixel_size;
-
-        this_line -= s->pixel_size;
-        prev_line -= s->pixel_size;
+        if (this_line >= p->data[0] + s->pixel_size)
+            this_line -= s->pixel_size;
+        if (prev_line >= s->previous_frame->data[0] + s->pixel_size)
+            prev_line -= s->pixel_size;
     }
 
     /* Good! Now we have the best sequence for this line, let's output it. */