avfilter/vf_transpose: Fix used plane count.
authorMichael Niedermayer <michael@niedermayer.cc>
Wed, 24 Jan 2018 18:38:05 +0000 (19:38 +0100)
committerMichael Niedermayer <michael@niedermayer.cc>
Mon, 19 Feb 2018 01:40:54 +0000 (02:40 +0100)
Fixes out of array access
Fixes: poc.mp4

Found-by: GwanYeong Kim <gy741.kim@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit c6939f65a116b1ffed345d29d8621ee4ffb32235)
(cherry picked from commit 3f621455d62e46745453568d915badd5b1e5bcd5)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
libavfilter/vf_transpose.c

index 9555ff2..8f2ff8e 100644 (file)
@@ -27,6 +27,7 @@
 
 #include <stdio.h>
 
+#include "libavutil/avassert.h"
 #include "libavutil/imgutils.h"
 #include "libavutil/internal.h"
 #include "libavutil/intreadwrite.h"
@@ -54,6 +55,7 @@ enum TransposeDir {
 typedef struct TransContext {
     const AVClass *class;
     int hsub, vsub;
+    int planes;
     int pixsteps[4];
 
     int passthrough;    ///< PassthroughType, landscape passthrough mode enabled
@@ -106,6 +108,10 @@ static int config_props_output(AVFilterLink *outlink)
 
     s->hsub = desc_in->log2_chroma_w;
     s->vsub = desc_in->log2_chroma_h;
+    s->planes = av_pix_fmt_count_planes(outlink->format);
+
+    av_assert0(desc_in->nb_components == desc_out->nb_components);
+
 
     av_image_fill_max_pixsteps(s->pixsteps, NULL, desc_out);
 
@@ -148,7 +154,7 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr,
     AVFrame *in = td->in;
     int plane;
 
-    for (plane = 0; out->data[plane]; plane++) {
+    for (plane = 0; plane < s->planes; plane++) {
         int hsub    = plane == 1 || plane == 2 ? s->hsub : 0;
         int vsub    = plane == 1 || plane == 2 ? s->vsub : 0;
         int pixstep = s->pixsteps[plane];