avcodec/jpeg2000: Only allocate Jpeg2000Pass for the encoder
authorMichael Niedermayer <michael@niedermayer.cc>
Thu, 30 Nov 2017 20:51:56 +0000 (21:51 +0100)
committerMichael Niedermayer <michael@niedermayer.cc>
Fri, 1 Dec 2017 22:12:07 +0000 (23:12 +0100)
Reduces memory needed.
Fixes: OOM
Fixes: 4427/clusterfuzz-testcase-minimized-5106919271301120

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
libavcodec/j2kenc.c
libavcodec/jpeg2000.c
libavcodec/jpeg2000.h

index 78ec88a..3e542af 100644 (file)
@@ -941,7 +941,9 @@ static int encode_tile(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile, int tileno
                         }
                         if (!prec->cblk[cblkno].data)
                             prec->cblk[cblkno].data = av_malloc(1 + 8192);
-                        if (!prec->cblk[cblkno].data)
+                        if (!prec->cblk[cblkno].passes)
+                            prec->cblk[cblkno].passes = av_malloc_array(JPEG2000_MAX_PASSES, sizeof (*prec->cblk[cblkno].passes));
+                        if (!prec->cblk[cblkno].data || !prec->cblk[cblkno].passes)
                             return AVERROR(ENOMEM);
                         encode_cblk(s, &t1, prec->cblk + cblkno, tile, xx1 - xx0, yy1 - yy0,
                                     bandpos, codsty->nreslevels - reslevelno - 1);
index 8551cf8..5f39650 100644 (file)
@@ -606,6 +606,7 @@ void ff_jpeg2000_cleanup(Jpeg2000Component *comp, Jpeg2000CodingStyle *codsty)
                         for (cblkno = 0; cblkno < nb_code_blocks; cblkno ++) {
                             Jpeg2000Cblk *cblk = &prec->cblk[cblkno];
                             av_freep(&cblk->data);
+                            av_freep(&cblk->passes);
                         }
                         av_freep(&prec->cblk);
                     }
index eaf7faf..752feae 100644 (file)
@@ -173,7 +173,7 @@ typedef struct Jpeg2000Cblk {
     int nb_terminations;
     int nb_terminationsinc;
     int data_start[JPEG2000_MAX_PASSES];
-    Jpeg2000Pass passes[JPEG2000_MAX_PASSES];
+    Jpeg2000Pass *passes;
     int coord[2][2]; // border coordinates {{x0, x1}, {y0, y1}}
 } Jpeg2000Cblk; // code block