j2kdec: Fix integer overflow leading to a segfault
authorMichael Niedermayer <michaelni@gmx.at>
Fri, 16 Dec 2011 03:16:01 +0000 (04:16 +0100)
committerMichael Niedermayer <michaelni@gmx.at>
Thu, 12 Jan 2012 20:52:31 +0000 (21:52 +0100)
Fixes Ticket776
Bug found by: Diana Elena Muscalu

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 1f99939a6361e2e6d6788494dd7c682b051c6c34)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
libavcodec/j2k_dwt.c

index ab7a1ab..48aa337 100644 (file)
@@ -321,7 +321,7 @@ int ff_j2k_dwt_init(DWTContext *s, uint16_t border[2][2], int decomp_levels, int
     int i, j, lev = decomp_levels, maxlen,
         b[2][2];
 
-    if (decomp_levels >= FF_DWT_MAX_DECLVLS)
+    if ((unsigned)decomp_levels >= FF_DWT_MAX_DECLVLS)
         return AVERROR_INVALIDDATA;
     s->ndeclevels = decomp_levels;
     s->type = type;