shorten: fix lossy decoding
authorPaul B Mahol <onemda@gmail.com>
Tue, 22 May 2012 11:56:18 +0000 (11:56 +0000)
committerPaul B Mahol <onemda@gmail.com>
Tue, 22 May 2012 11:56:18 +0000 (11:56 +0000)
Closes #1299, #1300 and #1301.

Signed-off-by: Paul B Mahol <onemda@gmail.com>
libavcodec/shorten.c

index b211e4f..a0b371d 100644 (file)
@@ -539,7 +539,7 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
                     sum += s->offset[channel][i];
                 coffset = sum / s->nmean;
                 if (s->version >= 2)
-                    coffset >>= FFMIN(1, s->bitshift);
+                    coffset = s->bitshift == 0 ? coffset : coffset >> s->bitshift - 1 >> 1;
             }
 
             /* decode samples for this channel */