lpcm: fix sample size calculation for 20bit LCPM.
authorRonald S. Bultje <rsbultje@gmail.com>
Wed, 7 Mar 2012 04:08:17 +0000 (20:08 -0800)
committerReinhard Tartler <siretart@tauware.de>
Tue, 13 Mar 2012 22:12:00 +0000 (23:12 +0100)
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
(cherry picked from commit f1320dc3bed281bb2f3c5531c52b6a6246e2394a)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>
libavcodec/pcm-mpeg.c

index 9ab6fc3..f010b97 100644 (file)
@@ -156,7 +156,7 @@ static int pcm_bluray_decode_frame(AVCodecContext *avctx, void *data,
 
     /* There's always an even number of channels in the source */
     num_source_channels = FFALIGN(avctx->channels, 2);
-    sample_size = (num_source_channels * avctx->bits_per_coded_sample) >> 3;
+    sample_size = (num_source_channels * (avctx->sample_fmt == AV_SAMPLE_FMT_S16 ? 16 : 24)) >> 3;
     samples = buf_size / sample_size;
 
     /* get output buffer */