avformat/utils: Prevent undefined shift with wrap_bits > 64.
authorDale Curtis <dalecurtis@chromium.org>
Fri, 17 Nov 2017 21:35:56 +0000 (13:35 -0800)
committerMichael Niedermayer <michael@niedermayer.cc>
Wed, 31 Jan 2018 21:56:14 +0000 (22:56 +0100)
2LL << (wrap_bits=64 - 1) does not fit in int64_t; change the
code to use a uint64_t (2ULL) and add an av_assert2() to
ensure wrap_bits <= 64.

Signed-off-by: Dale Curtis <dalecurtis@chromium.org>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 03fbc0daa7e37af024f8b017a28105c32bbe25ca)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
libavformat/utils.c

index f14174d..2f68cf3 100644 (file)
@@ -1501,13 +1501,14 @@ int av_read_frame(AVFormatContext *s, AVPacket *pkt)
 
             if (next_pkt->dts != AV_NOPTS_VALUE) {
                 int wrap_bits = s->streams[next_pkt->stream_index]->pts_wrap_bits;
+                av_assert2(wrap_bits <= 64);
                 // last dts seen for this stream. if any of packets following
                 // current one had no dts, we will set this to AV_NOPTS_VALUE.
                 int64_t last_dts = next_pkt->dts;
                 while (pktl && next_pkt->pts == AV_NOPTS_VALUE) {
                     if (pktl->pkt.stream_index == next_pkt->stream_index &&
-                        (av_compare_mod(next_pkt->dts, pktl->pkt.dts, 2LL << (wrap_bits - 1)) < 0)) {
-                        if (av_compare_mod(pktl->pkt.pts, pktl->pkt.dts, 2LL << (wrap_bits - 1))) {
+                        av_compare_mod(next_pkt->dts, pktl->pkt.dts, 2ULL << (wrap_bits - 1)) < 0) {
+                        if (av_compare_mod(pktl->pkt.pts, pktl->pkt.dts, 2ULL << (wrap_bits - 1))) {
                             // not B-frame
                             next_pkt->pts = pktl->pkt.dts;
                         }