swr/rematrix: Fix handling of AV_CH_LAYOUT_STEREO_DOWNMIX output

Fixes Ticket2859

Note, testcases related to the downmix channels are welcome.
(id like to make sure this is working correctly now, as obviously it didnt
 work before ...)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

diff --git a/libswresample/rematrix.c b/libswresample/rematrix.c
index f15cf71..5c98e68 100644 (file)
--- a/libswresample/rematrix.c
+++ b/libswresample/rematrix.c
@@ -82,9 +82,6 @@ static int even(int64_t layout){
 }
 
 static int clean_layout(SwrContext *s, int64_t layout){
-    if((layout & AV_CH_LAYOUT_STEREO_DOWNMIX) == AV_CH_LAYOUT_STEREO_DOWNMIX)
-        return AV_CH_LAYOUT_STEREO;
-
     if(layout && layout != AV_CH_FRONT_CENTER && !(layout&(layout-1))) {
         char buf[128];
         av_get_channel_layout_string(buf, sizeof(buf), -1, layout);
@@ -125,6 +122,11 @@ av_cold static int auto_matrix(SwrContext *s)
     in_ch_layout = clean_layout(s, s->in_ch_layout);
     out_ch_layout = clean_layout(s, s->out_ch_layout);
 
+    if(   out_ch_layout == AV_CH_LAYOUT_STEREO_DOWNMIX
+       && (in_ch_layout & AV_CH_LAYOUT_STEREO_DOWNMIX) == 0
+    )
+        out_ch_layout = AV_CH_LAYOUT_STEREO;
+
     if(!sane_layout(in_ch_layout)){
         av_get_channel_layout_string(buf, sizeof(buf), -1, s->in_ch_layout);
         av_log(s, AV_LOG_ERROR, "Input channel layout '%s' is not supported\n", buf);