postproc/postprocess: fix quant store for fq mode
authorMichael Niedermayer <michaelni@gmx.at>
Sun, 12 Oct 2014 18:26:27 +0000 (20:26 +0200)
committerMichael Niedermayer <michaelni@gmx.at>
Sun, 2 Nov 2014 10:14:25 +0000 (11:14 +0100)
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 941aaa39e8cd78ba4d16dfcec767290aec9a0136)

Conflicts:

tests/ref/fate/filter-pp3
(cherry picked from commit 705748caf3f6a4a3e74ad3d2fc547a5a0213a521)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
libpostproc/postprocess.c
tests/ref/lavfi/pp3

index facfd2c..3a1b78d 100644 (file)
@@ -1000,7 +1000,7 @@ void  pp_postprocess(const uint8_t * src[3], const int srcStride[3],
     if((pict_type&7)!=3){
         if (QPStride >= 0){
             int i;
-            const int count= mbHeight * QPStride;
+            const int count= mbHeight * FFMAX(QPStride, mbWidth);
             for(i=0; i<(count>>2); i++){
                 ((uint32_t*)c->nonBQPTable)[i] = ((const uint32_t*)QP_store)[i] & 0x3F3F3F3F;
             }
index ccf2eeb..1af8761 100644 (file)
@@ -1 +1 @@
-pp3                 39af1a30d0ea0e906df264773adfcaa6
+pp3                 c8277ef31ab01bad51356841c9634522