dct cleanup

more accurate mmx dct (dont discard bits for fun)
fixing mmx quantizer bug for qscale%2==1 (bias was slightly wrong)

Originally committed as revision 895 to svn://svn.ffmpeg.org/ffmpeg/trunk

6 files changed, 26 insertions, 24 deletions

diff --git a/libavcodec/dct-test.c b/libavcodec/dct-test.c
index e03e1e6436..a46223ed27 100644
--- a/libavcodec/dct-test.c
+++ b/libavcodec/dct-test.c
@@ -101,8 +101,12 @@ void dct_error(const char *name, int is_idct,
         case 0: 
             for(i=0;i<64;i++)
                 block1[i] = (random() % 512) -256;
-            if (is_idct)
+            if (is_idct){
                 fdct(block1);
+
+                for(i=0;i<64;i++)
+                    block1[i]>>=3;
+            }
         break;
         case 1:{
             int num= (random()%10)+1;
@@ -153,17 +157,11 @@ void dct_error(const char *name, int is_idct,
 
         if (fdct_func == fdct_ifast) {
             for(i=0; i<64; i++) {
-                scale = (1 << (AANSCALE_BITS + 11)) / aanscales[i];
+                scale = 8*(1 << (AANSCALE_BITS + 11)) / aanscales[i];
                 block[i] = (block[i] * scale /*+ (1<<(AANSCALE_BITS-1))*/) >> AANSCALE_BITS;
             }
         }
 
-        if (fdct_func == ff_jpeg_fdct_islow) {
-            for(i=0; i<64; i++) {
-                block[i] = (block[i]+3)>>3;
-            }
-        }
-
         fdct_ref(block1);
 
         blockSumErr=0;
@@ -212,8 +210,12 @@ void dct_error(const char *name, int is_idct,
     case 0: 
         for(i=0;i<64;i++)
             block1[i] = (random() % 512) -256;
-        if (is_idct)
+        if (is_idct){
             fdct(block1);
+
+            for(i=0;i<64;i++)
+                block1[i]>>=3;
+        }
     break;
     case 1:{
     case 2:
diff --git a/libavcodec/fdctref.c b/libavcodec/fdctref.c
index 245492496f..ae376f7941 100644
--- a/libavcodec/fdctref.c
+++ b/libavcodec/fdctref.c
@@ -103,6 +103,7 @@ short *block;
         	s += c[i][5] * tmp[8 * 5 + j];
         	s += c[i][6] * tmp[8 * 6 + j];
         	s += c[i][7] * tmp[8 * 7 + j];
+		s*=8.0;
 
     		block[8 * i + j] = (short)floor(s + 0.499999);
 /*
diff --git a/libavcodec/i386/fdct_mmx.c b/libavcodec/i386/fdct_mmx.c
index 8c30818485..e43871b5cd 100644
--- a/libavcodec/i386/fdct_mmx.c
+++ b/libavcodec/i386/fdct_mmx.c
@@ -25,7 +25,7 @@
 
 #define BITS_FRW_ACC	3 //; 2 or 3 for accuracy
 #define SHIFT_FRW_COL	BITS_FRW_ACC
-#define SHIFT_FRW_ROW	(BITS_FRW_ACC + 17)
+#define SHIFT_FRW_ROW	(BITS_FRW_ACC + 17 - 3)
 //#define RND_FRW_ROW		(262144 * (BITS_FRW_ACC - 1)) //; 1 << (SHIFT_FRW_ROW-1)
 #define RND_FRW_ROW		(1 << (SHIFT_FRW_ROW-1))
 //#define RND_FRW_COL		(2 * (BITS_FRW_ACC - 1)) //; 1 << (SHIFT_FRW_COL-1)
diff --git a/libavcodec/i386/mpegvideo_mmx_template.c b/libavcodec/i386/mpegvideo_mmx_template.c
index 6c3e52ea5f..185baaa420 100644
--- a/libavcodec/i386/mpegvideo_mmx_template.c
+++ b/libavcodec/i386/mpegvideo_mmx_template.c
@@ -46,9 +46,9 @@ static int RENAME(dct_quantize)(MpegEncContext *s,
     if (s->mb_intra) {
         int dummy;
         if (n < 4)
-            q = s->y_dc_scale;
+            q = s->y_dc_scale<<3;
         else
-            q = s->c_dc_scale;
+            q = s->c_dc_scale<<3;
         /* note: block[0] is assumed to be positive */
         if (!s->h263_aic) {
 #if 1
@@ -70,7 +70,7 @@ static int RENAME(dct_quantize)(MpegEncContext *s,
 #endif
         } else
             /* For AIC we skip quant/dequant of INTRADC */
-            level = block[0];
+            level = block[0]>>3;
             
         block[0]=0; //avoid fake overflow
 //        temp_block[0] = (block[0] + (q >> 1)) / q;
diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index f480a61dcd..7a15b761e4 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -90,7 +90,7 @@ static void convert_matrix(MpegEncContext *s, int (*qmat)[64], uint16_t (*qmat16
                 /* (1<<36)/19952 >= (1<<36)/(aanscales[i] * qscale * quant_matrix[i]) >= (1<<36)/249205026 */
                 /* 3444240       >= (1<<36)/(aanscales[i] * qscale * quant_matrix[i]) >= 275 */
                 
-                qmat[qscale][j] = (int)((UINT64_C(1) << (QMAT_SHIFT-3)) / 
+                qmat[qscale][j] = (int)((UINT64_C(1) << QMAT_SHIFT) / 
                                 (qscale * quant_matrix[j]));
             }
         } else if (s->fdct == fdct_ifast) {
@@ -101,7 +101,7 @@ static void convert_matrix(MpegEncContext *s, int (*qmat)[64], uint16_t (*qmat16
                 /* (1<<36)/19952 >= (1<<36)/(aanscales[i] * qscale * quant_matrix[i]) >= (1<<36)/249205026 */
                 /* 3444240       >= (1<<36)/(aanscales[i] * qscale * quant_matrix[i]) >= 275 */
                 
-                qmat[qscale][j] = (int)((UINT64_C(1) << (QMAT_SHIFT + 11)) / 
+                qmat[qscale][j] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) / 
                                 (aanscales[i] * qscale * quant_matrix[j]));
             }
         } else {
@@ -115,7 +115,6 @@ static void convert_matrix(MpegEncContext *s, int (*qmat)[64], uint16_t (*qmat16
                 qmat16[qscale][i] = (1 << QMAT_SHIFT_MMX) / (qscale * quant_matrix[block_permute_op(i)]);
 
                 if(qmat16[qscale][i]==0 || qmat16[qscale][i]==128*256) qmat16[qscale][i]=128*256-1;
-
                 qmat16_bias[qscale][i]= ROUNDED_DIV(bias<<(16-QUANT_BIAS_SHIFT), qmat16[qscale][i]);
             }
         }
@@ -2487,15 +2486,15 @@ static int dct_quantize_c(MpegEncContext *s,
         i = 1;
         last_non_zero = 0;
         qmat = s->q_intra_matrix[qscale];
-        bias= s->intra_quant_bias<<(QMAT_SHIFT - 3 - QUANT_BIAS_SHIFT);
+        bias= s->intra_quant_bias<<(QMAT_SHIFT - QUANT_BIAS_SHIFT);
     } else {
         i = 0;
         last_non_zero = -1;
         qmat = s->q_inter_matrix[qscale];
-        bias= s->inter_quant_bias<<(QMAT_SHIFT - 3 - QUANT_BIAS_SHIFT);
+        bias= s->inter_quant_bias<<(QMAT_SHIFT - QUANT_BIAS_SHIFT);
     }
-    threshold1= (1<<(QMAT_SHIFT - 3)) - bias - 1;
-    threshold2= threshold1<<1;
+    threshold1= (1<<QMAT_SHIFT) - bias - 1;
+    threshold2= (threshold1<<1);
 
     for(;i<64;i++) {
         j = zigzag_direct[i];
@@ -2506,10 +2505,10 @@ static int dct_quantize_c(MpegEncContext *s,
 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
         if(((unsigned)(level+threshold1))>threshold2){
             if(level>0){
-                level= (bias + level)>>(QMAT_SHIFT - 3);
+                level= (bias + level)>>QMAT_SHIFT;
                 block[j]= level;
             }else{
-                level= (bias - level)>>(QMAT_SHIFT - 3);
+                level= (bias - level)>>QMAT_SHIFT;
                 block[j]= -level;
             }
             max |=level;
diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h
index 6b7377b246..00e6ebe46e 100644
--- a/libavcodec/mpegvideo.h
+++ b/libavcodec/mpegvideo.h
@@ -30,8 +30,8 @@ enum OutputFormat {
 
 #define MPEG_BUF_SIZE (16 * 1024)
 
-#define QMAT_SHIFT_MMX 19
-#define QMAT_SHIFT 25
+#define QMAT_SHIFT_MMX 16
+#define QMAT_SHIFT 22
 
 #define MAX_FCODE 7
 #define MAX_MV 2048