fixing another assert q>0.0 issue caused by variance < 0, this fix allso changes the inter/intra decission very slightly -> all regression checksums need to be updated

Originally committed as revision 1122 to svn://svn.ffmpeg.org/ffmpeg/trunk

3 files changed, 22 insertions, 24 deletions

diff --git a/libavcodec/motion_est.c b/libavcodec/motion_est.c
index 547c619526..e94657dcaa 100644
--- a/libavcodec/motion_est.c
+++ b/libavcodec/motion_est.c
@@ -1135,14 +1135,13 @@ void ff_estimate_p_frame_motion(MpegEncContext * s,
     
     sum = pix_sum(pix, s->linesize);
     
-    sum= (sum+8)>>4;
-    varc = (pix_norm1(pix, s->linesize) - sum*sum + 500 + 128)>>8;
+    varc = (pix_norm1(pix, s->linesize) - (((unsigned)(sum*sum))>>8) + 500 + 128)>>8;
     vard = (pix_norm(pix, ppix, s->linesize)+128)>>8;
 
 //printf("%d %d %d %X %X %X\n", s->mb_width, mb_x, mb_y,(int)s, (int)s->mb_var, (int)s->mc_mb_var); fflush(stdout);
     s->mb_var   [s->mb_width * mb_y + mb_x] = varc;
     s->mc_mb_var[s->mb_width * mb_y + mb_x] = vard;
-    s->mb_mean  [s->mb_width * mb_y + mb_x] = (sum+7)>>4;
+    s->mb_mean  [s->mb_width * mb_y + mb_x] = (sum+128)>>8;
     s->mb_var_sum    += varc;
     s->mc_mb_var_sum += vard;
 //printf("E%d %d %d %X %X %X\n", s->mb_width, mb_x, mb_y,(int)s, (int)s->mb_var, (int)s->mc_mb_var); fflush(stdout);
diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index 4503e3e1e7..1e68e03e11 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -2590,11 +2590,10 @@ static void encode_picture(MpegEncContext *s, int picture_number)
                     int varc;
                     int sum = pix_sum(pix, s->linesize);
     
-                    sum= (sum+8)>>4;
-                    varc = (pix_norm1(pix, s->linesize) - sum*sum + 500 + 128)>>8;
+                    varc = (pix_norm1(pix, s->linesize) - (((unsigned)(sum*sum))>>8) + 500 + 128)>>8;
 
                     s->mb_var [s->mb_width * mb_y + mb_x] = varc;
-                    s->mb_mean[s->mb_width * mb_y + mb_x] = (sum+7)>>4;
+                    s->mb_mean[s->mb_width * mb_y + mb_x] = (sum+128)>>8;
                     s->mb_var_sum    += varc;
                 }
             }
diff --git a/tests/ffmpeg.regression.ref b/tests/ffmpeg.regression.ref
index 2b1a39064e..3f48fc0009 100644
--- a/tests/ffmpeg.regression.ref
+++ b/tests/ffmpeg.regression.ref
@@ -1,26 +1,26 @@
 ffmpeg regression test
-67be2deabb210740966ca5fda9a64bca *./data/a-mpeg1.mpg
-59a66b108e8bbd3867c4ee27f4bbe2ca *./data/out.yuv
-20f237122350f1248d9d51489bb79092 *./data/a-msmpeg4v2.avi
-74675eb582376359fbcf76ff2fb034b6 *./data/out.yuv
-d6367d6abb406b438cb57a7ebc4b927d *./data/a-msmpeg4.avi
-951aa2df4ef4c1eeee3ccc8904de8e48 *./data/out.yuv
-50b4590a3615a556361d75db8c6d43b3 *./data/a-wmv1.avi
-3ed8e11a8a8147c3e7d736593fccae79 *./data/out.yuv
-abe11239875a32f00fa2910828bba4fb *./data/a-h263.avi
-c1f6c8ee7a24d8345deddf1a24ca3756 *./data/out.yuv
-f022e9efcc88abcce80cc25232bc1cb1 *./data/a-h263p.avi
-d53012811b639e12a88476f2c82de0ca *./data/out.yuv
-8e41ac530894eaf3ecdb583d6d66c392 *./data/a-odivx.avi
-bf3be8fb78a9ca91dd6b252d48ce59bb *./data/out.yuv
-850ba6da4c9dad60b6719b0b06b204de *./data/a-mpeg4-rc.avi
-0ab0a58082e485478f9feab5ccc6fc42 *./data/out.yuv
+68c8a6bd81f69764063eabc2a66c9a59 *./data/a-mpeg1.mpg
+4971324b08d4bf0a98a708d3b1a0799b *./data/out.yuv
+01410257b88ee5d1c390737d4274bd3f *./data/a-msmpeg4v2.avi
+a1d54e9d006cc31a5b71dabadbb4df53 *./data/out.yuv
+6a9d8f162eb59b8bbf7c737ede86ef8b *./data/a-msmpeg4.avi
+bdd8dc1182fa731f54c3b5ebf9002e40 *./data/out.yuv
+8c242521ffa3854c774b3ac49c85aff4 *./data/a-wmv1.avi
+aeb7bc27b29d894e2dc558e9e2dce089 *./data/out.yuv
+30abed5bc6a127b04b73bb436ad3b55a *./data/a-h263.avi
+0059bed3addc017d9f089f3256769a51 *./data/out.yuv
+78b315bb537cc6e04bdd2a8843a4a7d8 *./data/a-h263p.avi
+1077099de78ec2b3a573c2ee0fd0303a *./data/out.yuv
+5b1886c7f754244126cb007b12db6fca *./data/a-odivx.avi
+46795a8e36efb498801a8edba6c76425 *./data/out.yuv
+1e5c838d1645e8a43f1323d615a5afce *./data/a-mpeg4-rc.avi
+e8166d2605192eebafa54d702d4691da *./data/out.yuv
 685812396445ac5c79f345965dca6b09 *./data/a-mpeg4-adv.avi
 1d6c3ba2b096ac6df63f1dcabe3ed603 *./data/out.yuv
 2846c8e3d97d7395eb746bfce44e0443 *./data/a-mjpeg.avi
 278033451d7a6bfeb8339abbe4228499 *./data/out.yuv
-ccbf683d781fa3cdfa18b618731fc74b *./data/a-rv10.rm
-c1f6c8ee7a24d8345deddf1a24ca3756 *./data/out.yuv
+bbb398b23e7e69bebf584e6697e3360c *./data/a-rv10.rm
+0059bed3addc017d9f089f3256769a51 *./data/out.yuv
 21f8ff9f1daacd9133683bb4ea0f50a4 *./data/a-mp2.mp2
 116d1290ba1b4eb98fdee52e423417b1 *./data/out.wav
 048b9c3444c788bac6ce5cc3a8f4db00 *./data/a-ac3.rm