lavfi/mp/mcdeint: avoid uninited data read

Do not read padding or out-of-buffer values when computing the output
value for a pixel close to the image buffer edge.

This avoids non visible artifacts which affected the output checksum.

See thread:
Subject: [FFmpeg-devel] [PATCH] lavfi/mp/mcdeint: avoid uninited data read
Date: Thu, 30 May 2013 18:57:14 +0200

1 files changed, 33 insertions, 11 deletions

diff --git a/libavfilter/libmpcodecs/vf_mcdeint.c b/libavfilter/libmpcodecs/vf_mcdeint.c
index b9ffaf21ca..0bb4f0a4e9 100644
--- a/libavfilter/libmpcodecs/vf_mcdeint.c
+++ b/libavfilter/libmpcodecs/vf_mcdeint.c
@@ -115,27 +115,49 @@ static void filter(struct vf_priv_s *p, uint8_t *dst[3], uint8_t *src[3], int ds
         for(y=0; y<h; y++){
             if((y ^ p->parity) & 1){
                 for(x=0; x<w; x++){
-                    if((x-2)+(y-1)*w>=0 && (x+2)+(y+1)*w<w*h){ //FIXME either alloc larger images or optimize this
+                    if(y>0 && y<h-1){
+                        int is_edge= x<3 || x>w-4;
                         uint8_t *filp= &p->frame_dec->data[i][x + y*fils];
                         uint8_t *srcp= &src[i][x + y*srcs];
                         int diff0= filp[-fils] - srcp[-srcs];
                         int diff1= filp[+fils] - srcp[+srcs];
-                        int spatial_score= ABS(srcp[-srcs-1] - srcp[+srcs-1])
-                                          +ABS(srcp[-srcs  ] - srcp[+srcs  ])
-                                          +ABS(srcp[-srcs+1] - srcp[+srcs+1]) - 1;
                         int temp= filp[0];
 
+#define DELTA(j) av_clip(j, -x, w-1-x)
+
+#define GET_SCORE_EDGE(j)\
+   ABS(srcp[-srcs+DELTA(-1+(j))] - srcp[+srcs+DELTA(-1-(j))])+\
+   ABS(srcp[-srcs+DELTA(j)     ] - srcp[+srcs+DELTA(  -(j))])+\
+   ABS(srcp[-srcs+DELTA(1+(j)) ] - srcp[+srcs+DELTA( 1-(j))])
+
+#define GET_SCORE(j)\
+   ABS(srcp[-srcs-1+(j)] - srcp[+srcs-1-(j)])+\
+   ABS(srcp[-srcs  +(j)] - srcp[+srcs  -(j)])+\
+   ABS(srcp[-srcs+1+(j)] - srcp[+srcs+1-(j)])
+
+#define CHECK_EDGE(j)\
+    {   int score= GET_SCORE_EDGE(j);\
+        if(score < spatial_score){\
+            spatial_score= score;\
+            diff0= filp[-fils+DELTA(j)]    - srcp[-srcs+DELTA(j)];\
+            diff1= filp[+fils+DELTA(-(j))] - srcp[+srcs+DELTA(-(j))];\
+
 #define CHECK(j)\
-    {   int score= ABS(srcp[-srcs-1+(j)] - srcp[+srcs-1-(j)])\
-                 + ABS(srcp[-srcs  +(j)] - srcp[+srcs  -(j)])\
-                 + ABS(srcp[-srcs+1+(j)] - srcp[+srcs+1-(j)]);\
+    {   int score= GET_SCORE(j);\
         if(score < spatial_score){\
             spatial_score= score;\
             diff0= filp[-fils+(j)] - srcp[-srcs+(j)];\
-            diff1= filp[+fils-(j)] - srcp[+srcs-(j)];
-
-                        CHECK(-1) CHECK(-2) }} }}
-                        CHECK( 1) CHECK( 2) }} }}
+            diff1= filp[+fils-(j)] - srcp[+srcs-(j)];\
+
+                        if (is_edge) {
+                            int spatial_score= GET_SCORE_EDGE(0)-1;
+                            CHECK_EDGE(-1) CHECK_EDGE(-2) }} }}
+                            CHECK_EDGE( 1) CHECK_EDGE( 2) }} }}
+                        } else {
+                            int spatial_score= GET_SCORE(0)-1;
+                            CHECK(-1) CHECK(-2) }} }}
+                            CHECK( 1) CHECK( 2) }} }}
+                        }
 #if 0
                         if((diff0 ^ diff1) > 0){
                             int mindiff= ABS(diff0) > ABS(diff1) ? diff1 : diff0;