vf_yadif: fix out-of line reads

Some changes in the border pixels, visually indistinguishable.

5 files changed, 200 insertions, 112 deletions

diff --git a/libavfilter/vf_yadif.c b/libavfilter/vf_yadif.c
index 280854093e..d1670902bb 100644
--- a/libavfilter/vf_yadif.c
+++ b/libavfilter/vf_yadif.c
@@ -34,15 +34,15 @@
 #define PERM_RWP AV_PERM_WRITE | AV_PERM_PRESERVE | AV_PERM_REUSE
 
 #define CHECK(j)\
-    {   int score = FFABS(cur[mrefs-1+(j)] - cur[prefs-1-(j)])\
+    {   int score = FFABS(cur[mrefs + off_left + (j)] - cur[prefs + off_left - (j)])\
                   + FFABS(cur[mrefs  +(j)] - cur[prefs  -(j)])\
-                  + FFABS(cur[mrefs+1+(j)] - cur[prefs+1-(j)]);\
+                  + FFABS(cur[mrefs + off_right + (j)] - cur[prefs + off_right - (j)]);\
         if (score < spatial_score) {\
             spatial_score= score;\
             spatial_pred= (cur[mrefs  +(j)] + cur[prefs  -(j)])>>1;\
 
-#define FILTER \
-    for (x = 0;  x < w; x++) { \
+#define FILTER(start, end) \
+    for (x = start;  x < end; x++) { \
         int c = cur[mrefs]; \
         int d = (prev2[0] + next2[0])>>1; \
         int e = cur[prefs]; \
@@ -51,11 +51,15 @@
         int temporal_diff2 =(FFABS(next[mrefs] - c) + FFABS(next[prefs] - e) )>>1; \
         int diff = FFMAX3(temporal_diff0 >> 1, temporal_diff1, temporal_diff2); \
         int spatial_pred = (c+e) >> 1; \
-        int spatial_score = FFABS(cur[mrefs - 1] - cur[prefs - 1]) + FFABS(c-e) \
-                          + FFABS(cur[mrefs + 1] - cur[prefs + 1]) - 1; \
+        int off_right = (x < w - 1) ? 1 : -1;\
+        int off_left  = x ? -1 : 1;\
+        int spatial_score = FFABS(cur[mrefs + off_left]  - cur[prefs + off_left]) + FFABS(c-e) \
+                          + FFABS(cur[mrefs + off_right] - cur[prefs + off_right]) - 1; \
  \
-        CHECK(-1) CHECK(-2) }} }} \
-        CHECK( 1) CHECK( 2) }} }} \
+        if (x > 2 && x < w - 3) {\
+            CHECK(-1) CHECK(-2) }} }} \
+            CHECK( 1) CHECK( 2) }} }} \
+        }\
  \
         if (mode < 2) { \
             int b = (prev2[2 * mrefs] + next2[2 * mrefs])>>1; \
@@ -93,9 +97,34 @@ static void filter_line_c(void *dst1,
     uint8_t *prev2 = parity ? prev : cur ;
     uint8_t *next2 = parity ? cur  : next;
 
-    FILTER
+    FILTER(0, w)
 }
 
+static void filter_edges(void *dst1, void *prev1, void *cur1, void *next1,
+                         int w, int prefs, int mrefs, int parity, int mode,
+                         int l_edge)
+{
+    uint8_t *dst  = dst1;
+    uint8_t *prev = prev1;
+    uint8_t *cur  = cur1;
+    uint8_t *next = next1;
+    int x;
+    uint8_t *prev2 = parity ? prev : cur ;
+    uint8_t *next2 = parity ? cur  : next;
+
+    FILTER(0, l_edge)
+
+    dst  = (uint8_t*)dst1  + w - 3;
+    prev = (uint8_t*)prev1 + w - 3;
+    cur  = (uint8_t*)cur1  + w - 3;
+    next = (uint8_t*)next1 + w - 3;
+    prev2 = (uint8_t*)(parity ? prev : cur);
+    next2 = (uint8_t*)(parity ? cur  : next);
+
+    FILTER(w - 3, w)
+}
+
+
 static void filter_line_c_16bit(void *dst1,
                                 void *prev1, void *cur1, void *next1,
                                 int w, int prefs, int mrefs, int parity,
@@ -111,7 +140,31 @@ static void filter_line_c_16bit(void *dst1,
     mrefs /= 2;
     prefs /= 2;
 
-    FILTER
+    FILTER(0, w)
+}
+
+static void filter_edges_16bit(void *dst1, void *prev1, void *cur1, void *next1,
+                               int w, int prefs, int mrefs, int parity, int mode,
+                               int l_edge)
+{
+    uint16_t *dst  = dst1;
+    uint16_t *prev = prev1;
+    uint16_t *cur  = cur1;
+    uint16_t *next = next1;
+    int x;
+    uint16_t *prev2 = parity ? prev : cur ;
+    uint16_t *next2 = parity ? cur  : next;
+
+    FILTER(0, l_edge)
+
+    dst   = (uint16_t*)dst1  + w - 3;
+    prev  = (uint16_t*)prev1 + w - 3;
+    cur   = (uint16_t*)cur1  + w - 3;
+    next  = (uint16_t*)next1 + w - 3;
+    prev2 = (uint16_t*)(parity ? prev : cur);
+    next2 = (uint16_t*)(parity ? cur  : next);
+
+    FILTER(w - 3, w)
 }
 
 static void filter(AVFilterContext *ctx, AVFilterBufferRef *dstpic,
@@ -125,6 +178,7 @@ static void filter(AVFilterContext *ctx, AVFilterBufferRef *dstpic,
         int h = dstpic->video->h;
         int refs = yadif->cur->linesize[i];
         int df = (yadif->csp->comp[i].depth_minus1 + 8) / 8;
+        int l_edge, l_edge_pix;
 
         if (i == 1 || i == 2) {
         /* Why is this not part of the per-plane description thing? */
@@ -132,6 +186,12 @@ static void filter(AVFilterContext *ctx, AVFilterBufferRef *dstpic,
             h >>= yadif->csp->log2_chroma_h;
         }
 
+        /* filtering reads 3 pixels to the left/right; to avoid invalid reads,
+         * we need to call the c variant which avoids this for border pixels
+         */
+        l_edge     = yadif->req_align;
+        l_edge_pix = l_edge / df;
+
         for (y = 0; y < h; y++) {
             if ((y ^ parity) & 1) {
                 uint8_t *prev = &yadif->prev->data[i][y * refs];
@@ -139,10 +199,22 @@ static void filter(AVFilterContext *ctx, AVFilterBufferRef *dstpic,
                 uint8_t *next = &yadif->next->data[i][y * refs];
                 uint8_t *dst  = &dstpic->data[i][y * dstpic->linesize[i]];
                 int     mode  = y == 1 || y + 2 == h ? 2 : yadif->mode;
-                yadif->filter_line(dst, prev, cur, next, w,
-                                   y + 1 < h ? refs : -refs,
-                                   y ? -refs : refs,
-                                   parity ^ tff, mode);
+                if (yadif->req_align) {
+                    yadif->filter_line(dst + l_edge, prev + l_edge, cur + l_edge,
+                                       next + l_edge, w - l_edge_pix - 3,
+                                       y + 1 < h ? refs : -refs,
+                                       y ? -refs : refs,
+                                       parity ^ tff, mode);
+                    yadif->filter_edges(dst, prev, cur, next, w,
+                                         y + 1 < h ? refs : -refs,
+                                         y ? -refs : refs,
+                                         parity ^ tff, mode, l_edge_pix);
+                } else {
+                    yadif->filter_line(dst, prev, cur, next + l_edge, w,
+                                       y + 1 < h ? refs : -refs,
+                                       y ? -refs : refs,
+                                       parity ^ tff, mode);
+                }
             } else {
                 memcpy(&dstpic->data[i][y * dstpic->linesize[i]],
                        &yadif->cur->data[i][y * refs], w * df);
@@ -391,9 +463,11 @@ static int config_props(AVFilterLink *link)
 
     s->csp = av_pix_fmt_desc_get(link->format);
     if (s->csp->comp[0].depth_minus1 / 8 == 1) {
-        s->filter_line = filter_line_c_16bit;
+        s->filter_line  = filter_line_c_16bit;
+        s->filter_edges = filter_edges_16bit;
     } else {
-        s->filter_line = filter_line_c;
+        s->filter_line  = filter_line_c;
+        s->filter_edges = filter_edges;
 
         if (ARCH_X86)
             ff_yadif_init_x86(s);
diff --git a/libavfilter/x86/vf_yadif_init.c b/libavfilter/x86/vf_yadif_init.c
index 2ffeca0f76..b330c73c59 100644
--- a/libavfilter/x86/vf_yadif_init.c
+++ b/libavfilter/x86/vf_yadif_init.c
@@ -43,12 +43,18 @@ av_cold void ff_yadif_init_x86(YADIFContext *yadif)
 
 #if HAVE_YASM
 #if ARCH_X86_32
-    if (EXTERNAL_MMXEXT(cpu_flags))
+    if (EXTERNAL_MMXEXT(cpu_flags)) {
         yadif->filter_line = ff_yadif_filter_line_mmxext;
+        yadif->req_align   = 8;
+    }
 #endif /* ARCH_X86_32 */
-    if (EXTERNAL_SSE2(cpu_flags))
+    if (EXTERNAL_SSE2(cpu_flags)) {
         yadif->filter_line = ff_yadif_filter_line_sse2;
-    if (EXTERNAL_SSSE3(cpu_flags))
+        yadif->req_align   = 16;
+    }
+    if (EXTERNAL_SSSE3(cpu_flags)) {
         yadif->filter_line = ff_yadif_filter_line_ssse3;
+        yadif->req_align   = 16;
+    }
 #endif /* HAVE_YASM */
 }
diff --git a/libavfilter/yadif.h b/libavfilter/yadif.h
index e6f713b26c..18c8624f1f 100644
--- a/libavfilter/yadif.h
+++ b/libavfilter/yadif.h
@@ -50,9 +50,17 @@ typedef struct YADIFContext {
     AVFilterBufferRef *next;
     AVFilterBufferRef *prev;
     AVFilterBufferRef *out;
+
+    /**
+     * Required alignment for filter_line
+     */
+    int req_align;
     void (*filter_line)(void *dst,
                         void *prev, void *cur, void *next,
                         int w, int prefs, int mrefs, int parity, int mode);
+    void (*filter_edges)(void *dst, void *prev, void *cur, void *next,
+                         int w, int prefs, int mrefs, int parity, int mode,
+                         int l_edge);
 
     const AVPixFmtDescriptor *csp;
     int eof;
diff --git a/tests/ref/fate/filter-yadif-mode0 b/tests/ref/fate/filter-yadif-mode0
index e260977141..d2c14d39b8 100644
--- a/tests/ref/fate/filter-yadif-mode0
+++ b/tests/ref/fate/filter-yadif-mode0
@@ -1,32 +1,32 @@
 #tb 0: 1/180000
-0,      64800,      64800,        0,   622080, 0x4440caef
-0,      72000,      72000,        0,   622080, 0xce67e69d
-0,      79200,      79200,        0,   622080, 0x1dbdc653
-0,      86400,      86400,        0,   622080, 0x82c591d1
-0,      93600,      93600,        0,   622080, 0x8193740b
-0,     100800,     100800,        0,   622080, 0xcb219711
-0,     108000,     108000,        0,   622080, 0x1870783b
-0,     115200,     115200,        0,   622080, 0x7080590b
-0,     122400,     122400,        0,   622080, 0x6df4175d
-0,     129600,     129600,        0,   622080, 0x6b530e95
-0,     136800,     136800,        0,   622080, 0x7f9d66f7
-0,     144000,     144000,        0,   622080, 0x338cda81
-0,     151200,     151200,        0,   622080, 0xb13797f8
-0,     158400,     158400,        0,   622080, 0xb51e7ca4
-0,     165600,     165600,        0,   622080, 0x353eed75
-0,     172800,     172800,        0,   622080, 0xf93e92b0
-0,     180000,     180000,        0,   622080, 0xd0811094
-0,     187200,     187200,        0,   622080, 0xb04a3141
-0,     194400,     194400,        0,   622080, 0x4ab84909
-0,     201600,     201600,        0,   622080, 0xa0fcb8fb
-0,     208800,     208800,        0,   622080, 0x9003aebb
-0,     216000,     216000,        0,   622080, 0x153faa3e
-0,     223200,     223200,        0,   622080, 0xae724063
-0,     230400,     230400,        0,   622080, 0xeb4de77a
-0,     237600,     237600,        0,   622080, 0x209ed8c7
-0,     244800,     244800,        0,   622080, 0xe2bbac96
-0,     252000,     252000,        0,   622080, 0xe945441e
-0,     259200,     259200,        0,   622080, 0x8f8cbd5f
-0,     266400,     266400,        0,   622080, 0xbc3cf717
-0,     273600,     273600,        0,   622080, 0x0109f125
-0,     280800,     280800,        0,   622080, 0x230c373f
+0,      64800,      64800,        0,   622080, 0x6331caee
+0,      72000,      72000,        0,   622080, 0xa459e690
+0,      79200,      79200,        0,   622080, 0x6429c648
+0,      86400,      86400,        0,   622080, 0xa49891ca
+0,      93600,      93600,        0,   622080, 0x2a887404
+0,     100800,     100800,        0,   622080, 0xe8d49705
+0,     108000,     108000,        0,   622080, 0x1b627835
+0,     115200,     115200,        0,   622080, 0x686858fd
+0,     122400,     122400,        0,   622080, 0x2675174f
+0,     129600,     129600,        0,   622080, 0x78470e7f
+0,     136800,     136800,        0,   622080, 0xffb366ec
+0,     144000,     144000,        0,   622080, 0xd575da72
+0,     151200,     151200,        0,   622080, 0x5fb297f7
+0,     158400,     158400,        0,   622080, 0xbac77ca0
+0,     165600,     165600,        0,   622080, 0x3276ed72
+0,     172800,     172800,        0,   622080, 0x264092b2
+0,     180000,     180000,        0,   622080, 0x20ba1094
+0,     187200,     187200,        0,   622080, 0x76cc3139
+0,     194400,     194400,        0,   622080, 0x469a4902
+0,     201600,     201600,        0,   622080, 0x0ed7b8f5
+0,     208800,     208800,        0,   622080, 0xdc51aeac
+0,     216000,     216000,        0,   622080, 0xee06aa36
+0,     223200,     223200,        0,   622080, 0x7372405f
+0,     230400,     230400,        0,   622080, 0x9e0ee776
+0,     237600,     237600,        0,   622080, 0x39e6d8c9
+0,     244800,     244800,        0,   622080, 0x51d9ac9a
+0,     252000,     252000,        0,   622080, 0x2b63441d
+0,     259200,     259200,        0,   622080, 0x58afbd5e
+0,     266400,     266400,        0,   622080, 0xb972f716
+0,     273600,     273600,        0,   622080, 0x6a6df129
+0,     280800,     280800,        0,   622080, 0x28b1373d
diff --git a/tests/ref/fate/filter-yadif-mode1 b/tests/ref/fate/filter-yadif-mode1
index 8917019ea0..ca5f290ee9 100644
--- a/tests/ref/fate/filter-yadif-mode1
+++ b/tests/ref/fate/filter-yadif-mode1
@@ -1,63 +1,63 @@
 #tb 0: 1/180000
-0,      64800,      64800,        0,   622080, 0x4440caef
-0,      68400,      68400,        0,   622080, 0xa5cea88b
-0,      72000,      72000,        0,   622080, 0xce67e69d
-0,      75600,      75600,        0,   622080, 0x9a57891f
-0,      79200,      79200,        0,   622080, 0x1dbdc653
-0,      82800,      82800,        0,   622080, 0xc171c0c5
-0,      86400,      86400,        0,   622080, 0x82c591d1
-0,      90000,      90000,        0,   622080, 0x20db9890
-0,      93600,      93600,        0,   622080, 0x8193740b
-0,      97200,      97200,        0,   622080, 0xdb181d52
-0,     100800,     100800,        0,   622080, 0xcb219711
-0,     104400,     104400,        0,   622080, 0xc2b913d1
-0,     108000,     108000,        0,   622080, 0x1870783b
-0,     111600,     111600,        0,   622080, 0xf1d9c5fb
-0,     115200,     115200,        0,   622080, 0x7080590b
-0,     118800,     118800,        0,   622080, 0x669c5775
-0,     122400,     122400,        0,   622080, 0x6df4175d
-0,     126000,     126000,        0,   622080, 0x01921a16
-0,     129600,     129600,        0,   622080, 0x6b530e95
-0,     133200,     133200,        0,   622080, 0xd5047bc9
-0,     136800,     136800,        0,   622080, 0x7f9d66f7
-0,     140400,     140400,        0,   622080, 0xa8b006eb
-0,     144000,     144000,        0,   622080, 0x338cda81
-0,     147600,     147600,        0,   622080, 0xf0e125a7
-0,     151200,     151200,        0,   622080, 0xb13797f8
-0,     154800,     154800,        0,   622080, 0x4afe2976
-0,     158400,     158400,        0,   622080, 0xb51e7ca4
-0,     162000,     162000,        0,   622080, 0x637fcbfe
-0,     165600,     165600,        0,   622080, 0x353eed75
-0,     169200,     169200,        0,   622080, 0xd9a8f5ac
-0,     172800,     172800,        0,   622080, 0xf93e92b0
-0,     176400,     176400,        0,   622080, 0x4540039f
-0,     180000,     180000,        0,   622080, 0xd0811094
-0,     183600,     183600,        0,   622080, 0x3039906f
-0,     187200,     187200,        0,   622080, 0xb04a3141
-0,     190800,     190800,        0,   622080, 0x638d2cf5
-0,     194400,     194400,        0,   622080, 0x4ab84909
-0,     198000,     198000,        0,   622080, 0x82de12ee
-0,     201600,     201600,        0,   622080, 0xa0fcb8fb
-0,     205200,     205200,        0,   622080, 0x7e849cc9
-0,     208800,     208800,        0,   622080, 0x9003aebb
-0,     212400,     212400,        0,   622080, 0xffe6f770
-0,     216000,     216000,        0,   622080, 0x153faa3e
-0,     219600,     219600,        0,   622080, 0xbf023231
-0,     223200,     223200,        0,   622080, 0xae724063
-0,     226800,     226800,        0,   622080, 0x15fe44b4
-0,     230400,     230400,        0,   622080, 0xeb4de77a
-0,     234000,     234000,        0,   622080, 0x380f8563
-0,     237600,     237600,        0,   622080, 0x209ed8c7
-0,     241200,     241200,        0,   622080, 0xb964d70f
-0,     244800,     244800,        0,   622080, 0xe2bbac96
-0,     248400,     248400,        0,   622080, 0x57e3f7f2
-0,     252000,     252000,        0,   622080, 0xe945441e
-0,     255600,     255600,        0,   622080, 0xd0afb742
-0,     259200,     259200,        0,   622080, 0x8f8cbd5f
-0,     262800,     262800,        0,   622080, 0xb9a15294
-0,     266400,     266400,        0,   622080, 0xbc3cf717
-0,     270000,     270000,        0,   622080, 0xb70b01a9
-0,     273600,     273600,        0,   622080, 0x0109f125
-0,     277200,     277200,        0,   622080, 0x5806371c
-0,     280800,     280800,        0,   622080, 0x230c373f
-0,     284400,     284400,        0,   622080, 0x82dfb1f2
+0,      64800,      64800,        0,   622080, 0x6331caee
+0,      68400,      68400,        0,   622080, 0x625da883
+0,      72000,      72000,        0,   622080, 0xa459e690
+0,      75600,      75600,        0,   622080, 0xce5d891e
+0,      79200,      79200,        0,   622080, 0x6429c648
+0,      82800,      82800,        0,   622080, 0x608cc0ba
+0,      86400,      86400,        0,   622080, 0xa49891ca
+0,      90000,      90000,        0,   622080, 0x9721987f
+0,      93600,      93600,        0,   622080, 0x2a887404
+0,      97200,      97200,        0,   622080, 0x60d71d47
+0,     100800,     100800,        0,   622080, 0xe8d49705
+0,     104400,     104400,        0,   622080, 0x821e13cb
+0,     108000,     108000,        0,   622080, 0x1b627835
+0,     111600,     111600,        0,   622080, 0x1806c5f4
+0,     115200,     115200,        0,   622080, 0x686858fd
+0,     118800,     118800,        0,   622080, 0xab865773
+0,     122400,     122400,        0,   622080, 0x2675174f
+0,     126000,     126000,        0,   622080, 0x43a61a14
+0,     129600,     129600,        0,   622080, 0x78470e7f
+0,     133200,     133200,        0,   622080, 0xeb877bc6
+0,     136800,     136800,        0,   622080, 0xffb366ec
+0,     140400,     140400,        0,   622080, 0xda0906e7
+0,     144000,     144000,        0,   622080, 0xd575da72
+0,     147600,     147600,        0,   622080, 0x23ae25a4
+0,     151200,     151200,        0,   622080, 0x5fb297f7
+0,     154800,     154800,        0,   622080, 0x99b32978
+0,     158400,     158400,        0,   622080, 0xbac77ca0
+0,     162000,     162000,        0,   622080, 0xc1cdcbf9
+0,     165600,     165600,        0,   622080, 0x3276ed72
+0,     169200,     169200,        0,   622080, 0x4061f5ab
+0,     172800,     172800,        0,   622080, 0x264092b2
+0,     176400,     176400,        0,   622080, 0xa4e2039e
+0,     180000,     180000,        0,   622080, 0x20ba1094
+0,     183600,     183600,        0,   622080, 0x984e906e
+0,     187200,     187200,        0,   622080, 0x76cc3139
+0,     190800,     190800,        0,   622080, 0xf70e2cf6
+0,     194400,     194400,        0,   622080, 0x469a4902
+0,     198000,     198000,        0,   622080, 0x235312e6
+0,     201600,     201600,        0,   622080, 0x0ed7b8f5
+0,     205200,     205200,        0,   622080, 0xd0269cc3
+0,     208800,     208800,        0,   622080, 0xdc51aeac
+0,     212400,     212400,        0,   622080, 0x1aa5f76e
+0,     216000,     216000,        0,   622080, 0xee06aa36
+0,     219600,     219600,        0,   622080, 0xa7103230
+0,     223200,     223200,        0,   622080, 0x7372405f
+0,     226800,     226800,        0,   622080, 0x8d7a44b5
+0,     230400,     230400,        0,   622080, 0x9e0ee776
+0,     234000,     234000,        0,   622080, 0xd41e8560
+0,     237600,     237600,        0,   622080, 0x39e6d8c9
+0,     241200,     241200,        0,   622080, 0x7a23d70c
+0,     244800,     244800,        0,   622080, 0x51d9ac9a
+0,     248400,     248400,        0,   622080, 0x8eacf7f2
+0,     252000,     252000,        0,   622080, 0x2b63441d
+0,     255600,     255600,        0,   622080, 0x9f71b742
+0,     259200,     259200,        0,   622080, 0x58afbd5e
+0,     262800,     262800,        0,   622080, 0x4d645292
+0,     266400,     266400,        0,   622080, 0xb972f716
+0,     270000,     270000,        0,   622080, 0xbb5d01a2
+0,     273600,     273600,        0,   622080, 0x6a6df129
+0,     277200,     277200,        0,   622080, 0x9e45371e
+0,     280800,     280800,        0,   622080, 0x28b1373d
+0,     284400,     284400,        0,   622080, 0xa1cdb1f2