Merge commit '64ed397635ef2666b0ca0c8d8c60a8bc44581d82'

* commit '64ed397635ef2666b0ca0c8d8c60a8bc44581d82':
  vf_yadif: fix out-of line reads

Conflicts:
	libavfilter/vf_yadif.c
	tests/ref/fate/filter-yadif-mode0
	tests/ref/fate/filter-yadif-mode1

Merged-by: Michael Niedermayer <michaelni@gmx.at>

5 files changed, 198 insertions, 133 deletions

diff --git a/libavfilter/vf_yadif.c b/libavfilter/vf_yadif.c
index 9ac1f3e548..fb17531dbf 100644
--- a/libavfilter/vf_yadif.c
+++ b/libavfilter/vf_yadif.c
@@ -34,15 +34,15 @@
 #define PERM_RWP AV_PERM_WRITE | AV_PERM_PRESERVE | AV_PERM_REUSE
 
 #define CHECK(j)\
-    {   int score = FFABS(cur[mrefs-1+(j)] - cur[prefs-1-(j)])\
+    {   int score = FFABS(cur[mrefs + off_left + (j)] - cur[prefs + off_left - (j)])\
                   + FFABS(cur[mrefs  +(j)] - cur[prefs  -(j)])\
-                  + FFABS(cur[mrefs+1+(j)] - cur[prefs+1-(j)]);\
+                  + FFABS(cur[mrefs + off_right + (j)] - cur[prefs + off_right - (j)]);\
         if (score < spatial_score) {\
             spatial_score= score;\
             spatial_pred= (cur[mrefs  +(j)] + cur[prefs  -(j)])>>1;\
 
-#define FILTER \
-    for (x = 0;  x < w; x++) { \
+#define FILTER(start, end) \
+    for (x = start;  x < end; x++) { \
         int c = cur[mrefs]; \
         int d = (prev2[0] + next2[0])>>1; \
         int e = cur[prefs]; \
@@ -51,11 +51,15 @@
         int temporal_diff2 =(FFABS(next[mrefs] - c) + FFABS(next[prefs] - e) )>>1; \
         int diff = FFMAX3(temporal_diff0 >> 1, temporal_diff1, temporal_diff2); \
         int spatial_pred = (c+e) >> 1; \
-        int spatial_score = FFABS(cur[mrefs - 1] - cur[prefs - 1]) + FFABS(c-e) \
-                          + FFABS(cur[mrefs + 1] - cur[prefs + 1]) - 1; \
+        int off_right = (x < w - 1) ? 1 : -1;\
+        int off_left  = x ? -1 : 1;\
+        int spatial_score = FFABS(cur[mrefs + off_left]  - cur[prefs + off_left]) + FFABS(c-e) \
+                          + FFABS(cur[mrefs + off_right] - cur[prefs + off_right]) - 1; \
  \
-        CHECK(-1) CHECK(-2) }} }} \
-        CHECK( 1) CHECK( 2) }} }} \
+        if (x > 2 && x < w - 3) {\
+            CHECK(-1) CHECK(-2) }} }} \
+            CHECK( 1) CHECK( 2) }} }} \
+        }\
  \
         if (mode < 2) { \
             int b = (prev2[2 * mrefs] + next2[2 * mrefs])>>1; \
@@ -93,9 +97,34 @@ static void filter_line_c(void *dst1,
     uint8_t *prev2 = parity ? prev : cur ;
     uint8_t *next2 = parity ? cur  : next;
 
-    FILTER
+    FILTER(0, w)
 }
 
+static void filter_edges(void *dst1, void *prev1, void *cur1, void *next1,
+                         int w, int prefs, int mrefs, int parity, int mode,
+                         int l_edge)
+{
+    uint8_t *dst  = dst1;
+    uint8_t *prev = prev1;
+    uint8_t *cur  = cur1;
+    uint8_t *next = next1;
+    int x;
+    uint8_t *prev2 = parity ? prev : cur ;
+    uint8_t *next2 = parity ? cur  : next;
+
+    FILTER(0, l_edge)
+
+    dst  = (uint8_t*)dst1  + w - 3;
+    prev = (uint8_t*)prev1 + w - 3;
+    cur  = (uint8_t*)cur1  + w - 3;
+    next = (uint8_t*)next1 + w - 3;
+    prev2 = (uint8_t*)(parity ? prev : cur);
+    next2 = (uint8_t*)(parity ? cur  : next);
+
+    FILTER(w - 3, w)
+}
+
+
 static void filter_line_c_16bit(void *dst1,
                                 void *prev1, void *cur1, void *next1,
                                 int w, int prefs, int mrefs, int parity,
@@ -111,7 +140,31 @@ static void filter_line_c_16bit(void *dst1,
     mrefs /= 2;
     prefs /= 2;
 
-    FILTER
+    FILTER(0, w)
+}
+
+static void filter_edges_16bit(void *dst1, void *prev1, void *cur1, void *next1,
+                               int w, int prefs, int mrefs, int parity, int mode,
+                               int l_edge)
+{
+    uint16_t *dst  = dst1;
+    uint16_t *prev = prev1;
+    uint16_t *cur  = cur1;
+    uint16_t *next = next1;
+    int x;
+    uint16_t *prev2 = parity ? prev : cur ;
+    uint16_t *next2 = parity ? cur  : next;
+
+    FILTER(0, l_edge)
+
+    dst   = (uint16_t*)dst1  + w - 3;
+    prev  = (uint16_t*)prev1 + w - 3;
+    cur   = (uint16_t*)cur1  + w - 3;
+    next  = (uint16_t*)next1 + w - 3;
+    prev2 = (uint16_t*)(parity ? prev : cur);
+    next2 = (uint16_t*)(parity ? cur  : next);
+
+    FILTER(w - 3, w)
 }
 
 static void filter(AVFilterContext *ctx, AVFilterBufferRef *dstpic,
@@ -126,6 +179,7 @@ static void filter(AVFilterContext *ctx, AVFilterBufferRef *dstpic,
         int refs = yadif->cur->linesize[i];
         int absrefs = FFABS(refs);
         int df = (yadif->csp->comp[i].depth_minus1 + 8) / 8;
+        int l_edge, l_edge_pix;
 
         if (i == 1 || i == 2) {
         /* Why is this not part of the per-plane description thing? */
@@ -133,11 +187,11 @@ static void filter(AVFilterContext *ctx, AVFilterBufferRef *dstpic,
             h >>= yadif->csp->log2_chroma_h;
         }
 
-        if(yadif->temp_line_size < absrefs) {
-            av_free(yadif->temp_line);
-            yadif->temp_line = av_mallocz(2*64 + 5*absrefs);
-            yadif->temp_line_size = absrefs;
-        }
+        /* filtering reads 3 pixels to the left/right; to avoid invalid reads,
+         * we need to call the c variant which avoids this for border pixels
+         */
+        l_edge     = yadif->req_align;
+        l_edge_pix = l_edge / df;
 
         for (y = 0; y < h; y++) {
             if ((y ^ parity) & 1) {
@@ -146,26 +200,22 @@ static void filter(AVFilterContext *ctx, AVFilterBufferRef *dstpic,
                 uint8_t *next = &yadif->next->data[i][y * refs];
                 uint8_t *dst  = &dstpic->data[i][y * dstpic->linesize[i]];
                 int     mode  = y == 1 || y + 2 == h ? 2 : yadif->mode;
-                int     prefs = y+1<h ? refs : -refs;
-                int     mrefs =     y ?-refs :  refs;
-
-                if(y<=1 || y+2>=h) {
-                    uint8_t *tmp = yadif->temp_line + 64 + 2*absrefs;
-                    if(mode<2)
-                        memcpy(tmp+2*mrefs, cur+2*mrefs, w*df);
-                    memcpy(tmp+mrefs, cur+mrefs, w*df);
-                    memcpy(tmp      , cur      , w*df);
-                    if(prefs != mrefs) {
-                        memcpy(tmp+prefs, cur+prefs, w*df);
-                        if(mode<2)
-                            memcpy(tmp+2*prefs, cur+2*prefs, w*df);
-                    }
-                    cur = tmp;
+                if (yadif->req_align) {
+                    yadif->filter_line(dst + l_edge, prev + l_edge, cur + l_edge,
+                                       next + l_edge, w - l_edge_pix - 3,
+                                       y + 1 < h ? refs : -refs,
+                                       y ? -refs : refs,
+                                       parity ^ tff, mode);
+                    yadif->filter_edges(dst, prev, cur, next, w,
+                                         y + 1 < h ? refs : -refs,
+                                         y ? -refs : refs,
+                                         parity ^ tff, mode, l_edge_pix);
+                } else {
+                    yadif->filter_line(dst, prev, cur, next + l_edge, w,
+                                       y + 1 < h ? refs : -refs,
+                                       y ? -refs : refs,
+                                       parity ^ tff, mode);
                 }
-
-                yadif->filter_line(dst, prev, cur, next, w,
-                                   prefs, mrefs,
-                                   parity ^ tff, mode);
             } else {
                 memcpy(&dstpic->data[i][y * dstpic->linesize[i]],
                        &yadif->cur->data[i][y * refs], w * df);
@@ -333,7 +383,6 @@ static av_cold void uninit(AVFilterContext *ctx)
     avfilter_unref_bufferp(&yadif->prev);
     avfilter_unref_bufferp(&yadif->cur );
     avfilter_unref_bufferp(&yadif->next);
-    av_freep(&yadif->temp_line); yadif->temp_line_size = 0;
     av_opt_free(yadif);
 }
 
@@ -416,9 +465,11 @@ static int config_props(AVFilterLink *link)
 
     s->csp = av_pix_fmt_desc_get(link->format);
     if (s->csp->comp[0].depth_minus1 / 8 == 1) {
-        s->filter_line = filter_line_c_16bit;
+        s->filter_line  = filter_line_c_16bit;
+        s->filter_edges = filter_edges_16bit;
     } else {
-        s->filter_line = filter_line_c;
+        s->filter_line  = filter_line_c;
+        s->filter_edges = filter_edges;
 
         if (ARCH_X86)
             ff_yadif_init_x86(s);
diff --git a/libavfilter/x86/vf_yadif_init.c b/libavfilter/x86/vf_yadif_init.c
index 8d5e76833f..2873744d4b 100644
--- a/libavfilter/x86/vf_yadif_init.c
+++ b/libavfilter/x86/vf_yadif_init.c
@@ -42,12 +42,18 @@ av_cold void ff_yadif_init_x86(YADIFContext *yadif)
 
 #if HAVE_YASM
 #if ARCH_X86_32
-    if (EXTERNAL_MMXEXT(cpu_flags))
+    if (EXTERNAL_MMXEXT(cpu_flags)) {
         yadif->filter_line = ff_yadif_filter_line_mmxext;
+        yadif->req_align   = 8;
+    }
 #endif /* ARCH_X86_32 */
-    if (EXTERNAL_SSE2(cpu_flags))
+    if (EXTERNAL_SSE2(cpu_flags)) {
         yadif->filter_line = ff_yadif_filter_line_sse2;
-    if (EXTERNAL_SSSE3(cpu_flags))
+        yadif->req_align   = 16;
+    }
+    if (EXTERNAL_SSSE3(cpu_flags)) {
         yadif->filter_line = ff_yadif_filter_line_ssse3;
+        yadif->req_align   = 16;
+    }
 #endif /* HAVE_YASM */
 }
diff --git a/libavfilter/yadif.h b/libavfilter/yadif.h
index b7e8852660..50fc856bf2 100644
--- a/libavfilter/yadif.h
+++ b/libavfilter/yadif.h
@@ -53,9 +53,17 @@ typedef struct YADIFContext {
     AVFilterBufferRef *next;
     AVFilterBufferRef *prev;
     AVFilterBufferRef *out;
+
+    /**
+     * Required alignment for filter_line
+     */
+    int req_align;
     void (*filter_line)(void *dst,
                         void *prev, void *cur, void *next,
                         int w, int prefs, int mrefs, int parity, int mode);
+    void (*filter_edges)(void *dst, void *prev, void *cur, void *next,
+                         int w, int prefs, int mrefs, int parity, int mode,
+                         int l_edge);
 
     const AVPixFmtDescriptor *csp;
     int eof;
diff --git a/tests/ref/fate/filter-yadif-mode0 b/tests/ref/fate/filter-yadif-mode0
index 7014333c4c..162ba96ac1 100644
--- a/tests/ref/fate/filter-yadif-mode0
+++ b/tests/ref/fate/filter-yadif-mode0
@@ -1,32 +1,32 @@
 #tb 0: 1/25
-0,          9,          9,        1,   622080, 0x4440caef
-0,         10,         10,        1,   622080, 0xce67e69d
-0,         11,         11,        1,   622080, 0x1dbdc653
-0,         12,         12,        1,   622080, 0x55c791d0
-0,         13,         13,        1,   622080, 0x8193740b
-0,         14,         14,        1,   622080, 0x7125970f
-0,         15,         15,        1,   622080, 0xeb63783a
-0,         16,         16,        1,   622080, 0x7080590b
-0,         17,         17,        1,   622080, 0x13f8175b
-0,         18,         18,        1,   622080, 0x3e550e94
-0,         19,         19,        1,   622080, 0x7f9d66f7
-0,         20,         20,        1,   622080, 0x068eda80
-0,         21,         21,        1,   622080, 0x843997f7
-0,         22,         22,        1,   622080, 0x88207ca3
-0,         23,         23,        1,   622080, 0x353eed75
-0,         24,         24,        1,   622080, 0xf93e92b0
-0,         25,         25,        1,   622080, 0xd0811094
-0,         26,         26,        1,   622080, 0xb04a3141
-0,         27,         27,        1,   622080, 0x4ab84909
-0,         28,         28,        1,   622080, 0x4700b8f9
-0,         29,         29,        1,   622080, 0x6305aeba
-0,         30,         30,        1,   622080, 0x153faa3e
-0,         31,         31,        1,   622080, 0xae724063
-0,         32,         32,        1,   622080, 0xbe4fe779
-0,         33,         33,        1,   622080, 0x209ed8c7
-0,         34,         34,        1,   622080, 0xe2bbac96
-0,         35,         35,        1,   622080, 0xe945441e
-0,         36,         36,        1,   622080, 0x3590bd5d
-0,         37,         37,        1,   622080, 0x8f3ef716
-0,         38,         38,        1,   622080, 0x0109f125
-0,         39,         39,        1,   622080, 0x230c373f
+0,          9,          9,        1,   622080, 0x6331caee
+0,         10,         10,        1,   622080, 0xa459e690
+0,         11,         11,        1,   622080, 0x6429c648
+0,         12,         12,        1,   622080, 0xa49891ca
+0,         13,         13,        1,   622080, 0x2a887404
+0,         14,         14,        1,   622080, 0xe8d49705
+0,         15,         15,        1,   622080, 0x1b627835
+0,         16,         16,        1,   622080, 0x686858fd
+0,         17,         17,        1,   622080, 0x2675174f
+0,         18,         18,        1,   622080, 0x78470e7f
+0,         19,         19,        1,   622080, 0xffb366ec
+0,         20,         20,        1,   622080, 0xd575da72
+0,         21,         21,        1,   622080, 0x5fb297f7
+0,         22,         22,        1,   622080, 0xbac77ca0
+0,         23,         23,        1,   622080, 0x3276ed72
+0,         24,         24,        1,   622080, 0x264092b2
+0,         25,         25,        1,   622080, 0x20ba1094
+0,         26,         26,        1,   622080, 0x76cc3139
+0,         27,         27,        1,   622080, 0x469a4902
+0,         28,         28,        1,   622080, 0x0ed7b8f5
+0,         29,         29,        1,   622080, 0xdc51aeac
+0,         30,         30,        1,   622080, 0xee06aa36
+0,         31,         31,        1,   622080, 0x7372405f
+0,         32,         32,        1,   622080, 0x9e0ee776
+0,         33,         33,        1,   622080, 0x39e6d8c9
+0,         34,         34,        1,   622080, 0x51d9ac9a
+0,         35,         35,        1,   622080, 0x2b63441d
+0,         36,         36,        1,   622080, 0x58afbd5e
+0,         37,         37,        1,   622080, 0xb972f716
+0,         38,         38,        1,   622080, 0x6a6df129
+0,         39,         39,        1,   622080, 0x28b1373d
diff --git a/tests/ref/fate/filter-yadif-mode1 b/tests/ref/fate/filter-yadif-mode1
index 87c8d97b6d..f1f513ef3c 100644
--- a/tests/ref/fate/filter-yadif-mode1
+++ b/tests/ref/fate/filter-yadif-mode1
@@ -1,63 +1,63 @@
 #tb 0: 1/50
-0,         18,         18,        1,   622080, 0x4440caef
-0,         19,         19,        1,   622080, 0xa5cea88b
-0,         20,         20,        1,   622080, 0xce67e69d
-0,         21,         21,        1,   622080, 0x9a57891f
-0,         22,         22,        1,   622080, 0x1dbdc653
-0,         23,         23,        1,   622080, 0xc171c0c5
-0,         24,         24,        1,   622080, 0x55c791d0
-0,         25,         25,        1,   622080, 0x20db9890
-0,         26,         26,        1,   622080, 0x8193740b
-0,         27,         27,        1,   622080, 0xdb181d52
-0,         28,         28,        1,   622080, 0x7125970f
-0,         29,         29,        1,   622080, 0xc2b913d1
-0,         30,         30,        1,   622080, 0xeb63783a
-0,         31,         31,        1,   622080, 0xf1d9c5fb
-0,         32,         32,        1,   622080, 0x7080590b
-0,         33,         33,        1,   622080, 0xeda55774
-0,         34,         34,        1,   622080, 0x13f8175b
-0,         35,         35,        1,   622080, 0x01921a16
-0,         36,         36,        1,   622080, 0x3e550e94
-0,         37,         37,        1,   622080, 0xd5047bc9
-0,         38,         38,        1,   622080, 0x7f9d66f7
-0,         39,         39,        1,   622080, 0x2fc806ea
-0,         40,         40,        1,   622080, 0x068eda80
-0,         41,         41,        1,   622080, 0xf0e125a7
-0,         42,         42,        1,   622080, 0x843997f7
-0,         43,         43,        1,   622080, 0x4afe2976
-0,         44,         44,        1,   622080, 0x88207ca3
-0,         45,         45,        1,   622080, 0x637fcbfe
-0,         46,         46,        1,   622080, 0x353eed75
-0,         47,         47,        1,   622080, 0xd9a8f5ac
-0,         48,         48,        1,   622080, 0xf93e92b0
-0,         49,         49,        1,   622080, 0x4540039f
-0,         50,         50,        1,   622080, 0xd0811094
-0,         51,         51,        1,   622080, 0x3039906f
-0,         52,         52,        1,   622080, 0xb04a3141
-0,         53,         53,        1,   622080, 0xe62d2cfa
-0,         54,         54,        1,   622080, 0x4ab84909
-0,         55,         55,        1,   622080, 0x82de12ee
-0,         56,         56,        1,   622080, 0x4700b8f9
-0,         57,         57,        1,   622080, 0x7e849cc9
-0,         58,         58,        1,   622080, 0x6305aeba
-0,         59,         59,        1,   622080, 0x939bf771
-0,         60,         60,        1,   622080, 0x153faa3e
-0,         61,         61,        1,   622080, 0xb67f3233
-0,         62,         62,        1,   622080, 0xae724063
-0,         63,         63,        1,   622080, 0xed2b44b3
-0,         64,         64,        1,   622080, 0xbe4fe779
-0,         65,         65,        1,   622080, 0x380f8563
-0,         66,         66,        1,   622080, 0x209ed8c7
-0,         67,         67,        1,   622080, 0xb964d70f
-0,         68,         68,        1,   622080, 0xe2bbac96
-0,         69,         69,        1,   622080, 0x4f60f7f4
-0,         70,         70,        1,   622080, 0xe945441e
-0,         71,         71,        1,   622080, 0xded0b740
-0,         72,         72,        1,   622080, 0x3590bd5d
-0,         73,         73,        1,   622080, 0xb9a15294
-0,         74,         74,        1,   622080, 0x8f3ef716
-0,         75,         75,        1,   622080, 0x3e2301a8
-0,         76,         76,        1,   622080, 0x0109f125
-0,         77,         77,        1,   622080, 0x5252371e
-0,         78,         78,        1,   622080, 0x230c373f
-0,         79,         79,        1,   622080, 0x5a1ab1f1
+0,         18,         18,        1,   622080, 0x6331caee
+0,         19,         19,        1,   622080, 0x625da883
+0,         20,         20,        1,   622080, 0xa459e690
+0,         21,         21,        1,   622080, 0xce5d891e
+0,         22,         22,        1,   622080, 0x6429c648
+0,         23,         23,        1,   622080, 0x608cc0ba
+0,         24,         24,        1,   622080, 0xa49891ca
+0,         25,         25,        1,   622080, 0x9721987f
+0,         26,         26,        1,   622080, 0x2a887404
+0,         27,         27,        1,   622080, 0x60d71d47
+0,         28,         28,        1,   622080, 0xe8d49705
+0,         29,         29,        1,   622080, 0x821e13cb
+0,         30,         30,        1,   622080, 0x1b627835
+0,         31,         31,        1,   622080, 0x1806c5f4
+0,         32,         32,        1,   622080, 0x686858fd
+0,         33,         33,        1,   622080, 0xab865773
+0,         34,         34,        1,   622080, 0x2675174f
+0,         35,         35,        1,   622080, 0x43a61a14
+0,         36,         36,        1,   622080, 0x78470e7f
+0,         37,         37,        1,   622080, 0xeb877bc6
+0,         38,         38,        1,   622080, 0xffb366ec
+0,         39,         39,        1,   622080, 0xda0906e7
+0,         40,         40,        1,   622080, 0xd575da72
+0,         41,         41,        1,   622080, 0x23ae25a4
+0,         42,         42,        1,   622080, 0x5fb297f7
+0,         43,         43,        1,   622080, 0x99b32978
+0,         44,         44,        1,   622080, 0xbac77ca0
+0,         45,         45,        1,   622080, 0xc1cdcbf9
+0,         46,         46,        1,   622080, 0x3276ed72
+0,         47,         47,        1,   622080, 0x4061f5ab
+0,         48,         48,        1,   622080, 0x264092b2
+0,         49,         49,        1,   622080, 0xa4e2039e
+0,         50,         50,        1,   622080, 0x20ba1094
+0,         51,         51,        1,   622080, 0x984e906e
+0,         52,         52,        1,   622080, 0x76cc3139
+0,         53,         53,        1,   622080, 0xf70e2cf6
+0,         54,         54,        1,   622080, 0x469a4902
+0,         55,         55,        1,   622080, 0x235312e6
+0,         56,         56,        1,   622080, 0x0ed7b8f5
+0,         57,         57,        1,   622080, 0xd0269cc3
+0,         58,         58,        1,   622080, 0xdc51aeac
+0,         59,         59,        1,   622080, 0x1aa5f76e
+0,         60,         60,        1,   622080, 0xee06aa36
+0,         61,         61,        1,   622080, 0xa7103230
+0,         62,         62,        1,   622080, 0x7372405f
+0,         63,         63,        1,   622080, 0x8d7a44b5
+0,         64,         64,        1,   622080, 0x9e0ee776
+0,         65,         65,        1,   622080, 0xd41e8560
+0,         66,         66,        1,   622080, 0x39e6d8c9
+0,         67,         67,        1,   622080, 0x7a23d70c
+0,         68,         68,        1,   622080, 0x51d9ac9a
+0,         69,         69,        1,   622080, 0x8eacf7f2
+0,         70,         70,        1,   622080, 0x2b63441d
+0,         71,         71,        1,   622080, 0x9f71b742
+0,         72,         72,        1,   622080, 0x58afbd5e
+0,         73,         73,        1,   622080, 0x4d645292
+0,         74,         74,        1,   622080, 0xb972f716
+0,         75,         75,        1,   622080, 0xbb5d01a2
+0,         76,         76,        1,   622080, 0x6a6df129
+0,         77,         77,        1,   622080, 0x9e45371e
+0,         78,         78,        1,   622080, 0x28b1373d
+0,         79,         79,        1,   622080, 0xa1cdb1f2