yadif: redesign first and last 2 lines handling.

The previous code dependent on the input buffer matching the
buffer that has been provided by yadifs get_buffer.
The API does in now way gurantee this though its often true.
This fixes some out of array reads.
The regression test checksums change due to "out of picture" values
being initialized differently.
There should be no visual difference in the filters output

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

4 files changed, 64 insertions, 36 deletions

diff --git a/libavfilter/vf_yadif.c b/libavfilter/vf_yadif.c
index 215c764c7b..a79df7f00a 100644
--- a/libavfilter/vf_yadif.c
+++ b/libavfilter/vf_yadif.c
@@ -112,6 +112,7 @@ static void filter(AVFilterContext *ctx, AVFilterBufferRef *dstpic,
         int w = dstpic->video->w;
         int h = dstpic->video->h;
         int refs = yadif->cur->linesize[i];
+        int absrefs = FFABS(refs);
         int df = (yadif->csp->comp[i].depth_minus1 + 8) / 8;
 
         if (i == 1 || i == 2) {
@@ -120,6 +121,12 @@ static void filter(AVFilterContext *ctx, AVFilterBufferRef *dstpic,
             h >>= yadif->csp->log2_chroma_h;
         }
 
+        if(yadif->temp_line_size < absrefs) {
+            av_free(yadif->temp_line);
+            yadif->temp_line = av_mallocz(2*64 + 5*absrefs);
+            yadif->temp_line_size = absrefs;
+        }
+
         for (y = 0; y < h; y++) {
             if ((y ^ parity) & 1) {
                 uint8_t *prev = &yadif->prev->data[i][y*refs];
@@ -127,7 +134,25 @@ static void filter(AVFilterContext *ctx, AVFilterBufferRef *dstpic,
                 uint8_t *next = &yadif->next->data[i][y*refs];
                 uint8_t *dst  = &dstpic->data[i][y*dstpic->linesize[i]];
                 int     mode  = y==1 || y+2==h ? 2 : yadif->mode;
-                yadif->filter_line(dst, prev, cur, next, w, y+1<h ? refs : -refs, y ? -refs : refs, parity ^ tff, mode);
+                int     prefs = y+1<h ? refs : -refs;
+                int     mrefs =     y ?-refs :  refs;
+
+                if(y<=1 || y+2>=h) {
+                    int j;
+                    uint8_t *tmp = yadif->temp_line + 64 + 2*absrefs;
+                    if(mode<2)
+                        memcpy(tmp+2*mrefs, cur+2*mrefs, w*df);
+                    memcpy(tmp+mrefs, cur+mrefs, w*df);
+                    memcpy(tmp      , cur      , w*df);
+                    if(prefs != mrefs) {
+                        memcpy(tmp+prefs, cur+prefs, w*df);
+                        if(mode<2)
+                            memcpy(tmp+2*prefs, cur+2*prefs, w*df);
+                    }
+                    cur = tmp;
+                }
+
+                yadif->filter_line(dst, prev, cur, next, w, prefs, mrefs, parity ^ tff, mode);
             } else {
                 memcpy(&dstpic->data[i][y*dstpic->linesize[i]],
                        &yadif->cur->data[i][y*refs], w*df);
@@ -343,6 +368,7 @@ static av_cold void uninit(AVFilterContext *ctx)
     if (yadif->prev) avfilter_unref_bufferp(&yadif->prev);
     if (yadif->cur ) avfilter_unref_bufferp(&yadif->cur );
     if (yadif->next) avfilter_unref_bufferp(&yadif->next);
+    av_freep(&yadif->temp_line); yadif->temp_line_size = 0;
 }
 
 static int query_formats(AVFilterContext *ctx)
diff --git a/libavfilter/yadif.h b/libavfilter/yadif.h
index 4d3d818990..45bb1dc8ee 100644
--- a/libavfilter/yadif.h
+++ b/libavfilter/yadif.h
@@ -56,6 +56,8 @@ typedef struct {
 
     const AVPixFmtDescriptor *csp;
     int eof;
+    uint8_t *temp_line;
+    int temp_line_size;
 } YADIFContext;
 
 void ff_yadif_init_x86(YADIFContext *yadif);
diff --git a/tests/ref/fate/filter-yadif-mode0 b/tests/ref/fate/filter-yadif-mode0
index 0ed47a4e6d..7014333c4c 100644
--- a/tests/ref/fate/filter-yadif-mode0
+++ b/tests/ref/fate/filter-yadif-mode0
@@ -2,31 +2,31 @@
 0,          9,          9,        1,   622080, 0x4440caef
 0,         10,         10,        1,   622080, 0xce67e69d
 0,         11,         11,        1,   622080, 0x1dbdc653
-0,         12,         12,        1,   622080, 0x82c591d1
+0,         12,         12,        1,   622080, 0x55c791d0
 0,         13,         13,        1,   622080, 0x8193740b
-0,         14,         14,        1,   622080, 0xcb219711
-0,         15,         15,        1,   622080, 0x1870783b
+0,         14,         14,        1,   622080, 0x7125970f
+0,         15,         15,        1,   622080, 0xeb63783a
 0,         16,         16,        1,   622080, 0x7080590b
-0,         17,         17,        1,   622080, 0x6df4175d
-0,         18,         18,        1,   622080, 0x6b530e95
+0,         17,         17,        1,   622080, 0x13f8175b
+0,         18,         18,        1,   622080, 0x3e550e94
 0,         19,         19,        1,   622080, 0x7f9d66f7
-0,         20,         20,        1,   622080, 0x338cda81
-0,         21,         21,        1,   622080, 0xb13797f8
-0,         22,         22,        1,   622080, 0xb51e7ca4
+0,         20,         20,        1,   622080, 0x068eda80
+0,         21,         21,        1,   622080, 0x843997f7
+0,         22,         22,        1,   622080, 0x88207ca3
 0,         23,         23,        1,   622080, 0x353eed75
 0,         24,         24,        1,   622080, 0xf93e92b0
 0,         25,         25,        1,   622080, 0xd0811094
 0,         26,         26,        1,   622080, 0xb04a3141
 0,         27,         27,        1,   622080, 0x4ab84909
-0,         28,         28,        1,   622080, 0xa0fcb8fb
-0,         29,         29,        1,   622080, 0x9003aebb
+0,         28,         28,        1,   622080, 0x4700b8f9
+0,         29,         29,        1,   622080, 0x6305aeba
 0,         30,         30,        1,   622080, 0x153faa3e
 0,         31,         31,        1,   622080, 0xae724063
-0,         32,         32,        1,   622080, 0xeb4de77a
+0,         32,         32,        1,   622080, 0xbe4fe779
 0,         33,         33,        1,   622080, 0x209ed8c7
 0,         34,         34,        1,   622080, 0xe2bbac96
 0,         35,         35,        1,   622080, 0xe945441e
-0,         36,         36,        1,   622080, 0x8f8cbd5f
-0,         37,         37,        1,   622080, 0xbc3cf717
+0,         36,         36,        1,   622080, 0x3590bd5d
+0,         37,         37,        1,   622080, 0x8f3ef716
 0,         38,         38,        1,   622080, 0x0109f125
 0,         39,         39,        1,   622080, 0x230c373f
diff --git a/tests/ref/fate/filter-yadif-mode1 b/tests/ref/fate/filter-yadif-mode1
index 87bd18c362..87c8d97b6d 100644
--- a/tests/ref/fate/filter-yadif-mode1
+++ b/tests/ref/fate/filter-yadif-mode1
@@ -5,27 +5,27 @@
 0,         21,         21,        1,   622080, 0x9a57891f
 0,         22,         22,        1,   622080, 0x1dbdc653
 0,         23,         23,        1,   622080, 0xc171c0c5
-0,         24,         24,        1,   622080, 0x82c591d1
+0,         24,         24,        1,   622080, 0x55c791d0
 0,         25,         25,        1,   622080, 0x20db9890
 0,         26,         26,        1,   622080, 0x8193740b
 0,         27,         27,        1,   622080, 0xdb181d52
-0,         28,         28,        1,   622080, 0xcb219711
+0,         28,         28,        1,   622080, 0x7125970f
 0,         29,         29,        1,   622080, 0xc2b913d1
-0,         30,         30,        1,   622080, 0x1870783b
+0,         30,         30,        1,   622080, 0xeb63783a
 0,         31,         31,        1,   622080, 0xf1d9c5fb
 0,         32,         32,        1,   622080, 0x7080590b
-0,         33,         33,        1,   622080, 0x669c5775
-0,         34,         34,        1,   622080, 0x6df4175d
+0,         33,         33,        1,   622080, 0xeda55774
+0,         34,         34,        1,   622080, 0x13f8175b
 0,         35,         35,        1,   622080, 0x01921a16
-0,         36,         36,        1,   622080, 0x6b530e95
+0,         36,         36,        1,   622080, 0x3e550e94
 0,         37,         37,        1,   622080, 0xd5047bc9
 0,         38,         38,        1,   622080, 0x7f9d66f7
-0,         39,         39,        1,   622080, 0xa8b006eb
-0,         40,         40,        1,   622080, 0x338cda81
+0,         39,         39,        1,   622080, 0x2fc806ea
+0,         40,         40,        1,   622080, 0x068eda80
 0,         41,         41,        1,   622080, 0xf0e125a7
-0,         42,         42,        1,   622080, 0xb13797f8
+0,         42,         42,        1,   622080, 0x843997f7
 0,         43,         43,        1,   622080, 0x4afe2976
-0,         44,         44,        1,   622080, 0xb51e7ca4
+0,         44,         44,        1,   622080, 0x88207ca3
 0,         45,         45,        1,   622080, 0x637fcbfe
 0,         46,         46,        1,   622080, 0x353eed75
 0,         47,         47,        1,   622080, 0xd9a8f5ac
@@ -34,30 +34,30 @@
 0,         50,         50,        1,   622080, 0xd0811094
 0,         51,         51,        1,   622080, 0x3039906f
 0,         52,         52,        1,   622080, 0xb04a3141
-0,         53,         53,        1,   622080, 0x52872cf9
+0,         53,         53,        1,   622080, 0xe62d2cfa
 0,         54,         54,        1,   622080, 0x4ab84909
 0,         55,         55,        1,   622080, 0x82de12ee
-0,         56,         56,        1,   622080, 0xa0fcb8fb
+0,         56,         56,        1,   622080, 0x4700b8f9
 0,         57,         57,        1,   622080, 0x7e849cc9
-0,         58,         58,        1,   622080, 0x9003aebb
-0,         59,         59,        1,   622080, 0xffe6f770
+0,         58,         58,        1,   622080, 0x6305aeba
+0,         59,         59,        1,   622080, 0x939bf771
 0,         60,         60,        1,   622080, 0x153faa3e
 0,         61,         61,        1,   622080, 0xb67f3233
 0,         62,         62,        1,   622080, 0xae724063
-0,         63,         63,        1,   622080, 0x15fe44b4
-0,         64,         64,        1,   622080, 0xeb4de77a
+0,         63,         63,        1,   622080, 0xed2b44b3
+0,         64,         64,        1,   622080, 0xbe4fe779
 0,         65,         65,        1,   622080, 0x380f8563
 0,         66,         66,        1,   622080, 0x209ed8c7
 0,         67,         67,        1,   622080, 0xb964d70f
 0,         68,         68,        1,   622080, 0xe2bbac96
 0,         69,         69,        1,   622080, 0x4f60f7f4
 0,         70,         70,        1,   622080, 0xe945441e
-0,         71,         71,        1,   622080, 0xd0afb742
-0,         72,         72,        1,   622080, 0x8f8cbd5f
+0,         71,         71,        1,   622080, 0xded0b740
+0,         72,         72,        1,   622080, 0x3590bd5d
 0,         73,         73,        1,   622080, 0xb9a15294
-0,         74,         74,        1,   622080, 0xbc3cf717
-0,         75,         75,        1,   622080, 0xb70b01a9
+0,         74,         74,        1,   622080, 0x8f3ef716
+0,         75,         75,        1,   622080, 0x3e2301a8
 0,         76,         76,        1,   622080, 0x0109f125
-0,         77,         77,        1,   622080, 0xcb3a371f
+0,         77,         77,        1,   622080, 0x5252371e
 0,         78,         78,        1,   622080, 0x230c373f
-0,         79,         79,        1,   622080, 0x82dfb1f2
+0,         79,         79,        1,   622080, 0x5a1ab1f1