swscale: fix ring buffer size when scaling slices of a frame

The ring buffer size should be able to store input lines
when there is not enough lines to output a single line.

2 files changed, 51 insertions, 6 deletions

diff --git a/libswscale/slice.c b/libswscale/slice.c
index 0159a739c3..e14456f75f 100644
--- a/libswscale/slice.c
+++ b/libswscale/slice.c
@@ -210,6 +210,42 @@ static void fill_ones(SwsSlice *s, int n, int is16bit)
     }
 }
 
+/*
+ Calculates the minimum ring buffer size, it should be able to store vFilterSize
+ more n lines where n is the max difference between each adjacent slice which
+ outputs a line.
+ The n lines are needed only when there is not enough src lines to output a single
+ dst line, then we should buffer these lines to process them on the next call to scale.
+*/
+static void get_min_buffer_size(SwsContext *c, int *out_lum_size, int *out_chr_size)
+{
+    int lumY;
+    int dstH = c->dstH;
+    int chrDstH = c->chrDstH;
+    int *lumFilterPos = c->vLumFilterPos;
+    int *chrFilterPos = c->vChrFilterPos;
+    int lumFilterSize = c->vLumFilterSize;
+    int chrFilterSize = c->vChrFilterSize;
+    int chrSubSample = c->chrSrcVSubSample;
+
+    *out_lum_size = lumFilterSize;
+    *out_chr_size = chrFilterSize;
+
+    for (lumY = 0; lumY < dstH; lumY++) {
+        int chrY      = (int64_t)lumY * chrDstH / dstH;
+        int nextSlice = FFMAX(lumFilterPos[lumY] + lumFilterSize - 1,
+                              ((chrFilterPos[chrY] + chrFilterSize - 1)
+                               << chrSubSample));
+
+        nextSlice >>= chrSubSample;
+        nextSlice <<= chrSubSample;
+        (*out_lum_size) = FFMAX((*out_lum_size), nextSlice - lumFilterPos[lumY]);
+        (*out_chr_size) = FFMAX((*out_chr_size), (nextSlice >> chrSubSample) - chrFilterPos[chrY]);
+    }
+}
+
+
+
 int ff_init_filters(SwsContext * c)
 {
     int i;
@@ -226,6 +262,13 @@ int ff_init_filters(SwsContext * c)
     uint32_t * pal = usePal(c->srcFormat) ? c->pal_yuv : (uint32_t*)c->input_rgb2yuv_table;
     int res = 0;
 
+    int lumBufSize;
+    int chrBufSize;
+
+    get_min_buffer_size(c, &lumBufSize, &chrBufSize);
+    lumBufSize = FFMAX(lumBufSize, c->vLumFilterSize + MAX_LINES_AHEAD);
+    chrBufSize = FFMAX(chrBufSize, c->vChrFilterSize + MAX_LINES_AHEAD);
+
     if (c->dstBpc == 16)
         dst_stride <<= 1;
 
@@ -248,13 +291,13 @@ int ff_init_filters(SwsContext * c)
     res = alloc_slice(&c->slice[0], c->srcFormat, c->srcH, c->chrSrcH, c->chrSrcHSubSample, c->chrSrcVSubSample, 0);
     if (res < 0) goto cleanup;
     for (i = 1; i < c->numSlice-2; ++i) {
-        res = alloc_slice(&c->slice[i], c->srcFormat, c->vLumFilterSize + MAX_LINES_AHEAD, c->vChrFilterSize + MAX_LINES_AHEAD, c->chrSrcHSubSample, c->chrSrcVSubSample, 0);
+        res = alloc_slice(&c->slice[i], c->srcFormat, lumBufSize, chrBufSize, c->chrSrcHSubSample, c->chrSrcVSubSample, 0);
         if (res < 0) goto cleanup;
         res = alloc_lines(&c->slice[i], FFALIGN(c->srcW*2+78, 16), c->srcW);
         if (res < 0) goto cleanup;
     }
     // horizontal scaler output
-    res = alloc_slice(&c->slice[i], c->srcFormat, c->vLumFilterSize + MAX_LINES_AHEAD, c->vChrFilterSize + MAX_LINES_AHEAD, c->chrDstHSubSample, c->chrDstVSubSample, 1);
+    res = alloc_slice(&c->slice[i], c->srcFormat, lumBufSize, chrBufSize, c->chrDstHSubSample, c->chrDstVSubSample, 1);
     if (res < 0) goto cleanup;
     res = alloc_lines(&c->slice[i], dst_stride, c->dstW);
     if (res < 0) goto cleanup;
diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 2e246d9997..bbea0feeab 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -413,8 +413,6 @@ static int swscale(SwsContext *c, const uint8_t *src[],
 
             lastInChrBuf = firstChrSrcY - 1;
         }
-        av_assert0(firstLumSrcY >= lastInLumBuf - vLumFilterSize + 1);
-        av_assert0(firstChrSrcY >= lastInChrBuf - vChrFilterSize + 1);
 
         DEBUG_BUFFERS("dstY: %d\n", dstY);
         DEBUG_BUFFERS("\tfirstLumSrcY: %d lastLumSrcY: %d lastInLumBuf: %d\n",
@@ -433,10 +431,14 @@ static int swscale(SwsContext *c, const uint8_t *src[],
                           lastLumSrcY, lastChrSrcY);
         }
 
+        av_assert0((lastLumSrcY - firstLumSrcY + 1) <= hout_slice->plane[0].available_lines);
+        av_assert0((lastChrSrcY - firstChrSrcY + 1) <= hout_slice->plane[1].available_lines);
+
+
         posY = hout_slice->plane[0].sliceY + hout_slice->plane[0].sliceH;
         if (posY <= lastLumSrcY && !hasLumHoles) {
             firstPosY = FFMAX(firstLumSrcY, posY);
-            lastPosY = FFMIN(lastLumSrcY + MAX_LINES_AHEAD, srcSliceY + srcSliceH - 1);
+            lastPosY = FFMIN(firstLumSrcY + hout_slice->plane[0].available_lines - 1, srcSliceY + srcSliceH - 1);
         } else {
             firstPosY = lastInLumBuf + 1;
             lastPosY = lastLumSrcY;
@@ -445,7 +447,7 @@ static int swscale(SwsContext *c, const uint8_t *src[],
         cPosY = hout_slice->plane[1].sliceY + hout_slice->plane[1].sliceH;
         if (cPosY <= lastChrSrcY && !hasChrHoles) {
             firstCPosY = FFMAX(firstChrSrcY, cPosY);
-            lastCPosY = FFMIN(lastChrSrcY + MAX_LINES_AHEAD, AV_CEIL_RSHIFT(srcSliceY + srcSliceH, c->chrSrcVSubSample) - 1);
+            lastCPosY = FFMIN(firstChrSrcY + hout_slice->plane[1].available_lines - 1, AV_CEIL_RSHIFT(srcSliceY + srcSliceH, c->chrSrcVSubSample) - 1);
         } else {
             firstCPosY = lastInChrBuf + 1;
             lastCPosY = lastChrSrcY;