swscale/alphablend: don't overread alpha plane on subsampled odd size

This function overreads the input plane for odd dimensions, because the
chroma plane is always rounded up, which means (xy << subsample) + 1 exceeds
the actual alpha plane size.

To verify:
  valgrind ffmpeg -pix_fmt yuva420p -f lavfi -i color -vf \
  "scale=1x1,format=yuva420p,scale=alphablend=uniform_color,format=yuv420p \
  -vframes 1 -f null -

Fixes: https://trac.ffmpeg.org/ticket/11692

1 files changed, 19 insertions, 13 deletions

diff --git a/libswscale/alphablend.c b/libswscale/alphablend.c
index f9484bcc91..45db5189d2 100644
--- a/libswscale/alphablend.c
+++ b/libswscale/alphablend.c
@@ -25,6 +25,8 @@ int ff_sws_alphablendaway(SwsInternal *c, const uint8_t *const src[],
                           uint8_t *const dst[], const int dstStride[])
 {
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(c->opts.src_format);
+    const int lum_w = c->opts.src_w;
+    const int lum_h = c->opts.src_h;
     int nb_components = desc->nb_components;
     int plane, x, ysrc;
     int plane_count = isGray(c->opts.src_format) ? 1 : 3;
@@ -52,7 +54,8 @@ int ff_sws_alphablendaway(SwsInternal *c, const uint8_t *const src[],
             int y_subsample = plane ? desc->log2_chroma_h: 0;
             for (ysrc = 0; ysrc < AV_CEIL_RSHIFT(srcSliceH, y_subsample); ysrc++) {
                 int y = ysrc + (srcSliceY >> y_subsample);
-                if (x_subsample || y_subsample) {
+                int subsample_row = y_subsample && (y << y_subsample) + 1 < lum_h;
+                if (x_subsample || subsample_row) {
                     int alpha;
                     unsigned u;
                     if (sixteen_bits) {
@@ -62,21 +65,23 @@ int ff_sws_alphablendaway(SwsInternal *c, const uint8_t *const src[],
                               uint16_t *d = (      uint16_t *)(dst[plane      ] +  dstStride[plane      ] * y);
                         if ((!isBE(c->opts.src_format)) == !HAVE_BIGENDIAN) {
                             for (x = 0; x < w; x++) {
-                                if (y_subsample) {
-                                    alpha = (a[2*x]              + a[2*x + 1] + 2 +
-                                             a[2*x + alpha_step] + a[2*x + alpha_step + 1]) >> 2;
+                                const int xnext = FFMIN(2*x + 1, lum_w - 1);
+                                if (subsample_row) {
+                                    alpha = (a[2*x]              + a[xnext] + 2 +
+                                             a[2*x + alpha_step] + a[xnext + alpha_step]) >> 2;
                                 } else
-                                    alpha = (a[2*x] + a[2*x + 1]) >> 1;
+                                    alpha = (a[2*x] + a[xnext]) >> 1;
                                 u = s[x]*alpha + target_table[((x^y)>>5)&1][plane]*(max-alpha) + off;
                                 d[x] = av_clip((u + (u >> shift)) >> shift, 0, max);
                             }
                         } else {
                             for (x = 0; x < w; x++) {
-                                if (y_subsample) {
-                                    alpha = (av_bswap16(a[2*x])              + av_bswap16(a[2*x + 1]) + 2 +
-                                             av_bswap16(a[2*x + alpha_step]) + av_bswap16(a[2*x + alpha_step + 1])) >> 2;
+                                const int xnext = FFMIN(2*x + 1, lum_w - 1);
+                                if (subsample_row) {
+                                    alpha = (av_bswap16(a[2*x])              + av_bswap16(a[xnext]) + 2 +
+                                             av_bswap16(a[2*x + alpha_step]) + av_bswap16(a[xnext + alpha_step])) >> 2;
                                 } else
-                                    alpha = (av_bswap16(a[2*x]) + av_bswap16(a[2*x + 1])) >> 1;
+                                    alpha = (av_bswap16(a[2*x]) + av_bswap16(a[xnext])) >> 1;
                                 u = av_bswap16(s[x])*alpha + target_table[((x^y)>>5)&1][plane]*(max-alpha) + off;
                                 d[x] = av_clip((u + (u >> shift)) >> shift, 0, max);
                             }
@@ -87,11 +92,12 @@ int ff_sws_alphablendaway(SwsInternal *c, const uint8_t *const src[],
                         const uint8_t *a = src[plane_count] + (srcStride[plane_count] * ysrc << y_subsample);
                               uint8_t *d = dst[plane      ] + dstStride[plane] * y;
                         for (x = 0; x < w; x++) {
-                            if (y_subsample) {
-                                alpha = (a[2*x]              + a[2*x + 1] + 2 +
-                                         a[2*x + alpha_step] + a[2*x + alpha_step + 1]) >> 2;
+                            const int xnext = FFMIN(2*x + 1, lum_w - 1);
+                            if (subsample_row) {
+                                alpha = (a[2*x]              + a[xnext] + 2 +
+                                         a[2*x + alpha_step] + a[xnext + alpha_step]) >> 2;
                             } else
-                                alpha = (a[2*x] + a[2*x + 1]) >> 1;
+                                alpha = (a[2*x] + a[xnext]) >> 1;
                             u = s[x]*alpha + target_table[((x^y)>>5)&1][plane]*(255-alpha) + 128;
                             d[x] = (257*u) >> 16;
                         }