swscale/output: don't leave the alpha channel undefined in vuyx and xv36le

It's non-determistic, as shown by poisoning avfilter buffers instead of zeroing them.

Signed-off-by: James Almer <jamrial@gmail.com>

1 files changed, 6 insertions, 32 deletions

diff --git a/libswscale/output.c b/libswscale/output.c
index e8dd2145ce..abfb0fd1ce 100644
--- a/libswscale/output.c
+++ b/libswscale/output.c
@@ -2650,7 +2650,7 @@ yuv2xv36le_X_c(SwsContext *c, const int16_t *lumFilter,
 {
     int i;
     for (i = 0; i < dstW; i++) {
-        int Y = 1 << 14, U = 1 << 14, V = 1 << 14;
+        int Y = 1 << 14, U = 1 << 14, V = 1 << 14, A = 255;
         int j;
 
         for (j = 0; j < lumFilterSize; j++)
@@ -2664,6 +2664,7 @@ yuv2xv36le_X_c(SwsContext *c, const int16_t *lumFilter,
         AV_WL16(dest + 8 * i + 2, av_clip_uintp2(Y >> 15, 12) << 4);
         AV_WL16(dest + 8 * i + 0, av_clip_uintp2(U >> 15, 12) << 4);
         AV_WL16(dest + 8 * i + 4, av_clip_uintp2(V >> 15, 12) << 4);
+        AV_WL16(dest + 8 * i + 6, A);
     }
 }
 
@@ -2672,10 +2673,8 @@ yuv2vuyX_X_c(SwsContext *c, const int16_t *lumFilter,
              const int16_t **lumSrc, int lumFilterSize,
              const int16_t *chrFilter, const int16_t **chrUSrc,
              const int16_t **chrVSrc, int chrFilterSize,
-             const int16_t **alpSrc, uint8_t *dest, int dstW, int y,
-             int destHasAlpha)
+             const int16_t **alpSrc, uint8_t *dest, int dstW, int y)
 {
-    int hasAlpha = destHasAlpha && (!!alpSrc);
     int i;
 
     for (i = 0; i < dstW; i++) {
@@ -2703,7 +2702,7 @@ yuv2vuyX_X_c(SwsContext *c, const int16_t *lumFilter,
         if (V  & 0x100)
             V = av_clip_uint8(V);
 
-        if (hasAlpha) {
+        if (alpSrc) {
             A = 1 << 18;
 
             for (j = 0; j < lumFilterSize; j++)
@@ -2718,33 +2717,10 @@ yuv2vuyX_X_c(SwsContext *c, const int16_t *lumFilter,
         dest[4 * i    ] = V;
         dest[4 * i + 1] = U;
         dest[4 * i + 2] = Y;
-        if (destHasAlpha)
-            dest[4 * i + 3] = A;
+        dest[4 * i + 3] = A;
     }
 }
 
-static void
-yuv2vuya_X_c(SwsContext *c, const int16_t *lumFilter,
-             const int16_t **lumSrc, int lumFilterSize,
-             const int16_t *chrFilter, const int16_t **chrUSrc,
-             const int16_t **chrVSrc, int chrFilterSize,
-             const int16_t **alpSrc, uint8_t *dest, int dstW, int y)
-{
-    yuv2vuyX_X_c(c, lumFilter, lumSrc, lumFilterSize, chrFilter,
-                 chrUSrc, chrVSrc, chrFilterSize, alpSrc, dest, dstW, y, 1);
-}
-
-static void
-yuv2vuyx_X_c(SwsContext *c, const int16_t *lumFilter,
-             const int16_t **lumSrc, int lumFilterSize,
-             const int16_t *chrFilter, const int16_t **chrUSrc,
-             const int16_t **chrVSrc, int chrFilterSize,
-             const int16_t **alpSrc, uint8_t *dest, int dstW, int y)
-{
-    yuv2vuyX_X_c(c, lumFilter, lumSrc, lumFilterSize, chrFilter,
-                 chrUSrc, chrVSrc, chrFilterSize, alpSrc, dest, dstW, y, 0);
-}
-
 #define output_pixel(pos, val, bits) \
     AV_WL16(pos, av_clip_uintp2(val >> shift, bits) << output_shift);
 
@@ -3298,10 +3274,8 @@ av_cold void ff_sws_init_output_funcs(SwsContext *c,
         *yuv2packedX = yuv2ayuv64le_X_c;
         break;
     case AV_PIX_FMT_VUYA:
-        *yuv2packedX = yuv2vuya_X_c;
-        break;
     case AV_PIX_FMT_VUYX:
-        *yuv2packedX = yuv2vuyx_X_c;
+        *yuv2packedX = yuv2vuyX_X_c;
         break;
     case AV_PIX_FMT_XV30LE:
         *yuv2packedX = yuv2xv30le_X_c;