avfilter/vf_scale: reset color matrix in case of identity & non-RGB

Fixes passing through mismatching metadata from the input side
when RGB input (from f.ex. H.264 or HEVC) gets converted to YCbCr.

Fixes #9132

(cherry picked from commit 9dd410c80416197188337e3b7e1600be41d2ea64)

1 files changed, 9 insertions, 0 deletions

diff --git a/libavfilter/vf_scale.c b/libavfilter/vf_scale.c
index 5ad9334d02..6e2294a4d2 100644
--- a/libavfilter/vf_scale.c
+++ b/libavfilter/vf_scale.c
@@ -739,6 +739,15 @@ scale:
     out->width  = outlink->w;
     out->height = outlink->h;
 
+    // Sanity check: If we've got the RGB/XYZ (identity) matrix configured, and
+    //               the output is no longer RGB, unset the matrix.
+    //               In theory this should be in swscale itself as the AVFrame
+    //               based API gets in, so that not every swscale API user has
+    //               to go through duplicating such sanity checks.
+    if (out->colorspace == AVCOL_SPC_RGB &&
+        !(av_pix_fmt_desc_get(out->format)->flags & AV_PIX_FMT_FLAG_RGB))
+        out->colorspace = AVCOL_SPC_UNSPECIFIED;
+
     if (scale->output_is_pal)
         avpriv_set_systematic_pal2((uint32_t*)out->data[1], outlink->format == AV_PIX_FMT_PAL8 ? AV_PIX_FMT_BGR8 : outlink->format);