avfilter: add link negotiation for AVAlphaMode

8 files changed, 158 insertions, 5 deletions

diff --git a/doc/APIchanges b/doc/APIchanges
index d49cdd8bdc..87357e5909 100644
--- a/doc/APIchanges
+++ b/doc/APIchanges
@@ -2,6 +2,9 @@ The last version increases of all libraries were on 2025-03-28
 
 API changes, most recent first:
 
+2025-09-xx - xxxxxxxxxx - lavfi 11.6.100 - avfilter.h
+  Add AVFilterLink.alpha_mode.
+
 2025-09-xx - xxxxxxxxxx - lavu 60.11.100 - frame.h pixfmt.h
   Add AVAlphaMode, AVFrame.alpha_mode, av_alpha_mode_name() and
   av_alpha_mode_from_name().
diff --git a/libavfilter/avfilter.c b/libavfilter/avfilter.c
index 5bcf0b4ef7..f9b64810b8 100644
--- a/libavfilter/avfilter.c
+++ b/libavfilter/avfilter.c
@@ -312,6 +312,9 @@ int avfilter_insert_filter(AVFilterLink *link, AVFilterContext *filt,
     if (link->outcfg.color_ranges)
         ff_formats_changeref(&link->outcfg.color_ranges,
                              &filt->outputs[filt_dstpad_idx]->outcfg.color_ranges);
+    if (link->outcfg.alpha_modes)
+        ff_formats_changeref(&link->outcfg.alpha_modes,
+                             &filt->outputs[filt_dstpad_idx]->outcfg.alpha_modes);
     if (link->outcfg.samplerates)
         ff_formats_changeref(&link->outcfg.samplerates,
                              &filt->outputs[filt_dstpad_idx]->outcfg.samplerates);
@@ -784,6 +787,8 @@ static void free_link(AVFilterLink *link)
     ff_formats_unref(&link->outcfg.color_spaces);
     ff_formats_unref(&link->incfg.color_ranges);
     ff_formats_unref(&link->outcfg.color_ranges);
+    ff_formats_unref(&link->incfg.alpha_modes);
+    ff_formats_unref(&link->outcfg.alpha_modes);
     ff_formats_unref(&link->incfg.samplerates);
     ff_formats_unref(&link->outcfg.samplerates);
     ff_channel_layouts_unref(&link->incfg.channel_layouts);
diff --git a/libavfilter/avfilter.h b/libavfilter/avfilter.h
index a92144b0a6..02b58c42c2 100644
--- a/libavfilter/avfilter.h
+++ b/libavfilter/avfilter.h
@@ -141,6 +141,11 @@ typedef struct AVFilterFormatsConfig {
     AVFilterFormats *color_spaces;  ///< AVColorSpace
     AVFilterFormats *color_ranges;  ///< AVColorRange
 
+    /**
+     * List of supported alpha modes, only for video with an alpha channel.
+     */
+    AVFilterFormats *alpha_modes;  ///< AVAlphaMode
+
 } AVFilterFormatsConfig;
 
 /**
@@ -428,6 +433,8 @@ struct AVFilterLink {
     AVFrameSideData **side_data;
     int nb_side_data;
 
+    enum AVAlphaMode alpha_mode; ///< alpha mode (for videos with an alpha channel)
+
     /*****************************************************************
      * All fields below this line are not part of the public API. They
      * may not be used outside of libavfilter and can be changed and
diff --git a/libavfilter/avfiltergraph.c b/libavfilter/avfiltergraph.c
index 91aea7d36c..4c80204f01 100644
--- a/libavfilter/avfiltergraph.c
+++ b/libavfilter/avfiltergraph.c
@@ -303,7 +303,8 @@ static int filter_link_check_formats(void *log, AVFilterLink *link, AVFilterForm
     case AVMEDIA_TYPE_VIDEO:
         if ((ret = ff_formats_check_pixel_formats(log, cfg->formats)) < 0 ||
             (ret = ff_formats_check_color_spaces(log, cfg->color_spaces)) < 0 ||
-            (ret = ff_formats_check_color_ranges(log, cfg->color_ranges)) < 0)
+            (ret = ff_formats_check_color_ranges(log, cfg->color_ranges)) < 0 ||
+            (ret = ff_formats_check_alpha_modes(log, cfg->alpha_modes)) < 0)
             return ret;
         break;
 
@@ -418,7 +419,8 @@ static int formats_declared(AVFilterContext *f)
             return 0;
         if (f->inputs[i]->type == AVMEDIA_TYPE_VIDEO &&
             !(f->inputs[i]->outcfg.color_ranges &&
-              f->inputs[i]->outcfg.color_spaces))
+              f->inputs[i]->outcfg.color_spaces &&
+              f->inputs[i]->outcfg.alpha_modes))
             return 0;
         if (f->inputs[i]->type == AVMEDIA_TYPE_AUDIO &&
             !(f->inputs[i]->outcfg.samplerates &&
@@ -430,7 +432,8 @@ static int formats_declared(AVFilterContext *f)
             return 0;
         if (f->outputs[i]->type == AVMEDIA_TYPE_VIDEO &&
             !(f->outputs[i]->incfg.color_ranges &&
-              f->outputs[i]->incfg.color_spaces))
+              f->outputs[i]->incfg.color_spaces &&
+              f->outputs[i]->incfg.alpha_modes))
             return 0;
         if (f->outputs[i]->type == AVMEDIA_TYPE_AUDIO &&
             !(f->outputs[i]->incfg.samplerates &&
@@ -641,6 +644,10 @@ retry:
                     av_assert0( inlink->outcfg.color_ranges->refcount > 0);
                     av_assert0(outlink-> incfg.color_ranges->refcount > 0);
                     av_assert0(outlink->outcfg.color_ranges->refcount > 0);
+                    av_assert0( inlink-> incfg.alpha_modes->refcount > 0);
+                    av_assert0( inlink->outcfg.alpha_modes->refcount > 0);
+                    av_assert0(outlink-> incfg.alpha_modes->refcount > 0);
+                    av_assert0(outlink->outcfg.alpha_modes->refcount > 0);
                 } else if (outlink->type == AVMEDIA_TYPE_AUDIO) {
                     av_assert0( inlink-> incfg.samplerates->refcount > 0);
                     av_assert0( inlink->outcfg.samplerates->refcount > 0);
@@ -820,8 +827,8 @@ static int pick_format(AVFilterLink *link, AVFilterLink *ref)
             swfmt = AV_PIX_FMT_YUV420P;
         }
 
+        const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(swfmt);
         if (!ff_fmt_is_regular_yuv(swfmt)) {
-            const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(swfmt);
             /* These fields are explicitly documented as affecting YUV only,
              * so set them to sane values for other formats. */
             if (desc->flags & AV_PIX_FMT_FLAG_FLOAT)
@@ -856,6 +863,19 @@ static int pick_format(AVFilterLink *link, AVFilterLink *ref)
                 link->color_range = link->incfg.color_ranges->formats[0];
             }
         }
+
+        if (desc->flags & AV_PIX_FMT_FLAG_ALPHA) {
+            if (!link->incfg.alpha_modes->nb_formats) {
+                av_log(link->src, AV_LOG_ERROR, "Cannot select alpha mode for"
+                       " the link between filters %s and %s.\n", link->src->name,
+                       link->dst->name);
+                return AVERROR(EINVAL);
+            }
+            link->incfg.alpha_modes->nb_formats = 1;
+            link->alpha_mode = link->incfg.alpha_modes->formats[0];
+        } else {
+            link->alpha_mode = AVALPHA_MODE_UNSPECIFIED;
+        }
     } else if (link->type == AVMEDIA_TYPE_AUDIO) {
         int ret;
 
@@ -894,6 +914,8 @@ static int pick_format(AVFilterLink *link, AVFilterLink *ref)
     ff_formats_unref(&link->outcfg.color_spaces);
     ff_formats_unref(&link->incfg.color_ranges);
     ff_formats_unref(&link->outcfg.color_ranges);
+    ff_formats_unref(&link->incfg.alpha_modes);
+    ff_formats_unref(&link->outcfg.alpha_modes);
 
     return 0;
 }
@@ -947,6 +969,8 @@ static int reduce_formats_on_filter(AVFilterContext *filter)
                    nb_formats, ff_add_format);
     REDUCE_FORMATS(int,      AVFilterFormats,        color_ranges,    formats,
                    nb_formats, ff_add_format);
+    REDUCE_FORMATS(int,      AVFilterFormats,        alpha_modes,     formats,
+                   nb_formats, ff_add_format);
 
     /* reduce channel layouts */
     for (i = 0; i < filter->nb_inputs; i++) {
diff --git a/libavfilter/formats.c b/libavfilter/formats.c
index 81c63e50cd..4aa171a06e 100644
--- a/libavfilter/formats.c
+++ b/libavfilter/formats.c
@@ -372,6 +372,11 @@ static const AVFilterFormatsMerger mergers_video[] = {
         .can_merge  = can_merge_generic,
         CONVERSION_FILTER_SWSCALE
     },
+    {
+        .offset     = offsetof(AVFilterFormatsConfig, alpha_modes),
+        .merge      = merge_generic,
+        .can_merge  = can_merge_generic,
+    },
 };
 
 static const AVFilterFormatsMerger mergers_audio[] = {
@@ -665,6 +670,17 @@ AVFilterFormats *ff_all_color_ranges(void)
     return ret;
 }
 
+AVFilterFormats *ff_all_alpha_modes(void)
+{
+    AVFilterFormats *ret = NULL;
+    for (int range = 0; range < AVALPHA_MODE_NB; range++) {
+        if (ff_add_format(&ret, range) < 0)
+            return NULL;
+    }
+
+    return ret;
+}
+
 #define FORMATS_REF(f, ref, unref_fn)                                           \
     void *tmp;                                                                  \
                                                                                 \
@@ -870,6 +886,24 @@ int ff_set_common_all_color_ranges(AVFilterContext *ctx)
     return ff_set_common_color_ranges(ctx, ff_all_color_ranges());
 }
 
+int ff_set_common_alpha_modes(AVFilterContext *ctx,
+                              AVFilterFormats *alpha_modes)
+{
+    SET_COMMON_FORMATS(ctx, alpha_modes, AVMEDIA_TYPE_VIDEO,
+                       ff_formats_ref, ff_formats_unref);
+}
+
+int ff_set_common_alpha_modes_from_list(AVFilterContext *ctx,
+                                        const int *alpha_modes)
+{
+    return ff_set_common_alpha_modes(ctx, ff_make_format_list(alpha_modes));
+}
+
+int ff_set_common_all_alpha_modes(AVFilterContext *ctx)
+{
+    return ff_set_common_alpha_modes(ctx, ff_all_alpha_modes());
+}
+
 /**
  * A helper for query_formats() which sets all links to the same list of
  * formats. If there are no links hooked to this filter, the list of formats is
@@ -1015,6 +1049,30 @@ int ff_set_common_all_color_ranges2(const AVFilterContext *ctx,
     return ff_set_common_color_ranges2(ctx, cfg_in, cfg_out, ff_all_color_ranges());
 }
 
+int ff_set_common_alpha_modes2(const AVFilterContext *ctx,
+                               AVFilterFormatsConfig **cfg_in,
+                               AVFilterFormatsConfig **cfg_out,
+                               AVFilterFormats *alpha_modes)
+{
+    SET_COMMON_FORMATS2(ctx, cfg_in, cfg_out, alpha_modes, AVMEDIA_TYPE_VIDEO,
+                        ff_formats_ref, ff_formats_unref);
+}
+
+int ff_set_common_alpha_modes_from_list2(const AVFilterContext *ctx,
+                                         AVFilterFormatsConfig **cfg_in,
+                                         AVFilterFormatsConfig **cfg_out,
+                                         const int *alpha_modes)
+{
+    return ff_set_common_alpha_modes2(ctx, cfg_in, cfg_out, ff_make_format_list(alpha_modes));
+}
+
+int ff_set_common_all_alpha_modes2(const AVFilterContext *ctx,
+                                   AVFilterFormatsConfig **cfg_in,
+                                   AVFilterFormatsConfig **cfg_out)
+{
+    return ff_set_common_alpha_modes2(ctx, cfg_in, cfg_out, ff_all_alpha_modes());
+}
+
 int ff_set_common_formats2(const AVFilterContext *ctx,
                            AVFilterFormatsConfig **cfg_in,
                            AVFilterFormatsConfig **cfg_out,
@@ -1080,6 +1138,9 @@ int ff_default_query_formats(AVFilterContext *ctx)
         ret = ff_set_common_all_color_ranges(ctx);
         if (ret < 0)
             return ret;
+        ret = ff_set_common_all_alpha_modes(ctx);
+        if (ret < 0)
+            return ret;
     }
     if (type != AVMEDIA_TYPE_VIDEO) {
         ret = ff_set_common_all_channel_counts(ctx);
@@ -1147,6 +1208,11 @@ int ff_formats_check_color_ranges(void *log, const AVFilterFormats *fmts)
     return check_list(log, "color range", fmts);
 }
 
+int ff_formats_check_alpha_modes(void *log, const AVFilterFormats *fmts)
+{
+    return check_list(log, "alpha mode", fmts);
+}
+
 static int layouts_compatible(const AVChannelLayout *a, const AVChannelLayout *b)
 {
     return !av_channel_layout_compare(a, b) ||
diff --git a/libavfilter/formats.h b/libavfilter/formats.h
index b23ebf82f9..0c92ecad3f 100644
--- a/libavfilter/formats.h
+++ b/libavfilter/formats.h
@@ -145,6 +145,12 @@ av_warn_unused_result
 AVFilterFormats *ff_all_color_ranges(void);
 
 /**
+ * Construct an AVFilterFormats representing all possible alpha modes.
+ */
+av_warn_unused_result
+AVFilterFormats *ff_all_alpha_modes(void);
+
+/**
  * Helpers for query_formats() which set all free audio links to the same list
  * of channel layouts/sample rates. If there are no links hooked to this list,
  * the list is freed.
@@ -211,6 +217,23 @@ int ff_set_common_color_ranges_from_list(AVFilterContext *ctx,
 av_warn_unused_result
 int ff_set_common_all_color_ranges(AVFilterContext *ctx);
 
+av_warn_unused_result
+int ff_set_common_alpha_modes(AVFilterContext *ctx,
+                              AVFilterFormats *alpha_modes);
+/**
+ * Equivalent to ff_set_common_alpha_modes(ctx, ff_make_format_list(alpha_modes))
+ */
+av_warn_unused_result
+int ff_set_common_alpha_modes_from_list(AVFilterContext *ctx,
+                                        const int *alpha_modes);
+
+/**
+ * Equivalent to ff_set_common_alpha_modes(ctx, ff_all_alpha_modes())
+ */
+av_warn_unused_result
+int ff_set_common_all_alpha_modes(AVFilterContext *ctx);
+
+
 /**
  * A helper for query_formats() which sets all links to the same list of
  * formats. If there are no links hooked to this filter, the list of formats is
@@ -298,6 +321,23 @@ int ff_set_common_all_color_ranges2(const AVFilterContext *ctx,
                                     AVFilterFormatsConfig **cfg_out);
 
 av_warn_unused_result
+int ff_set_common_alpha_modes2(const AVFilterContext *ctx,
+                               AVFilterFormatsConfig **cfg_in,
+                               AVFilterFormatsConfig **cfg_out,
+                               AVFilterFormats *alpha_modes);
+
+av_warn_unused_result
+int ff_set_common_alpha_modes_from_list2(const AVFilterContext *ctx,
+                                         AVFilterFormatsConfig **cfg_in,
+                                         AVFilterFormatsConfig **cfg_out,
+                                         const int *alpha_modes);
+
+av_warn_unused_result
+int ff_set_common_all_alpha_modes2(const AVFilterContext *ctx,
+                                   AVFilterFormatsConfig **cfg_in,
+                                   AVFilterFormatsConfig **cfg_out);
+
+av_warn_unused_result
 int ff_set_common_formats2(const AVFilterContext *ctx,
                            AVFilterFormatsConfig **cfg_in,
                            AVFilterFormatsConfig **cfg_out,
@@ -466,6 +506,13 @@ int ff_formats_check_channel_layouts(void *log, const AVFilterChannelLayouts *fm
 int ff_formats_check_color_spaces(void *log, const AVFilterFormats *fmts);
 int ff_formats_check_color_ranges(void *log, const AVFilterFormats *fmts);
 
+/**
+ * Check that fmts is a valid formats list for alpha modes.
+ *
+ * In particular, check for duplicates.
+ */
+int ff_formats_check_alpha_modes(void *log, const AVFilterFormats *fmts);
+
 typedef struct AVFilterFormatMerger {
     unsigned offset;
     int (*merge)(void *a, void *b);
diff --git a/libavfilter/version.h b/libavfilter/version.h
index 523a7fe0a6..f191d98883 100644
--- a/libavfilter/version.h
+++ b/libavfilter/version.h
@@ -31,7 +31,7 @@
 
 #include "version_major.h"
 
-#define LIBAVFILTER_VERSION_MINOR   5
+#define LIBAVFILTER_VERSION_MINOR   6
 #define LIBAVFILTER_VERSION_MICRO 100
 
 
diff --git a/libavfilter/video.c b/libavfilter/video.c
index 1de7f2ef59..13dab90b65 100644
--- a/libavfilter/video.c
+++ b/libavfilter/video.c
@@ -104,6 +104,7 @@ AVFrame *ff_default_get_video_buffer2(AVFilterLink *link, int w, int h, int alig
     frame->sample_aspect_ratio = link->sample_aspect_ratio;
     frame->colorspace  = link->colorspace;
     frame->color_range = link->color_range;
+    frame->alpha_mode  = link->alpha_mode;
 
     return frame;
 }