lavfi: support unknown channel layouts.

1 files changed, 73 insertions, 9 deletions

diff --git a/libavfilter/formats.c b/libavfilter/formats.c
index ea9a184eaa..296e4f4b7c 100644
--- a/libavfilter/formats.c
+++ b/libavfilter/formats.c
@@ -19,6 +19,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#include "libavutil/avassert.h"
 #include "libavutil/channel_layout.h"
 #include "libavutil/common.h"
 #include "libavutil/eval.h"
@@ -28,6 +29,8 @@
 #include "internal.h"
 #include "formats.h"
 
+#define KNOWN(l) (!FF_LAYOUT2COUNT(l)) /* for readability */
+
 /**
  * Add all refs from a to ret and destroy a.
  */
@@ -136,21 +139,77 @@ AVFilterChannelLayouts *ff_merge_channel_layouts(AVFilterChannelLayouts *a,
                                                  AVFilterChannelLayouts *b)
 {
     AVFilterChannelLayouts *ret = NULL;
+    unsigned a_all = a->all_layouts + a->all_counts;
+    unsigned b_all = b->all_layouts + b->all_counts;
+    int ret_max, ret_nb = 0, i, j, round;
 
     if (a == b) return a;
 
-    if (a->nb_channel_layouts && b->nb_channel_layouts) {
-        MERGE_FORMATS(ret, a, b, channel_layouts, nb_channel_layouts,
-                      AVFilterChannelLayouts, fail);
-    } else if (a->nb_channel_layouts) {
-        MERGE_REF(a, b, channel_layouts, AVFilterChannelLayouts, fail);
-        ret = a;
-    } else {
+    /* Put the most generic set in a, to avoid doing everything twice */
+    if (a_all < b_all) {
+        FFSWAP(AVFilterChannelLayouts *, a, b);
+        FFSWAP(unsigned, a_all, b_all);
+    }
+    if (a_all) {
+        if (a_all == 1 && !b_all) {
+            /* keep only known layouts in b; works also for b_all = 1 */
+            for (i = j = 0; i < b->nb_channel_layouts; i++)
+                if (KNOWN(b->channel_layouts[i]))
+                    b->channel_layouts[j++] = b->channel_layouts[i];
+            b->nb_channel_layouts = j;
+        }
         MERGE_REF(b, a, channel_layouts, AVFilterChannelLayouts, fail);
-        ret = b;
+        return b;
     }
 
+    ret_max = a->nb_channel_layouts + b->nb_channel_layouts;
+    if (!(ret = av_mallocz(sizeof(*ret))) ||
+        !(ret->channel_layouts = av_malloc(sizeof(*ret->channel_layouts) *
+                                           ret_max)))
+        goto fail;
+
+    /* a[known] intersect b[known] */
+    for (i = 0; i < a->nb_channel_layouts; i++) {
+        if (!KNOWN(a->channel_layouts[i]))
+            continue;
+        for (j = 0; j < b->nb_channel_layouts; j++) {
+            if (a->channel_layouts[i] == b->channel_layouts[j]) {
+                ret->channel_layouts[ret_nb++] = a->channel_layouts[i];
+                a->channel_layouts[i] = b->channel_layouts[j] = 0;
+            }
+        }
+    }
+    /* 1st round: a[known] intersect b[generic]
+       2nd round: a[generic] intersect b[known] */
+    for (round = 0; round < 2; round++) {
+        for (i = 0; i < a->nb_channel_layouts; i++) {
+            uint64_t fmt = a->channel_layouts[i], bfmt;
+            if (!fmt || !KNOWN(fmt))
+                continue;
+            bfmt = FF_COUNT2LAYOUT(av_get_channel_layout_nb_channels(fmt));
+            for (j = 0; j < b->nb_channel_layouts; j++)
+                if (b->channel_layouts[j] == bfmt)
+                    ret->channel_layouts[ret_nb++] = a->channel_layouts[i];
+        }
+        /* 1st round: swap to prepare 2nd round; 2nd round: put it back */
+        FFSWAP(AVFilterChannelLayouts *, a, b);
+    }
+    /* a[generic] intersect b[generic] */
+    for (i = 0; i < a->nb_channel_layouts; i++) {
+        if (KNOWN(a->channel_layouts[i]))
+            continue;
+        for (j = 0; j < b->nb_channel_layouts; j++)
+            if (a->channel_layouts[i] == b->channel_layouts[j])
+                ret->channel_layouts[ret_nb++] = a->channel_layouts[i];
+    }
+
+    ret->nb_channel_layouts = ret_nb;
+    if (!ret->nb_channel_layouts)
+        goto fail;
+    MERGE_REF(ret, a, channel_layouts, AVFilterChannelLayouts, fail);
+    MERGE_REF(ret, b, channel_layouts, AVFilterChannelLayouts, fail);
     return ret;
+
 fail:
     if (ret) {
         av_freep(&ret->refs);
@@ -248,17 +307,19 @@ do {                                                        \
                                                             \
     (*f)->list = fmts;                                      \
     (*f)->list[(*f)->nb++] = fmt;                           \
-    return 0;                                               \
 } while (0)
 
 int ff_add_format(AVFilterFormats **avff, int64_t fmt)
 {
     ADD_FORMAT(avff, fmt, int, formats, format_count);
+    return 0;
 }
 
 int ff_add_channel_layout(AVFilterChannelLayouts **l, uint64_t channel_layout)
 {
+    av_assert1(!(*l && (*l)->all_layouts));
     ADD_FORMAT(l, channel_layout, uint64_t, channel_layouts, nb_channel_layouts);
+    return 0;
 }
 
 AVFilterFormats *ff_all_formats(enum AVMediaType type)
@@ -309,6 +370,9 @@ AVFilterFormats *ff_all_samplerates(void)
 AVFilterChannelLayouts *ff_all_channel_layouts(void)
 {
     AVFilterChannelLayouts *ret = av_mallocz(sizeof(*ret));
+    if (!ret)
+        return NULL;
+    ret->all_layouts = 1;
     return ret;
 }