lavr: mix: reduce the mixing matrix when possible

If the matrix results in an output channel not getting a contribution
from any input channel and the corresponding input channel does not
contribute to any outputs, we can skip the channel during mixing and
silence it after mixing.

If the matrix results in an input channel not contributing to any output
channels and it is not in the output mix, or if the input channel only
contributes fully to the same output channel, we can skip the channel
during mixing.

If the matrix results in an output channel only getting full
contribution from the corresponding input channel and that input channel
does not contribute to any other output channels, we can skip the
channel during mixing.

1 files changed, 186 insertions, 30 deletions

diff --git a/libavresample/audio_mix.c b/libavresample/audio_mix.c
index 2b3d9f1f7a..8df8a0223b 100644
--- a/libavresample/audio_mix.c
+++ b/libavresample/audio_mix.c
@@ -47,6 +47,11 @@ struct AudioMix {
     mix_func *mix;
     mix_func *mix_generic;
 
+    int in_matrix_channels;
+    int out_matrix_channels;
+    int output_zero[AVRESAMPLE_MAX_CHANNELS];
+    int input_skip[AVRESAMPLE_MAX_CHANNELS];
+    int output_skip[AVRESAMPLE_MAX_CHANNELS];
     int16_t *matrix_q8[AVRESAMPLE_MAX_CHANNELS];
     int32_t *matrix_q15[AVRESAMPLE_MAX_CHANNELS];
     float   *matrix_flt[AVRESAMPLE_MAX_CHANNELS];
@@ -59,8 +64,8 @@ void ff_audio_mix_set_func(AudioMix *am, enum AVSampleFormat fmt,
                            const char *descr, void *mix_func)
 {
     if (fmt == am->fmt && coeff_type == am->coeff_type &&
-        ( in_channels ==  am->in_channels ||  in_channels == 0) &&
-        (out_channels == am->out_channels || out_channels == 0)) {
+        ( in_channels ==  am->in_matrix_channels ||  in_channels == 0) &&
+        (out_channels == am->out_matrix_channels || out_channels == 0)) {
         char chan_str[16];
         am->mix           = mix_func;
         am->func_descr    = descr;
@@ -278,6 +283,12 @@ static void mix_2_to_6_fltp_flt_c(float **samples, float **matrix, int len,
 
 static int mix_function_init(AudioMix *am)
 {
+    /* no need to set a mix function when we're skipping mixing */
+    if (!am->in_matrix_channels || !am->out_matrix_channels) {
+        am->func_descr = "n/a";
+        return 0;
+    }
+
     /* any-to-any C versions */
 
     ff_audio_mix_set_func(am, AV_SAMPLE_FMT_FLTP, AV_MIX_COEFF_TYPE_FLT,
@@ -379,25 +390,34 @@ AudioMix *ff_audio_mix_alloc(AVAudioResampleContext *avr)
             goto error;
         }
 
+        ret = ff_audio_mix_set_matrix(am, matrix_dbl, avr->in_channels);
+        if (ret < 0) {
+            av_log(avr, AV_LOG_ERROR, "error setting mix matrix\n");
+            av_free(matrix_dbl);
+            goto error;
+        }
+
         av_get_channel_layout_string(in_layout_name, sizeof(in_layout_name),
                                      avr->in_channels, avr->in_channel_layout);
         av_get_channel_layout_string(out_layout_name, sizeof(out_layout_name),
                                      avr->out_channels, avr->out_channel_layout);
         av_log(avr, AV_LOG_DEBUG, "audio_mix: %s to %s\n",
                in_layout_name, out_layout_name);
+        av_log(avr, AV_LOG_DEBUG, "matrix size: %d x %d\n",
+               am->in_matrix_channels, am->out_matrix_channels);
         for (i = 0; i < avr->out_channels; i++) {
             for (j = 0; j < avr->in_channels; j++) {
-                av_log(avr, AV_LOG_DEBUG, "  %0.3f ",
-                       matrix_dbl[i * avr->in_channels + j]);
+                if (am->output_zero[i])
+                    av_log(avr, AV_LOG_DEBUG, "  (ZERO)");
+                else if (am->input_skip[j] || am->output_skip[i])
+                    av_log(avr, AV_LOG_DEBUG, "  (SKIP)");
+                else
+                    av_log(avr, AV_LOG_DEBUG, "  %0.3f ",
+                           matrix_dbl[i * avr->in_channels + j]);
             }
             av_log(avr, AV_LOG_DEBUG, "\n");
         }
 
-        ret = ff_audio_mix_set_matrix(am, matrix_dbl, avr->in_channels);
-        if (ret < 0) {
-            av_free(matrix_dbl);
-            goto error;
-        }
         av_free(matrix_dbl);
     }
 
@@ -435,6 +455,7 @@ int ff_audio_mix(AudioMix *am, AudioData *src)
 {
     int use_generic = 1;
     int len = src->nb_samples;
+    int i, j;
 
     /* determine whether to use the optimized function based on pointer and
        samples alignment in both the input and output */
@@ -450,11 +471,35 @@ int ff_audio_mix(AudioMix *am, AudioData *src)
             src->nb_samples, am->in_channels, am->out_channels,
             use_generic ? am->func_descr_generic : am->func_descr);
 
-    if (use_generic)
-        am->mix_generic(src->data, am->matrix, len, am->out_channels,
-                        am->in_channels);
-    else
-        am->mix(src->data, am->matrix, len, am->out_channels, am->in_channels);
+    if (am->in_matrix_channels && am->out_matrix_channels) {
+        uint8_t **data;
+        uint8_t *data0[AVRESAMPLE_MAX_CHANNELS];
+
+        if (am->out_matrix_channels < am->out_channels ||
+             am->in_matrix_channels <  am->in_channels) {
+            for (i = 0, j = 0; i < FFMAX(am->in_channels, am->out_channels); i++) {
+                if (am->input_skip[i] || am->output_skip[i] || am->output_zero[i])
+                    continue;
+                data0[j++] = src->data[i];
+            }
+            data = data0;
+        } else {
+            data = src->data;
+        }
+
+        if (use_generic)
+            am->mix_generic(data, am->matrix, len, am->out_matrix_channels,
+                            am->in_matrix_channels);
+        else
+            am->mix(data, am->matrix, len, am->out_matrix_channels,
+                    am->in_matrix_channels);
+    }
+
+    if (am->out_matrix_channels < am->out_channels) {
+        for (i = 0; i < am->out_channels; i++)
+            if (am->output_zero[i])
+                av_samples_set_silence(&src->data[i], 0, len, 1, am->fmt);
+    }
 
     ff_audio_data_set_channels(src, am->out_channels);
 
@@ -463,7 +508,7 @@ int ff_audio_mix(AudioMix *am, AudioData *src)
 
 int ff_audio_mix_get_matrix(AudioMix *am, double *matrix, int stride)
 {
-    int i, o;
+    int i, o, i0, o0;
 
     if ( am->in_channels <= 0 ||  am->in_channels > AVRESAMPLE_MAX_CHANNELS ||
         am->out_channels <= 0 || am->out_channels > AVRESAMPLE_MAX_CHANNELS) {
@@ -476,9 +521,19 @@ int ff_audio_mix_get_matrix(AudioMix *am, double *matrix, int stride)
         av_log(am->avr, AV_LOG_ERROR, "matrix is not set\n");               \
         return AVERROR(EINVAL);                                             \
     }                                                                       \
-    for (o = 0; o < am->out_channels; o++)                                  \
-        for (i = 0; i < am->in_channels; i++)                               \
-            matrix[o * stride + i] = am->matrix_ ## suffix[o][i] * (scale);
+    for (o = 0, o0 = 0; o < am->out_channels; o++) {                        \
+        for (i = 0, i0 = 0; i < am->in_channels; i++) {                     \
+            if (am->input_skip[i] || am->output_zero[o])                    \
+                matrix[o * stride + i] = 0.0;                               \
+            else                                                            \
+                matrix[o * stride + i] = am->matrix_ ## suffix[o0][i0] *    \
+                                         (scale);                           \
+            if (!am->input_skip[i])                                         \
+                i0++;                                                       \
+        }                                                                   \
+        if (!am->output_zero[o])                                            \
+            o0++;                                                           \
+    }
 
     switch (am->coeff_type) {
     case AV_MIX_COEFF_TYPE_Q8:
@@ -500,7 +555,7 @@ int ff_audio_mix_get_matrix(AudioMix *am, double *matrix, int stride)
 
 int ff_audio_mix_set_matrix(AudioMix *am, const double *matrix, int stride)
 {
-    int i, o;
+    int i, o, i0, o0;
 
     if ( am->in_channels <= 0 ||  am->in_channels > AVRESAMPLE_MAX_CHANNELS ||
         am->out_channels <= 0 || am->out_channels > AVRESAMPLE_MAX_CHANNELS) {
@@ -513,19 +568,123 @@ int ff_audio_mix_set_matrix(AudioMix *am, const double *matrix, int stride)
         am->matrix = NULL;
     }
 
+    memset(am->output_zero, 0, sizeof(am->output_zero));
+    memset(am->input_skip,  0, sizeof(am->input_skip));
+    memset(am->output_skip, 0, sizeof(am->output_zero));
+    am->in_matrix_channels  = am->in_channels;
+    am->out_matrix_channels = am->out_channels;
+
+    /* exclude output channels if they can be zeroed instead of mixed */
+    for (o = 0; o < am->out_channels; o++) {
+        int zero = 1;
+
+        /* check if the output is always silent */
+        for (i = 0; i < am->in_channels; i++) {
+            if (matrix[o * stride + i] != 0.0) {
+                zero = 0;
+                break;
+            }
+        }
+        /* check if the corresponding input channel makes a contribution to
+           any output channel */
+        if (o < am->in_channels) {
+            for (i = 0; i < am->out_channels; i++) {
+                if (matrix[i * stride + o] != 0.0) {
+                    zero = 0;
+                    break;
+                }
+            }
+        }
+        if (zero) {
+            am->output_zero[o] = 1;
+            am->out_matrix_channels--;
+        }
+    }
+    if (am->out_matrix_channels == 0) {
+        am->in_matrix_channels = 0;
+        return 0;
+    }
+
+    /* skip input channels that contribute fully only to the corresponding
+       output channel */
+    for (i = 0; i < FFMIN(am->in_channels, am->out_channels); i++) {
+        int skip = 1;
+
+        for (o = 0; o < am->out_channels; o++) {
+            if ((o != i && matrix[o * stride + i] != 0.0) ||
+                (o == i && matrix[o * stride + i] != 1.0)) {
+                skip = 0;
+                break;
+            }
+        }
+        if (skip) {
+            am->input_skip[i] = 1;
+            am->in_matrix_channels--;
+        }
+    }
+    /* skip input channels that do not contribute to any output channel */
+    for (; i < am->in_channels; i++) {
+        int contrib = 0;
+
+        for (o = 0; o < am->out_channels; o++) {
+            if (matrix[o * stride + i] != 0.0) {
+                contrib = 1;
+                break;
+            }
+        }
+        if (!contrib) {
+            am->input_skip[i] = 1;
+            am->in_matrix_channels--;
+        }
+    }
+    if (am->in_matrix_channels == 0) {
+        am->out_matrix_channels = 0;
+        return 0;
+    }
+
+    /* skip output channels that only get full contribution from the
+       corresponding input channel */
+    for (o = 0; o < FFMIN(am->in_channels, am->out_channels); o++) {
+        int skip = 1;
+
+        for (i = 0; i < am->in_channels; i++) {
+            if ((o != i && matrix[o * stride + i] != 0.0) ||
+                (o == i && matrix[o * stride + i] != 1.0)) {
+                skip = 0;
+                break;
+            }
+        }
+        if (skip) {
+            am->output_skip[o] = 1;
+            am->out_matrix_channels--;
+        }
+    }
+    if (am->out_matrix_channels == 0) {
+        am->in_matrix_channels = 0;
+        return 0;
+    }
+
 #define CONVERT_MATRIX(type, expr)                                          \
-    am->matrix_## type[0] = av_mallocz(am->out_channels * am->in_channels * \
+    am->matrix_## type[0] = av_mallocz(am->out_matrix_channels *            \
+                                       am->in_matrix_channels  *            \
                                        sizeof(*am->matrix_## type[0]));     \
     if (!am->matrix_## type[0])                                             \
         return AVERROR(ENOMEM);                                             \
-    for (o = 0; o < am->out_channels; o++) {                                \
-        if (o > 0)                                                          \
-            am->matrix_## type[o] = am->matrix_## type[o - 1] +             \
-                                    am->in_channels;                        \
-        for (i = 0; i < am->in_channels; i++) {                             \
-            double v = matrix[o * stride + i];                              \
-            am->matrix_## type[o][i] = expr;                                \
+    for (o = 0, o0 = 0; o < am->out_channels; o++) {                        \
+        if (am->output_zero[o] || am->output_skip[o])                       \
+            continue;                                                       \
+        if (o0 > 0)                                                         \
+            am->matrix_## type[o0] = am->matrix_## type[o0 - 1] +           \
+                                     am->in_matrix_channels;                \
+        for (i = 0, i0 = 0; i < am->in_channels; i++) {                     \
+            double v;                                                       \
+            if (am->input_skip[i])                                          \
+                continue;                                                   \
+            v = matrix[o * stride + i];                                     \
+            am->matrix_## type[o0][i0] = expr;                              \
+            i0++;                                                           \
         }                                                                   \
+        o0++;                                                               \
     }                                                                       \
     am->matrix = (void **)am->matrix_## type;
 
@@ -544,8 +703,5 @@ int ff_audio_mix_set_matrix(AudioMix *am, const double *matrix, int stride)
         return AVERROR(EINVAL);
     }
 
-    /* TODO: detect situations where we can just swap around pointers
-             instead of doing matrix multiplications with 0.0 and 1.0 */
-
     return 0;
 }