avfilter/af_afir: reduce output gain with default parameters

It was unreasonably high. Also change scaling to reduce
rare quantization errors.

3 files changed, 28 insertions, 20 deletions

diff --git a/libavfilter/af_afir.c b/libavfilter/af_afir.c
index cb1332841f..83b9a1ba02 100644
--- a/libavfilter/af_afir.c
+++ b/libavfilter/af_afir.c
@@ -160,6 +160,8 @@ static int init_segment(AVFilterContext *ctx, AudioFIRSegment *seg,
 {
     AudioFIRContext *s = ctx->priv;
     const size_t cpu_align = av_cpu_max_align();
+    union { double d; float f; } cscale, scale, iscale;
+    enum AVTXType tx_type;
     int ret;
 
     seg->tx  = av_calloc(ctx->inputs[0]->ch_layout.nb_channels, sizeof(*seg->tx));
@@ -167,7 +169,7 @@ static int init_segment(AVFilterContext *ctx, AudioFIRSegment *seg,
     if (!seg->tx || !seg->itx)
         return AVERROR(ENOMEM);
 
-    seg->fft_length    = part_size * 2 + 1;
+    seg->fft_length    = part_size * 2 + 2;
     seg->part_size     = part_size;
     seg->block_size    = FFALIGN(seg->fft_length, cpu_align);
     seg->coeff_size    = FFALIGN(seg->part_size + 1, cpu_align);
@@ -180,22 +182,27 @@ static int init_segment(AVFilterContext *ctx, AudioFIRSegment *seg,
     if (!seg->part_index || !seg->output_offset)
         return AVERROR(ENOMEM);
 
-    for (int ch = 0; ch < ctx->inputs[0]->ch_layout.nb_channels && part_size >= 1; ch++) {
-        union { double d; float f; } scale, iscale;
-        enum AVTXType tx_type;
+    switch (s->format) {
+    case AV_SAMPLE_FMT_FLTP:
+        cscale.f = 1.f;
+        scale.f  = 1.f / sqrtf(2.f * part_size);
+        iscale.f = 1.f / sqrtf(2.f * part_size);
+        tx_type  = AV_TX_FLOAT_RDFT;
+        break;
+    case AV_SAMPLE_FMT_DBLP:
+        cscale.d = 1.0;
+        scale.d  = 1.0 / sqrt(2.0 * part_size);
+        iscale.d = 1.0 / sqrt(2.0 * part_size);
+        tx_type  = AV_TX_DOUBLE_RDFT;
+        break;
+    }
 
-        switch (s->format) {
-        case AV_SAMPLE_FMT_FLTP:
-            scale.f  = 1.f;
-            iscale.f = 1.f / part_size;
-            tx_type  = AV_TX_FLOAT_RDFT;
-            break;
-        case AV_SAMPLE_FMT_DBLP:
-            scale.d  = 1.0;
-            iscale.d = 1.0 / part_size;
-            tx_type  = AV_TX_DOUBLE_RDFT;
-            break;
-        }
+    ret = av_tx_init(&seg->ctx, &seg->ctx_fn, tx_type,
+                     0, 2 * part_size, &cscale,  0);
+    if (ret < 0)
+        return ret;
+
+    for (int ch = 0; ch < ctx->inputs[0]->ch_layout.nb_channels && part_size >= 1; ch++) {
         ret = av_tx_init(&seg->tx[ch],  &seg->tx_fn,  tx_type,
                          0, 2 * part_size, &scale,  0);
         if (ret < 0)
@@ -224,6 +231,8 @@ static void uninit_segment(AVFilterContext *ctx, AudioFIRSegment *seg)
 {
     AudioFIRContext *s = ctx->priv;
 
+    av_tx_uninit(&seg->ctx);
+
     if (seg->tx) {
         for (int ch = 0; ch < s->nb_channels; ch++) {
             av_tx_uninit(&seg->tx[ch]);
diff --git a/libavfilter/af_afir.h b/libavfilter/af_afir.h
index e82fc58589..6a071eddf7 100644
--- a/libavfilter/af_afir.h
+++ b/libavfilter/af_afir.h
@@ -49,8 +49,8 @@ typedef struct AudioFIRSegment {
     AVFrame *input;
     AVFrame *output;
 
-    AVTXContext **tx, **itx;
-    av_tx_fn tx_fn, itx_fn;
+    AVTXContext *ctx, **tx, **itx;
+    av_tx_fn ctx_fn, tx_fn, itx_fn;
 } AudioFIRSegment;
 
 typedef struct AudioFIRContext {
diff --git a/libavfilter/afir_template.c b/libavfilter/afir_template.c
index 45055bba87..712f4b7417 100644
--- a/libavfilter/afir_template.c
+++ b/libavfilter/afir_template.c
@@ -159,7 +159,7 @@ static void fn(convert_channels)(AVFilterContext *ctx, AudioFIRContext *s)
                 memset(blockin, 0, sizeof(*blockin) * seg->fft_length);
                 memcpy(blockin, time + toffset, size * sizeof(*blockin));
 
-                seg->tx_fn(seg->tx[0], blockout, blockin, sizeof(ftype));
+                seg->ctx_fn(seg->ctx, blockout, blockin, sizeof(ftype));
 
                 for (int n = 0; n < seg->part_size + 1; n++) {
                     coeff[coffset + n].re = blockout[2 * n];
@@ -316,7 +316,6 @@ static int fn(fir_quantum)(AVFilterContext *ctx, AVFrame *out, int ch, int offse
 #else
             s->afirdsp.dcmul_add(sumin, blockout, (const ftype *)coeff, part_size);
 #endif
-
             if (j == 0)
                 j = seg->nb_partitions;
             j--;