Remove VLA in ff_kbd_window_init, limit window size to 1024

Originally committed as revision 23755 to svn://svn.ffmpeg.org/ffmpeg/trunk

2 files changed, 9 insertions, 2 deletions

diff --git a/libavcodec/fft.h b/libavcodec/fft.h
index 1f5b2e86da..10ca1b140e 100644
--- a/libavcodec/fft.h
+++ b/libavcodec/fft.h
@@ -151,10 +151,15 @@ static inline void ff_mdct_calc(FFTContext *s, FFTSample *output,
 }
 
 /**
+ * Maximum window size for ff_kbd_window_init.
+ */
+#define FF_KBD_WINDOW_MAX 1024
+
+/**
  * Generate a Kaiser-Bessel Derived Window.
  * @param   window  pointer to half window
  * @param   alpha   determines window shape
- * @param   n       size of half window
+ * @param   n       size of half window, max FF_KBD_WINDOW_MAX
  */
 void ff_kbd_window_init(float *window, float alpha, int n);
 
diff --git a/libavcodec/mdct.c b/libavcodec/mdct.c
index 69e1bbff22..c511188d22 100644
--- a/libavcodec/mdct.c
+++ b/libavcodec/mdct.c
@@ -36,9 +36,11 @@ av_cold void ff_kbd_window_init(float *window, float alpha, int n)
 {
    int i, j;
    double sum = 0.0, bessel, tmp;
-   double local_window[n];
+   double local_window[FF_KBD_WINDOW_MAX];
    double alpha2 = (alpha * M_PI / n) * (alpha * M_PI / n);
 
+   assert(n <= FF_KBD_WINDOW_MAX);
+
    for (i = 0; i < n; i++) {
        tmp = i * (n - i) * alpha2;
        bessel = 1.0;