resample: replace VLA with malloc/free

Originally committed as revision 24142 to svn://svn.ffmpeg.org/ffmpeg/trunk

1 files changed, 13 insertions, 3 deletions

diff --git a/libavcodec/resample2.c b/libavcodec/resample2.c
index 05821eddb8..b940059d84 100644
--- a/libavcodec/resample2.c
+++ b/libavcodec/resample2.c
@@ -94,12 +94,17 @@ static double bessel(double x){
  * @param factor resampling factor
  * @param scale wanted sum of coefficients for each filter
  * @param type 0->cubic, 1->blackman nuttall windowed sinc, 2..16->kaiser windowed sinc beta=2..16
+ * @return 0 on success, negative on error
  */
-static void build_filter(FELEM *filter, double factor, int tap_count, int phase_count, int scale, int type){
+static int build_filter(FELEM *filter, double factor, int tap_count, int phase_count, int scale, int type){
     int ph, i;
-    double x, y, w, tab[tap_count];
+    double x, y, w;
+    double *tab = av_malloc(tap_count * sizeof(*tab));
     const int center= (tap_count-1)/2;
 
+    if (!tab)
+        return AVERROR(ENOMEM);
+
     /* if upsampling, only need to interpolate, no filter */
     if (factor > 1.0)
         factor = 1.0;
@@ -176,6 +181,9 @@ static void build_filter(FELEM *filter, double factor, int tap_count, int phase_
         }
     }
 #endif
+
+    av_free(tab);
+    return 0;
 }
 
 AVResampleContext *av_resample_init(int out_rate, int in_rate, int filter_size, int phase_shift, int linear, double cutoff){
@@ -194,7 +202,8 @@ AVResampleContext *av_resample_init(int out_rate, int in_rate, int filter_size,
     c->filter_bank= av_mallocz(c->filter_length*(phase_count+1)*sizeof(FELEM));
     if (!c->filter_bank)
         goto error;
-    build_filter(c->filter_bank, factor, c->filter_length, phase_count, 1<<FILTER_SHIFT, WINDOW_TYPE);
+    if (build_filter(c->filter_bank, factor, c->filter_length, phase_count, 1<<FILTER_SHIFT, WINDOW_TYPE))
+        goto error;
     memcpy(&c->filter_bank[c->filter_length*phase_count+1], c->filter_bank, (c->filter_length-1)*sizeof(FELEM));
     c->filter_bank[c->filter_length*phase_count]= c->filter_bank[c->filter_length - 1];
 
@@ -204,6 +213,7 @@ AVResampleContext *av_resample_init(int out_rate, int in_rate, int filter_size,
 
     return c;
 error:
+    av_free(c->filter_bank);
     av_free(c);
     return NULL;
 }