handle malloc failures in ff_get_wav_header

ff_get_wav_header is reading data from a WAVE file and then uses it
(without validation) to malloc a buffer. It then proceeded to read
data into the buffer, without verifying that the allocation succeeded.

To address this, change ff_get_wav_header to return an error if
allocation failed, and adapted all calling code to handle that error.

Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

9 files changed, 36 insertions, 10 deletions

diff --git a/libavformat/asfdec.c b/libavformat/asfdec.c
index 57e8fb6df4..30828ad6ff 100644
--- a/libavformat/asfdec.c
+++ b/libavformat/asfdec.c
@@ -288,7 +288,9 @@ static int asf_read_stream_properties(AVFormatContext *s, int64_t size)
 
     st->codec->codec_type = type;
     if (type == AVMEDIA_TYPE_AUDIO) {
-        ff_get_wav_header(pb, st->codec, type_specific_size);
+        int ret = ff_get_wav_header(pb, st->codec, type_specific_size);
+        if (ret < 0)
+            return ret;
         if (is_dvr_ms_audio) {
             // codec_id and codec_tag are unreliable in dvr_ms
             // files. Set them later by probing stream.
diff --git a/libavformat/avidec.c b/libavformat/avidec.c
index ae8d32045f..ec3aa9335a 100644
--- a/libavformat/avidec.c
+++ b/libavformat/avidec.c
@@ -345,6 +345,7 @@ static int avi_read_header(AVFormatContext *s, AVFormatParameters *ap)
     int avih_width=0, avih_height=0;
     int amv_file_format=0;
     uint64_t list_end = 0;
+    int ret;
 
     avi->stream_index= -1;
 
@@ -623,7 +624,9 @@ static int avi_read_header(AVFormatContext *s, AVFormatParameters *ap)
 //                    avio_skip(pb, size - 5 * 4);
                     break;
                 case AVMEDIA_TYPE_AUDIO:
-                    ff_get_wav_header(pb, st->codec, size);
+                    ret = ff_get_wav_header(pb, st->codec, size);
+                    if (ret < 0)
+                        return ret;
                     ast->dshow_block_align= st->codec->block_align;
                     if(ast->sample_size && st->codec->block_align && ast->sample_size != st->codec->block_align){
                         av_log(s, AV_LOG_WARNING, "sample size (%d) != block align (%d)\n", ast->sample_size, st->codec->block_align);
diff --git a/libavformat/dxa.c b/libavformat/dxa.c
index 7fec26fd9e..1e1d50581c 100644
--- a/libavformat/dxa.c
+++ b/libavformat/dxa.c
@@ -60,6 +60,7 @@ static int dxa_read_header(AVFormatContext *s, AVFormatParameters *ap)
     int w, h;
     int num, den;
     int flags;
+    int ret;
 
     tag = avio_rl32(pb);
     if (tag != MKTAG('D', 'E', 'X', 'A'))
@@ -102,7 +103,9 @@ static int dxa_read_header(AVFormatContext *s, AVFormatParameters *ap)
         ast = av_new_stream(s, 0);
         if (!ast)
             return -1;
-        ff_get_wav_header(pb, ast->codec, fsize);
+        ret = ff_get_wav_header(pb, ast->codec, fsize);
+        if (ret < 0)
+            return ret;
         // find 'data' chunk
         while(avio_tell(pb) < c->vidpos && !pb->eof_reached){
             tag = avio_rl32(pb);
diff --git a/libavformat/matroskadec.c b/libavformat/matroskadec.c
index d1567e7cbe..19c7ca6384 100644
--- a/libavformat/matroskadec.c
+++ b/libavformat/matroskadec.c
@@ -1329,9 +1329,12 @@ static int matroska_read_header(AVFormatContext *s, AVFormatParameters *ap)
         } else if (!strcmp(track->codec_id, "A_MS/ACM")
                    && track->codec_priv.size >= 14
                    && track->codec_priv.data != NULL) {
+            int ret;
             ffio_init_context(&b, track->codec_priv.data, track->codec_priv.size,
                           AVIO_RDONLY, NULL, NULL, NULL, NULL);
-            ff_get_wav_header(&b, st->codec, track->codec_priv.size);
+            ret = ff_get_wav_header(&b, st->codec, track->codec_priv.size);
+            if (ret < 0)
+                return ret;
             codec_id = st->codec->codec_id;
             extradata_offset = FFMIN(track->codec_priv.size, 18);
         } else if (!strcmp(track->codec_id, "V_QUICKTIME")
diff --git a/libavformat/riff.c b/libavformat/riff.c
index 792c206678..c73152d40f 100644
--- a/libavformat/riff.c
+++ b/libavformat/riff.c
@@ -480,7 +480,7 @@ void ff_put_bmp_header(AVIOContext *pb, AVCodecContext *enc, const AVCodecTag *t
  * WAVEFORMATEX adds 'WORD  cbSize' and basically makes itself
  * an openended structure.
  */
-void ff_get_wav_header(AVIOContext *pb, AVCodecContext *codec, int size)
+int ff_get_wav_header(AVIOContext *pb, AVCodecContext *codec, int size)
 {
     int id;
 
@@ -510,6 +510,8 @@ void ff_get_wav_header(AVIOContext *pb, AVCodecContext *codec, int size)
         codec->extradata_size = cbSize;
         if (cbSize > 0) {
             codec->extradata = av_mallocz(codec->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
+            if (!codec->extradata)
+                return AVERROR(ENOMEM);
             avio_read(pb, codec->extradata, codec->extradata_size);
             size -= cbSize;
         }
@@ -524,6 +526,8 @@ void ff_get_wav_header(AVIOContext *pb, AVCodecContext *codec, int size)
         codec->channels    = 0;
         codec->sample_rate = 0;
     }
+
+    return 0;
 }
 
 
diff --git a/libavformat/riff.h b/libavformat/riff.h
index 174301264f..a2fa8b7b1c 100644
--- a/libavformat/riff.h
+++ b/libavformat/riff.h
@@ -45,7 +45,7 @@ int ff_get_bmp_header(AVIOContext *pb, AVStream *st);
 void ff_put_bmp_header(AVIOContext *pb, AVCodecContext *enc, const AVCodecTag *tags, int for_asf);
 int ff_put_wav_header(AVIOContext *pb, AVCodecContext *enc);
 enum CodecID ff_wav_codec_get_id(unsigned int tag, int bps);
-void ff_get_wav_header(AVIOContext *pb, AVCodecContext *codec, int size);
+int ff_get_wav_header(AVIOContext *pb, AVCodecContext *codec, int size);
 
 extern const AVCodecTag ff_codec_bmp_tags[];
 extern const AVCodecTag ff_codec_wav_tags[];
diff --git a/libavformat/wav.c b/libavformat/wav.c
index 47685d466a..21374e8b93 100644
--- a/libavformat/wav.c
+++ b/libavformat/wav.c
@@ -196,6 +196,7 @@ static int wav_read_header(AVFormatContext *s,
     AVIOContext *pb = s->pb;
     AVStream *st;
     WAVContext *wav = s->priv_data;
+    int ret;
 
     /* check RIFF header */
     tag = avio_rl32(pb);
@@ -228,7 +229,9 @@ static int wav_read_header(AVFormatContext *s,
     if (!st)
         return AVERROR(ENOMEM);
 
-    ff_get_wav_header(pb, st->codec, size);
+    ret = ff_get_wav_header(pb, st->codec, size);
+    if (ret < 0)
+        return ret;
     st->need_parsing = AVSTREAM_PARSE_FULL;
 
     av_set_pts_info(st, 64, 1, st->codec->sample_rate);
@@ -384,6 +387,7 @@ static int w64_read_header(AVFormatContext *s, AVFormatParameters *ap)
     WAVContext    *wav = s->priv_data;
     AVStream *st;
     uint8_t guid[16];
+    int ret;
 
     avio_read(pb, guid, 16);
     if (memcmp(guid, guid_riff, 16))
@@ -409,7 +413,9 @@ static int w64_read_header(AVFormatContext *s, AVFormatParameters *ap)
         return AVERROR(ENOMEM);
 
     /* subtract chunk header size - normal wav file doesn't count it */
-    ff_get_wav_header(pb, st->codec, size - 24);
+    ret = ff_get_wav_header(pb, st->codec, size - 24);
+    if (ret < 0)
+        return ret;
     avio_skip(pb, FFALIGN(size, INT64_C(8)) - size);
 
     st->need_parsing = AVSTREAM_PARSE_FULL;
diff --git a/libavformat/wtv.c b/libavformat/wtv.c
index 8260fe19b2..0a18c8e517 100644
--- a/libavformat/wtv.c
+++ b/libavformat/wtv.c
@@ -675,7 +675,9 @@ static AVStream * parse_media_type(AVFormatContext *s, AVStream *st, int sid,
         if (!st)
             return NULL;
         if (!ff_guidcmp(formattype, format_waveformatex)) {
-            ff_get_wav_header(pb, st->codec, size);
+            int ret = ff_get_wav_header(pb, st->codec, size);
+            if (ret < 0)
+                return NULL;
         } else {
             if (ff_guidcmp(formattype, format_none))
                 av_log(s, AV_LOG_WARNING, "unknown formattype:"PRI_GUID"\n", ARG_GUID(formattype));
diff --git a/libavformat/xwma.c b/libavformat/xwma.c
index 4238054476..ba2ee05395 100644
--- a/libavformat/xwma.c
+++ b/libavformat/xwma.c
@@ -40,6 +40,7 @@ static int xwma_probe(AVProbeData *p)
 static int xwma_read_header(AVFormatContext *s, AVFormatParameters *ap)
 {
     int64_t size, av_uninit(data_size);
+    int ret;
     uint32_t dpds_table_size = 0;
     uint32_t *dpds_table = 0;
     unsigned int tag;
@@ -70,7 +71,9 @@ static int xwma_read_header(AVFormatContext *s, AVFormatParameters *ap)
     if (!st)
         return AVERROR(ENOMEM);
 
-    ff_get_wav_header(pb, st->codec, size);
+    ret = ff_get_wav_header(pb, st->codec, size);
+    if (ret < 0)
+        return ret;
     st->need_parsing = AVSTREAM_PARSE_NONE;
 
     /* All xWMA files I have seen contained WMAv2 data. If there are files