lavf: Reset the entry count and allocation size variables on av_reallocp failures

When av_reallocp fails, the associated variables that keep track of
the number of elements in the array (and in some cases, the
separate number of allocated elements) need to be reset.

Not all of these might technically be needed, but it's better to
reset them if in doubt, to make sure variables don't end up
conflicting.

Signed-off-by: Martin Storsjö <martin@martin.st>

13 files changed, 47 insertions, 16 deletions

diff --git a/libavformat/avidec.c b/libavformat/avidec.c
index ea7ecab768..1212c6a09b 100644
--- a/libavformat/avidec.c
+++ b/libavformat/avidec.c
@@ -657,9 +657,10 @@ static int avi_read_header(AVFormatContext *s)
                         st->codec->extradata_size += 9;
                         if ((ret = av_reallocp(&st->codec->extradata,
                                                st->codec->extradata_size +
-                                               FF_INPUT_BUFFER_PADDING_SIZE)) < 0)
+                                               FF_INPUT_BUFFER_PADDING_SIZE)) < 0) {
+                            st->codec->extradata_size = 0;
                             return ret;
-                        else
+                        } else
                             memcpy(st->codec->extradata + st->codec->extradata_size - 9,
                                    "BottomUp", 9);
                     }
diff --git a/libavformat/avienc.c b/libavformat/avienc.c
index e6d9dae20c..66339af2b0 100644
--- a/libavformat/avienc.c
+++ b/libavformat/avienc.c
@@ -538,8 +538,11 @@ static int avi_write_packet(AVFormatContext *s, AVPacket *pkt)
         int cl = idx->entry / AVI_INDEX_CLUSTER_SIZE;
         int id = idx->entry % AVI_INDEX_CLUSTER_SIZE;
         if (idx->ents_allocated <= idx->entry) {
-            if ((err = av_reallocp(&idx->cluster, (cl + 1) * sizeof(*idx->cluster))) < 0)
+            if ((err = av_reallocp(&idx->cluster, (cl + 1) * sizeof(*idx->cluster))) < 0) {
+                idx->ents_allocated = 0;
+                idx->entry = 0;
                 return err;
+            }
             idx->cluster[cl] = av_malloc(AVI_INDEX_CLUSTER_SIZE*sizeof(AVIIentry));
             if (!idx->cluster[cl])
                 return -1;
diff --git a/libavformat/aviobuf.c b/libavformat/aviobuf.c
index 3f27d6976c..5064eb826b 100644
--- a/libavformat/aviobuf.c
+++ b/libavformat/aviobuf.c
@@ -880,8 +880,11 @@ static int dyn_buf_write(void *opaque, uint8_t *buf, int buf_size)
 
     if (new_allocated_size > d->allocated_size) {
         int err;
-        if ((err = av_reallocp(&d->buffer, new_allocated_size)) < 0)
+        if ((err = av_reallocp(&d->buffer, new_allocated_size)) < 0) {
+            d->allocated_size = 0;
+            d->size = 0;
             return err;
+        }
         d->allocated_size = new_allocated_size;
     }
     memcpy(d->buffer + d->pos, buf, buf_size);
diff --git a/libavformat/mmst.c b/libavformat/mmst.c
index 41d01c4fc6..a17b4c12a8 100644
--- a/libavformat/mmst.c
+++ b/libavformat/mmst.c
@@ -337,8 +337,10 @@ static MMSSCPacketType get_tcp_server_response(MMSTContext *mmst)
                 if(!mms->header_parsed) {
                     if ((err = av_reallocp(&mms->asf_header,
                                            mms->asf_header_size +
-                                           mms->remaining_in_len)) < 0)
+                                           mms->remaining_in_len)) < 0) {
+                        mms->asf_header_size = 0;
                         return err;
+                    }
                     memcpy(mms->asf_header + mms->asf_header_size,
                            mms->read_in_ptr, mms->remaining_in_len);
                     mms->asf_header_size += mms->remaining_in_len;
diff --git a/libavformat/mov.c b/libavformat/mov.c
index a84fae84ba..c3d857be4d 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -886,8 +886,10 @@ static int mov_read_extradata(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     size= (uint64_t)st->codec->extradata_size + atom.size + 8 + FF_INPUT_BUFFER_PADDING_SIZE;
     if (size > INT_MAX || (uint64_t)atom.size > INT_MAX)
         return AVERROR_INVALIDDATA;
-    if ((err = av_reallocp(&st->codec->extradata, size)) < 0)
+    if ((err = av_reallocp(&st->codec->extradata, size)) < 0) {
+        st->codec->extradata_size = 0;
         return err;
+    }
     buf = st->codec->extradata + st->codec->extradata_size;
     st->codec->extradata_size= size - FF_INPUT_BUFFER_PADDING_SIZE;
     AV_WB32(       buf    , atom.size + 8);
diff --git a/libavformat/oggparsetheora.c b/libavformat/oggparsetheora.c
index 94e9eba35c..25210ab450 100644
--- a/libavformat/oggparsetheora.c
+++ b/libavformat/oggparsetheora.c
@@ -124,8 +124,10 @@ theora_header (AVFormatContext * s, int idx)
     }
 
     if ((err = av_reallocp(&st->codec->extradata,
-                           cds + FF_INPUT_BUFFER_PADDING_SIZE)) < 0)
+                           cds + FF_INPUT_BUFFER_PADDING_SIZE)) < 0) {
+        st->codec->extradata_size = 0;
         return err;
+    }
     cdp = st->codec->extradata + st->codec->extradata_size;
     *cdp++ = os->psize >> 8;
     *cdp++ = os->psize & 0xff;
diff --git a/libavformat/oggparsevorbis.c b/libavformat/oggparsevorbis.c
index 1a9776ef15..6e106b0d2b 100644
--- a/libavformat/oggparsevorbis.c
+++ b/libavformat/oggparsevorbis.c
@@ -283,9 +283,12 @@ vorbis_header (AVFormatContext * s, int idx)
             }
         }
     } else {
-        int ret;
-        st->codec->extradata_size =
-            fixup_vorbis_headers(s, priv, &st->codec->extradata);
+        int ret = fixup_vorbis_headers(s, priv, &st->codec->extradata);
+        if (ret < 0) {
+            st->codec->extradata_size = 0;
+            return ret;
+        }
+        st->codec->extradata_size = ret;
         if ((ret = avpriv_vorbis_parse_extradata(st->codec, &priv->vp))) {
             av_freep(&st->codec->extradata);
             st->codec->extradata_size = 0;
diff --git a/libavformat/rdt.c b/libavformat/rdt.c
index d691ae904f..33b0eb827e 100644
--- a/libavformat/rdt.c
+++ b/libavformat/rdt.c
@@ -423,8 +423,10 @@ rdt_parse_sdp_line (AVFormatContext *s, int st_index,
                 if (first == -1) first = n;
                 if (rdt->nb_rmst < count) {
                     if ((err = av_reallocp(&rdt->rmst,
-                                           count * sizeof(*rdt->rmst))) < 0)
+                                           count * sizeof(*rdt->rmst))) < 0) {
+                        rdt->nb_rmst = 0;
                         return err;
+                    }
                     memset(rdt->rmst + rdt->nb_rmst, 0,
                            (count - rdt->nb_rmst) * sizeof(*rdt->rmst));
                     rdt->nb_rmst = count;
diff --git a/libavformat/rtmphttp.c b/libavformat/rtmphttp.c
index 5de1857a28..89a661435c 100644
--- a/libavformat/rtmphttp.c
+++ b/libavformat/rtmphttp.c
@@ -89,8 +89,11 @@ static int rtmp_http_write(URLContext *h, const uint8_t *buf, int size)
     if (rt->out_size + size > rt->out_capacity) {
         int err;
         rt->out_capacity = (rt->out_size + size) * 2;
-        if ((err = av_reallocp(&rt->out_data, rt->out_capacity)) < 0)
+        if ((err = av_reallocp(&rt->out_data, rt->out_capacity)) < 0) {
+            rt->out_size = 0;
+            rt->out_capacity = 0;
             return err;
+        }
     }
 
     memcpy(rt->out_data + rt->out_size, buf, size);
diff --git a/libavformat/rtmpproto.c b/libavformat/rtmpproto.c
index d748a30fe3..05b28a4d04 100644
--- a/libavformat/rtmpproto.c
+++ b/libavformat/rtmpproto.c
@@ -156,8 +156,11 @@ static int add_tracked_method(RTMPContext *rt, const char *name, int id)
     if (rt->nb_tracked_methods + 1 > rt->tracked_methods_size) {
         rt->tracked_methods_size = (rt->nb_tracked_methods + 1) * 2;
         if ((err = av_reallocp(&rt->tracked_methods, rt->tracked_methods_size *
-                               sizeof(*rt->tracked_methods))) < 0)
+                               sizeof(*rt->tracked_methods))) < 0) {
+            rt->nb_tracked_methods = 0;
+            rt->tracked_methods_size = 0;
             return err;
+        }
     }
 
     rt->tracked_methods[rt->nb_tracked_methods].name = av_strdup(name);
diff --git a/libavformat/rtpdec_qt.c b/libavformat/rtpdec_qt.c
index bb0a73b986..2d9c603fcd 100644
--- a/libavformat/rtpdec_qt.c
+++ b/libavformat/rtpdec_qt.c
@@ -174,8 +174,10 @@ static int qt_rtp_parse_packet(AVFormatContext *s, PayloadContext *qt,
         if (qt->pkt.size > 0 && qt->timestamp == *timestamp) {
             int err;
             if ((err = av_reallocp(&qt->pkt.data, qt->pkt.size + alen +
-                                   FF_INPUT_BUFFER_PADDING_SIZE)) < 0)
+                                   FF_INPUT_BUFFER_PADDING_SIZE)) < 0) {
+                qt->pkt.size = 0;
                 return err;
+            }
         } else {
             av_freep(&qt->pkt.data);
             av_init_packet(&qt->pkt);
diff --git a/libavformat/smacker.c b/libavformat/smacker.c
index e68c3fd467..5af5e50784 100644
--- a/libavformat/smacker.c
+++ b/libavformat/smacker.c
@@ -315,8 +315,10 @@ static int smacker_read_packet(AVFormatContext *s, AVPacket *pkt)
                 frame_size -= size;
                 frame_size -= 4;
                 smk->curstream++;
-                if ((err = av_reallocp(&smk->bufs[smk->curstream], size)) < 0)
+                if ((err = av_reallocp(&smk->bufs[smk->curstream], size)) < 0) {
+                    smk->buf_sizes[smk->curstream] = 0;
                     return err;
+                }
                 smk->buf_sizes[smk->curstream] = size;
                 ret = avio_read(s->pb, smk->bufs[smk->curstream], size);
                 if(ret != size)
diff --git a/libavformat/smoothstreamingenc.c b/libavformat/smoothstreamingenc.c
index 9937f498dc..2fe01b1f59 100644
--- a/libavformat/smoothstreamingenc.c
+++ b/libavformat/smoothstreamingenc.c
@@ -450,8 +450,11 @@ static int add_fragment(OutputStream *os, const char *file, const char *infofile
     if (os->nb_fragments >= os->fragments_size) {
         os->fragments_size = (os->fragments_size + 1) * 2;
         if ((err = av_reallocp(&os->fragments, sizeof(*os->fragments) *
-                               os->fragments_size)) < 0)
+                               os->fragments_size)) < 0) {
+            os->fragments_size = 0;
+            os->nb_fragments = 0;
             return err;
+        }
     }
     frag = av_mallocz(sizeof(*frag));
     if (!frag)