nut: Fix unchecked allocations

CC: libav-stable@libav.org

(cherry picked from commit b1fcdc08ceb5df69fac34aa0d57c56905d32b8b4)
Signed-off-by: Derek Buitenhuis <derek.buitenhuis@gmail.com>

4 files changed, 16 insertions, 4 deletions

diff --git a/libavformat/nut.c b/libavformat/nut.c
index 196e04e54f..65d84d1d41 100644
--- a/libavformat/nut.c
+++ b/libavformat/nut.c
@@ -179,10 +179,16 @@ int ff_nut_sp_pts_cmp(const Syncpoint *a, const Syncpoint *b){
     return ((a->ts - b->ts) >> 32) - ((b->ts - a->ts) >> 32);
 }
 
-void ff_nut_add_sp(NUTContext *nut, int64_t pos, int64_t back_ptr, int64_t ts){
+int ff_nut_add_sp(NUTContext *nut, int64_t pos, int64_t back_ptr, int64_t ts){
     Syncpoint *sp= av_mallocz(sizeof(Syncpoint));
     struct AVTreeNode *node = av_tree_node_alloc();
 
+    if (!sp || !node) {
+        av_freep(&sp);
+        av_freep(&node);
+        return AVERROR(ENOMEM);
+    }
+
     sp->pos= pos;
     sp->back_ptr= back_ptr;
     sp->ts= ts;
@@ -191,6 +197,8 @@ void ff_nut_add_sp(NUTContext *nut, int64_t pos, int64_t back_ptr, int64_t ts){
         av_free(sp);
         av_free(node);
     }
+
+    return 0;
 }
 
 static int enu_free(void *opaque, void *elem)
diff --git a/libavformat/nut.h b/libavformat/nut.h
index 89b0248fa4..066d186f25 100644
--- a/libavformat/nut.h
+++ b/libavformat/nut.h
@@ -119,7 +119,7 @@ void ff_nut_reset_ts(NUTContext *nut, AVRational time_base, int64_t val);
 int64_t ff_lsb2full(StreamContext *stream, int64_t lsb);
 int ff_nut_sp_pos_cmp(const Syncpoint *a, const Syncpoint *b);
 int ff_nut_sp_pts_cmp(const Syncpoint *a, const Syncpoint *b);
-void ff_nut_add_sp(NUTContext *nut, int64_t pos, int64_t back_ptr, int64_t ts);
+int ff_nut_add_sp(NUTContext *nut, int64_t pos, int64_t back_ptr, int64_t ts);
 void ff_nut_free_sp(NUTContext *nut);
 
 extern const Dispositions ff_nut_dispositions[];
diff --git a/libavformat/nutdec.c b/libavformat/nutdec.c
index b705987795..1a9390c9d6 100644
--- a/libavformat/nutdec.c
+++ b/libavformat/nutdec.c
@@ -526,6 +526,7 @@ static int decode_syncpoint(NUTContext *nut, int64_t *ts, int64_t *back_ptr)
     AVFormatContext *s = nut->avf;
     AVIOContext *bc    = s->pb;
     int64_t end, tmp;
+    int ret;
 
     nut->last_syncpoint_pos = avio_tell(bc) - 8;
 
@@ -547,7 +548,9 @@ static int decode_syncpoint(NUTContext *nut, int64_t *ts, int64_t *back_ptr)
 
     *ts = tmp / s->nb_streams *
           av_q2d(nut->time_base[tmp % s->nb_streams]) * AV_TIME_BASE;
-    ff_nut_add_sp(nut, nut->last_syncpoint_pos, *back_ptr, *ts);
+
+    if ((ret = ff_nut_add_sp(nut, nut->last_syncpoint_pos, *back_ptr, *ts)) < 0)
+        return ret;
 
     return 0;
 }
diff --git a/libavformat/nutenc.c b/libavformat/nutenc.c
index df70f94122..51bddf00e7 100644
--- a/libavformat/nutenc.c
+++ b/libavformat/nutenc.c
@@ -815,7 +815,8 @@ static int nut_write_packet(AVFormatContext *s, AVPacket *pkt)
         ff_put_v(dyn_bc, sp ? (nut->last_syncpoint_pos - sp->pos) >> 4 : 0);
         put_packet(nut, bc, dyn_bc, 1, SYNCPOINT_STARTCODE);
 
-        ff_nut_add_sp(nut, nut->last_syncpoint_pos, 0 /*unused*/, pkt->dts);
+        if ((ret = ff_nut_add_sp(nut, nut->last_syncpoint_pos, 0 /*unused*/, pkt->dts)) < 0)
+            return ret;
     }
     assert(nus->last_pts != AV_NOPTS_VALUE);