md5: Allocate a normal private context for the opaque md5 context pointer

This avoids having to overestimate the md5 context size, which
isn't known beforehand, allowing us to use the new allocate functions
instead.

Signed-off-by: Martin Storsjö <martin@martin.st>

2 files changed, 53 insertions, 26 deletions

diff --git a/libavformat/md5enc.c b/libavformat/md5enc.c
index fbd4d99397..16412c9a34 100644
--- a/libavformat/md5enc.c
+++ b/libavformat/md5enc.c
@@ -23,13 +23,16 @@
 #include "avformat.h"
 #include "internal.h"
 
-#define PRIVSIZE 512
+struct MD5Context {
+    struct AVMD5 *md5;
+};
 
 static void md5_finish(struct AVFormatContext *s, char *buf)
 {
+    struct MD5Context *c = s->priv_data;
     uint8_t md5[16];
     int i, offset = strlen(buf);
-    av_md5_final(s->priv_data, md5);
+    av_md5_final(c->md5, md5);
     for (i = 0; i < sizeof(md5); i++) {
         snprintf(buf + offset, 3, "%02"PRIx8, md5[i]);
         offset += 2;
@@ -44,25 +47,29 @@ static void md5_finish(struct AVFormatContext *s, char *buf)
 #if CONFIG_MD5_MUXER
 static int write_header(struct AVFormatContext *s)
 {
-    if (PRIVSIZE < av_md5_size) {
-        av_log(s, AV_LOG_ERROR, "Insuffient size for md5 context\n");
-        return -1;
-    }
-    av_md5_init(s->priv_data);
+    struct MD5Context *c = s->priv_data;
+    c->md5 = av_md5_alloc();
+    if (!c->md5)
+        return AVERROR(ENOMEM);
+    av_md5_init(c->md5);
     return 0;
 }
 
 static int write_packet(struct AVFormatContext *s, AVPacket *pkt)
 {
-    av_md5_update(s->priv_data, pkt->data, pkt->size);
+    struct MD5Context *c = s->priv_data;
+    av_md5_update(c->md5, pkt->data, pkt->size);
     return 0;
 }
 
 static int write_trailer(struct AVFormatContext *s)
 {
+    struct MD5Context *c = s->priv_data;
     char buf[64] = "MD5=";
 
     md5_finish(s, buf);
+
+    av_freep(&c->md5);
     return 0;
 }
 
@@ -70,7 +77,7 @@ AVOutputFormat ff_md5_muxer = {
     .name              = "md5",
     .long_name         = NULL_IF_CONFIG_SMALL("MD5 testing"),
     .extensions        = "",
-    .priv_data_size    = PRIVSIZE,
+    .priv_data_size    = sizeof(struct MD5Context),
     .audio_codec       = AV_CODEC_ID_PCM_S16LE,
     .video_codec       = AV_CODEC_ID_RAWVIDEO,
     .write_header      = write_header,
@@ -81,15 +88,21 @@ AVOutputFormat ff_md5_muxer = {
 #endif
 
 #if CONFIG_FRAMEMD5_MUXER
+static int framemd5_write_header(struct AVFormatContext *s)
+{
+    struct MD5Context *c = s->priv_data;
+    c->md5 = av_md5_alloc();
+    if (!c->md5)
+        return AVERROR(ENOMEM);
+    return ff_framehash_write_header(s);
+}
+
 static int framemd5_write_packet(struct AVFormatContext *s, AVPacket *pkt)
 {
+    struct MD5Context *c = s->priv_data;
     char buf[256];
-    if (PRIVSIZE < av_md5_size) {
-        av_log(s, AV_LOG_ERROR, "Insuffient size for md5 context\n");
-        return -1;
-    }
-    av_md5_init(s->priv_data);
-    av_md5_update(s->priv_data, pkt->data, pkt->size);
+    av_md5_init(c->md5);
+    av_md5_update(c->md5, pkt->data, pkt->size);
 
     snprintf(buf, sizeof(buf) - 64, "%d, %10"PRId64", %10"PRId64", %8d, %8d, ",
              pkt->stream_index, pkt->dts, pkt->pts, pkt->duration, pkt->size);
@@ -97,15 +110,23 @@ static int framemd5_write_packet(struct AVFormatContext *s, AVPacket *pkt)
     return 0;
 }
 
+static int framemd5_write_trailer(struct AVFormatContext *s)
+{
+    struct MD5Context *c = s->priv_data;
+    av_freep(&c->md5);
+    return 0;
+}
+
 AVOutputFormat ff_framemd5_muxer = {
     .name              = "framemd5",
     .long_name         = NULL_IF_CONFIG_SMALL("Per-frame MD5 testing"),
     .extensions        = "",
-    .priv_data_size    = PRIVSIZE,
+    .priv_data_size    = sizeof(struct MD5Context),
     .audio_codec       = AV_CODEC_ID_PCM_S16LE,
     .video_codec       = AV_CODEC_ID_RAWVIDEO,
-    .write_header      = ff_framehash_write_header,
+    .write_header      = framemd5_write_header,
     .write_packet      = framemd5_write_packet,
+    .write_trailer     = framemd5_write_trailer,
     .flags             = AVFMT_VARIABLE_FPS | AVFMT_TS_NONSTRICT,
 };
 #endif
diff --git a/libavformat/md5proto.c b/libavformat/md5proto.c
index 796b4ea682..12ddde3d8c 100644
--- a/libavformat/md5proto.c
+++ b/libavformat/md5proto.c
@@ -27,37 +27,41 @@
 #include "avio.h"
 #include "url.h"
 
-#define PRIV_SIZE 128
+struct MD5Context {
+    struct AVMD5 *md5;
+};
 
 static int md5_open(URLContext *h, const char *filename, int flags)
 {
-    if (PRIV_SIZE < av_md5_size) {
-        av_log(NULL, AV_LOG_ERROR, "Insuffient size for MD5 context\n");
-        return -1;
-    }
+    struct MD5Context *c = h->priv_data;
 
     if (!(flags & AVIO_FLAG_WRITE))
         return AVERROR(EINVAL);
 
-    av_md5_init(h->priv_data);
+    c->md5 = av_md5_alloc();
+    if (!c->md5)
+        return AVERROR(ENOMEM);
+    av_md5_init(c->md5);
 
     return 0;
 }
 
 static int md5_write(URLContext *h, const unsigned char *buf, int size)
 {
-    av_md5_update(h->priv_data, buf, size);
+    struct MD5Context *c = h->priv_data;
+    av_md5_update(c->md5, buf, size);
     return size;
 }
 
 static int md5_close(URLContext *h)
 {
+    struct MD5Context *c = h->priv_data;
     const char *filename = h->filename;
     uint8_t md5[16], buf[64];
     URLContext *out;
     int i, err = 0;
 
-    av_md5_final(h->priv_data, md5);
+    av_md5_final(c->md5, md5);
     for (i = 0; i < sizeof(md5); i++)
         snprintf(buf + i*2, 3, "%02x", md5[i]);
     buf[i*2] = '\n';
@@ -76,6 +80,8 @@ static int md5_close(URLContext *h)
             err = AVERROR(errno);
     }
 
+    av_freep(&c->md5);
+
     return err;
 }
 
@@ -85,5 +91,5 @@ URLProtocol ff_md5_protocol = {
     .url_open            = md5_open,
     .url_write           = md5_write,
     .url_close           = md5_close,
-    .priv_data_size      = PRIV_SIZE,
+    .priv_data_size      = sizeof(struct MD5Context),
 };