avformat/aviobuf: discard part of the IO buffer in ffio_ensure_seekback if needed

Previously ffio_ensure_seekback never flushed the buffer, so successive
ffio_ensure_seekback calls were all respected. This could eventually cause
unlimited memory and CPU usage if a demuxer called ffio_ensure_seekback on all
it's read data.

Most demuxers however only rely on being able to seek back till the position of
the last ffio_ensure_seekback call, therefore we change the semantics of
ffio_ensure_seekback so that a new call can invalidate seek guarantees of the
old. In order to support some level of "nested" ffio_ensure_seekback calls, we
document that the function only invalidates the old window (and potentially
discards the already read data from the IO buffer), if the newly requested
window does not fit into the old one.

This way we limit the memory usage for ffio_ensure_seekback calls requesting
consecutive data windows.

Signed-off-by: Marton Balint <cus@passwd.hu>

2 files changed, 20 insertions, 11 deletions

diff --git a/libavformat/avio_internal.h b/libavformat/avio_internal.h
index c575df8035..fe87f2a288 100644
--- a/libavformat/avio_internal.h
+++ b/libavformat/avio_internal.h
@@ -100,7 +100,9 @@ int ffio_realloc_buf(AVIOContext *s, int buf_size);
  *
  * Will ensure that when reading sequentially up to buf_size, seeking
  * within the current pos and pos+buf_size is possible.
- * Once the stream position moves outside this window this guarantee is lost.
+ * Once the stream position moves outside this window or another
+ * ffio_ensure_seekback call requests a buffer outside this window this
+ * guarantee is lost.
  */
 int ffio_ensure_seekback(AVIOContext *s, int64_t buf_size);
 
diff --git a/libavformat/aviobuf.c b/libavformat/aviobuf.c
index aa1d6c0830..75ddf52efc 100644
--- a/libavformat/aviobuf.c
+++ b/libavformat/aviobuf.c
@@ -991,35 +991,42 @@ URLContext* ffio_geturlcontext(AVIOContext *s)
         return NULL;
 }
 
+static void update_checksum(AVIOContext *s)
+{
+    if (s->update_checksum && s->buf_ptr > s->checksum_ptr) {
+        s->checksum = s->update_checksum(s->checksum, s->checksum_ptr,
+                                         s->buf_ptr - s->checksum_ptr);
+    }
+}
+
 int ffio_ensure_seekback(AVIOContext *s, int64_t buf_size)
 {
     uint8_t *buffer;
     int max_buffer_size = s->max_packet_size ?
                           s->max_packet_size : IO_BUFFER_SIZE;
-    int filled = s->buf_end - s->buffer;
-    ptrdiff_t checksum_ptr_offset = s->checksum_ptr ? s->checksum_ptr - s->buffer : -1;
+    ptrdiff_t filled = s->buf_end - s->buf_ptr;
 
     if (buf_size <= s->buf_end - s->buf_ptr)
         return 0;
 
-    buf_size += s->buf_ptr - s->buffer + max_buffer_size - 1;
+    buf_size += max_buffer_size - 1;
 
-    if (buf_size <= s->buffer_size || s->seekable || !s->read_packet)
+    if (buf_size + s->buf_ptr - s->buffer <= s->buffer_size || s->seekable || !s->read_packet)
         return 0;
     av_assert0(!s->write_flag);
 
+    buf_size = FFMAX(buf_size, s->buffer_size);
     buffer = av_malloc(buf_size);
     if (!buffer)
         return AVERROR(ENOMEM);
-
-    memcpy(buffer, s->buffer, filled);
+    update_checksum(s);
+    memcpy(buffer, s->buf_ptr, filled);
     av_free(s->buffer);
-    s->buf_ptr = buffer + (s->buf_ptr - s->buffer);
-    s->buf_end = buffer + (s->buf_end - s->buffer);
     s->buffer = buffer;
     s->buffer_size = buf_size;
-    if (checksum_ptr_offset >= 0)
-        s->checksum_ptr = s->buffer + checksum_ptr_offset;
+    s->buf_ptr = s->buffer;
+    s->buf_end = s->buffer + filled;
+    s->checksum_ptr = s->buffer;
     return 0;
 }