avutil/buffer: Fix race in pool.

This race will always happen sooner or later in a multi-threaded
environment and it will over time lead to OOM.
This fix works by spinning, there are other ways by which this
can be fixed, like simply detecting the issue after it happened
and freeing the over-allocated memory or simply using a mutex.

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

2 files changed, 9 insertions, 0 deletions

diff --git a/libavutil/buffer.c b/libavutil/buffer.c
index 5c753abce5..854733b32d 100644
--- a/libavutil/buffer.c
+++ b/libavutil/buffer.c
@@ -307,6 +307,7 @@ static AVBufferRef *pool_alloc_buffer(AVBufferPool *pool)
     ret->buffer->free   = pool_release_buffer;
 
     avpriv_atomic_int_add_and_fetch(&pool->refcount, 1);
+    avpriv_atomic_int_add_and_fetch(&pool->nb_allocated, 1);
 
     return ret;
 }
@@ -318,6 +319,12 @@ AVBufferRef *av_buffer_pool_get(AVBufferPool *pool)
 
     /* check whether the pool is empty */
     buf = get_pool(pool);
+    if (!buf && pool->refcount <= pool->nb_allocated) {
+        av_log(NULL, AV_LOG_DEBUG, "Pool race dectected, spining to avoid overallocation and eventual OOM\n");
+        while (!buf && avpriv_atomic_int_get(&pool->refcount) <= avpriv_atomic_int_get(&pool->nb_allocated))
+            buf = get_pool(pool);
+    }
+
     if (!buf)
         return pool_alloc_buffer(pool);
 
diff --git a/libavutil/buffer_internal.h b/libavutil/buffer_internal.h
index b2602f8809..c29190839e 100644
--- a/libavutil/buffer_internal.h
+++ b/libavutil/buffer_internal.h
@@ -85,6 +85,8 @@ struct AVBufferPool {
      */
     volatile int refcount;
 
+    volatile int nb_allocated;
+
     int size;
     AVBufferRef* (*alloc)(int size);
 };