Merge commit '1b1bb2c4efc126d74d44d8c421860c85f932ecb1'

* commit '1b1bb2c4efc126d74d44d8c421860c85f932ecb1':
  rl: Add error checking to ff_rl_init().

Conflicts:
	libavcodec/rl.h

Merged-by: Michael Niedermayer <michaelni@gmx.at>

2 files changed, 21 insertions, 7 deletions

diff --git a/libavcodec/rl.c b/libavcodec/rl.c
index 8b1854c049..b206c6f2ce 100644
--- a/libavcodec/rl.c
+++ b/libavcodec/rl.c
@@ -34,8 +34,8 @@ void ff_rl_free(RLTable *rl)
     }
 }
 
-av_cold void ff_rl_init(RLTable *rl,
-                        uint8_t static_store[2][2 * MAX_RUN + MAX_LEVEL + 3])
+av_cold int ff_rl_init(RLTable *rl,
+                       uint8_t static_store[2][2 * MAX_RUN + MAX_LEVEL + 3])
 {
     int8_t  max_level[MAX_RUN + 1], max_run[MAX_LEVEL + 1];
     uint8_t index_run[MAX_RUN + 1];
@@ -43,7 +43,7 @@ av_cold void ff_rl_init(RLTable *rl,
 
     /* If table is static, we can quit if rl->max_level[0] is not NULL */
     if (static_store && rl->max_level[0])
-        return;
+        return 0;
 
     /* compute max_level[], max_run[] and index_run[] */
     for (last = 0; last < 2; last++) {
@@ -70,20 +70,34 @@ av_cold void ff_rl_init(RLTable *rl,
         }
         if (static_store)
             rl->max_level[last] = static_store[last];
-        else
+        else {
             rl->max_level[last] = av_malloc(MAX_RUN + 1);
+            if (!rl->max_level[last])
+                goto fail;
+        }
         memcpy(rl->max_level[last], max_level, MAX_RUN + 1);
         if (static_store)
             rl->max_run[last]   = static_store[last] + MAX_RUN + 1;
-        else
+        else {
             rl->max_run[last]   = av_malloc(MAX_LEVEL + 1);
+            if (!rl->max_run[last])
+                goto fail;
+        }
         memcpy(rl->max_run[last], max_run, MAX_LEVEL + 1);
         if (static_store)
             rl->index_run[last] = static_store[last] + MAX_RUN + MAX_LEVEL + 2;
-        else
+        else {
             rl->index_run[last] = av_malloc(MAX_RUN + 1);
+            if (!rl->index_run[last])
+                goto fail;
+        }
         memcpy(rl->index_run[last], index_run, MAX_RUN + 1);
     }
+    return 0;
+
+fail:
+    ff_rl_free(rl);
+    return AVERROR(ENOMEM);
 }
 
 av_cold void ff_rl_init_vlc(RLTable *rl, unsigned static_size)
diff --git a/libavcodec/rl.h b/libavcodec/rl.h
index e4a6faa22c..33f4ef88de 100644
--- a/libavcodec/rl.h
+++ b/libavcodec/rl.h
@@ -52,7 +52,7 @@ typedef struct RLTable {
  * @param static_store static uint8_t array[2][2*MAX_RUN + MAX_LEVEL + 3] which will hold
  *                     the level and run tables, if this is NULL av_malloc() will be used
  */
-void ff_rl_init(RLTable *rl, uint8_t static_store[2][2*MAX_RUN + MAX_LEVEL + 3]);
+int ff_rl_init(RLTable *rl, uint8_t static_store[2][2*MAX_RUN + MAX_LEVEL + 3]);
 void ff_rl_init_vlc(RLTable *rl, unsigned static_size);
 
 /**