cook: extend channel uncoupling tables so the full bit range is covered.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
(cherry picked from commit 37cc8600d0313838cab5b886b9d373e5819aa24f)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>

2 files changed, 24 insertions, 7 deletions

diff --git a/libavcodec/cook.c b/libavcodec/cook.c
index 7c499f0c93..d869c4200e 100644
--- a/libavcodec/cook.c
+++ b/libavcodec/cook.c
@@ -836,8 +836,8 @@ static void joint_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer1,
         cpl_tmp = cplband[i];
         idx -= decouple_tab[cpl_tmp];
         cplscale = q->cplscales[p->js_vlc_bits - 2];  // choose decoupler table
-        f1 = cplscale[decouple_tab[cpl_tmp]];
-        f2 = cplscale[idx - 1];
+        f1 = cplscale[decouple_tab[cpl_tmp] + 1];
+        f2 = cplscale[idx];
         q->decouple(q, p, i, f1, f2, decode_buffer, mlt_buffer1, mlt_buffer2);
         idx = (1 << p->js_vlc_bits) - 1;
     }
diff --git a/libavcodec/cookdata.h b/libavcodec/cookdata.h
index 6825a4459c..c4c26fae5f 100644
--- a/libavcodec/cookdata.h
+++ b/libavcodec/cookdata.h
@@ -510,23 +510,37 @@ static const int cplband[51] = {
     19,
 };
 
-static const float cplscale2[3] = {
+// The 1 and 0 at the beginning/end are to prevent overflows with
+// bitstream-read indexes. E.g. if n_bits=5, we can access any
+// index from [1, (1<<n_bits)] for the first decoupling coeff,
+// and (1<<n_bits)-coeff1 as index for coeff2, i.e.:
+// coeff1_idx = [1, 32], and coeff2_idx = [0, 31].
+// These values aren't part of the tables in the original binary.
+
+static const float cplscale2[5] = {
+1,
 0.953020632266998,0.70710676908493,0.302905440330505,
+0,
 };
 
-static const float cplscale3[7] = {
+static const float cplscale3[9] = {
+1,
 0.981279790401459,0.936997592449188,0.875934481620789,0.70710676908493,
 0.482430040836334,0.349335819482803,0.192587479948997,
+0,
 };
 
-static const float cplscale4[15] = {
+static const float cplscale4[17] = {
+1,
 0.991486728191376,0.973249018192291,0.953020632266998,0.930133521556854,
 0.903453230857849,0.870746195316315,0.826180458068848,0.70710676908493,
 0.563405573368073,0.491732746362686,0.428686618804932,0.367221474647522,
 0.302905440330505,0.229752898216248,0.130207896232605,
+0,
 };
 
-static const float cplscale5[31] = {
+static const float cplscale5[33] = {
+1,
 0.995926380157471,0.987517595291138,0.978726446628571,0.969505727291107,
 0.95979779958725,0.949531257152557,0.938616216182709,0.926936149597168,
 0.914336204528809,0.900602877140045,0.885426938533783,0.868331849575043,
@@ -535,9 +549,11 @@ static const float cplscale5[31] = {
 0.464778542518616,0.434642940759659,0.404955863952637,0.375219136476517,
 0.344963222742081,0.313672333955765,0.280692428350449,0.245068684220314,
 0.205169528722763,0.157508864998817,0.0901700109243393,
+0,
 };
 
-static const float cplscale6[63] = {
+static const float cplscale6[65] = {
+1,
 0.998005926609039,0.993956744670868,0.989822506904602,0.985598564147949,
 0.981279790401459,0.976860702037811,0.972335040569305,0.967696130275726,
 0.962936460971832,0.958047747612000,0.953020632266998,0.947844684123993,
@@ -554,6 +570,7 @@ static const float cplscale6[63] = {
 0.302905440330505,0.286608695983887,0.269728302955627,0.252119421958923,
 0.233590632677078,0.213876649737358,0.192587479948997,0.169101938605309,
 0.142307326197624,0.109772264957428,0.0631198287010193,
+0,
 };
 
 static const float* const cplscales[5] = {