avcodec/diracdec: Fix qfactor/offset tables

It seems the previous tables where calculated with 32bit integers ignoring
overflows.
Also check for the max qindex, the value is choosen so that the qfactor/offset
fit in int32.

Fixes: 070b7914fd5dfe8f93248bea71363410/asan_static-oob_c8d034_2764_258e20f4a3c79158aecddb61a833d756.drc
Fixes out of array reads

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

1 files changed, 63 insertions, 45 deletions

diff --git a/libavcodec/diracdec.c b/libavcodec/diracdec.c
index 942376bb3c..ee07ff8bb3 100644
--- a/libavcodec/diracdec.c
+++ b/libavcodec/diracdec.c
@@ -253,53 +253,61 @@ static const uint8_t default_qmat[][4][4] = {
     { { 3,  1,  1,  0}, { 0,  4,  4,  2}, { 0,  6,  6,  5}, { 0,  9,  9,  7} },
 };
 
-static const int32_t qscale_tab[128] = {
-        4,         5,         6,         7,          8,        10,        11,        13,
-       16,        19,        23,        27,         32,        38,        45,        54,
-       64,        76,        91,       108,        128,       152,       181,       215,
-      256,       304,       362,       431,        512,       609,       724,       861,
-     1024,      1218,      1448,      1722,       2048,      2435,      2896,      3444,
-     4096,      4871,      5793,      6889,       8192,      9742,     11585,     13777,
-    16384,     19484,    -13317,     27554,      32768,     -1581,      9853,    -10518,
-    65536,     -3164,    -16782,    -21037,     131072,     -6328,      2922,     23552,
-   262144,    -12658,      5844,    -18524,     524288,     15232,     11689,     28578,
-  1048576,    -10085,    -13110,     -8471,    2097152,    -20170,     10267,    -16943,
-  4194304,       208,    -15954,     31741,    8388608,       416,      4579,     -2146,
- 16777216,       832,      9158,     -4293,   33554432,      1663,    -18172,     -8587,
- 67108864,      3326,       143,    -17175,   134217728,     6653,       285,     31276,
-268435456,     13306,       570,     -3075,   536870912,   -13938,      1140,     -6152,
-1073741824,    12672,      2281,    -12304, -2147483648,   -15205,      4561,    -24610,
-         0,    10138,      9122,     16407,           0,   -20274,    -18243,    -32813,
+static const int32_t qscale_tab[116] = {
+         4,         5,         6,         7,         8,        10,        11,        13,
+        16,        19,        23,        27,        32,        38,        45,        54,
+        64,        76,        91,       108,       128,       152,       181,       215,
+       256,       304,       362,       431,       512,       609,       724,       861,
+      1024,      1218,      1448,      1722,      2048,      2435,      2896,      3444,
+      4096,      4871,      5793,      6889,      8192,      9742,     11585,     13777,
+     16384,     19484,     23170,     27554,     32768,     38968,     46341,     55109,
+     65536,     77936,     92682,    110218,    131072,    155872,    185364,    220436,
+    262144,    311744,    370728,    440872,    524288,    623487,    741455,    881744,
+   1048576,   1246974,   1482910,   1763488,   2097152,   2493948,   2965821,   3526975,
+   4194304,   4987896,   5931642,   7053950,   8388608,   9975792,  11863283,  14107901,
+  16777216,  19951585,  23726566,  28215802,  33554432,  39903169,  47453133,  56431603,
+  67108864,  79806339,  94906266, 112863206, 134217728, 159612677, 189812531, 225726413,
+ 268435456, 319225354, 379625062, 451452825, 536870912, 638450708, 759250125, 902905651,
+1073741824,1276901417,1518500250,1805811301,/*2147483648,2553802834,3037000500,3611622603,
+4294967296*/
 };
 
-static const int32_t qoffset_intra_tab[128] = {
+static const int32_t qoffset_intra_tab[120] = {
         1,         2,         3,         4,         4,         5,         6,         7,
-        8,        10,        12,        14,          16,        19,        23,        27,
-       32,        38,        46,        54,          64,        76,        91,       108,
-      128,       152,       181,       216,         256,       305,       362,       431,
-      512,       609,       724,       861,        1024,      1218,      1448,      1722,
-     2048,      2436,      2897,      3445,        4096,      4871,      5793,      6889,
-     8192,      9742,     -6658,     13777,       16384,      -790,      4927,     -5258,
-    32768,     -1581,     -8390,    -10518,       65536,     -3163,      1461,     11776,
-   131072,     -6328,      2922,     -9261,      262144,      7616,      5845,     14289,
-   524288,     -5042,     -6554,     -4235,     1048576,    -10084,      5134,     -8471,
-  2097152,       104,     -7976,     15871,     4194304,       208,      2290,     -1072,
-  8388608,       416,      4579,     -2146,    16777216,       832,     -9085,     -4293,
- 33554432,      1663,        72,     -8587,    67108864,      3327,       143,     15638,
-134217728,      6653,       285,     -1537,   268435456,     -6968,       570,     -3075,
-536870912,      6336,      1141,     -6151, -1073741823,     -7602,      2281,    -12304,
-        0,      5069,      4561,      8204,           0,    -10136,     -9121,    -16406,
+        8,        10,        12,        14,        16,        19,        23,        27,
+       32,        38,        46,        54,        64,        76,        91,       108,
+      128,       152,       181,       216,       256,       305,       362,       431,
+      512,       609,       724,       861,      1024,      1218,      1448,      1722,
+     2048,      2436,      2897,      3445,      4096,      4871,      5793,      6889,
+     8192,      9742,     11585,     13777,     16384,     19484,     23171,     27555,
+    32768,     38968,     46341,     55109,     65536,     77936,     92682,    110218,
+   131072,    155872,    185364,    220436,    262144,    311744,    370728,    440872,
+   524288,    623487,    741455,    881744,   1048576,   1246974,   1482911,   1763488,
+  2097152,   2493948,   2965821,   3526975,   4194304,   4987896,   5931642,   7053951,
+  8388608,   9975793,  11863283,  14107901,  16777216,  19951585,  23726567,  28215802,
+ 33554432,  39903170,  47453133,  56431603,  67108864,  79806339,  94906266, 112863207,
+134217728, 159612677, 189812531, 225726413, 268435456, 319225354, 379625063, 451452826,
+536870912, 638450709, 759250125, 902905651,1073741824,1276901417,1518500250,1805811302,
+/*2147483648, 2553802834, 3037000500, 3611622603, 4294967296,*/
 };
 
-static const int qoffset_inter_tab[MAX_QUANT+1] = {
-    1,     2,     2,     3,     3,     4,     4,     5,
-    6,     7,     9,    10,    12,    14,    17,    20,
-    24,    29,    34,    41,    48,    57,    68,    81,
-    96,   114,   136,   162,   192,   228,   272,   323,
-    384,   457,   543,   646,   768,   913,  1086,  1292,
-    1536,  1827,  2172,  2583,  3072,  3653,  4344,  5166,
-    6144,  7307,  8689, 10333, 12288, 14613, 17378, 20666,
-    24576, 29226
+static const int qoffset_inter_tab[122] = {
+        1,         2,         2,         3,         3,         4,         4,         5,
+        6,         7,         9,        10,        12,        14,        17,        20,
+       24,        29,        34,        41,        48,        57,        68,        81,
+       96,       114,       136,       162,       192,       228,       272,       323,
+      384,       457,       543,       646,       768,       913,      1086,      1292,
+     1536,      1827,      2172,      2583,      3072,      3653,      4344,      5166,
+     6144,      7307,      8689,     10333,     12288,     14613,     17378,     20666,
+    24576,     29226,     34756,     41332,     49152,     58452,     69512,     82664,
+    98304,    116904,    139023,    165327,    196608,    233808,    278046,    330654,
+   393216,    467615,    556091,    661308,    786432,    935231,   1112183,   1322616,
+  1572864,   1870461,   2224366,   2645231,   3145728,   3740922,   4448731,   5290463,
+  6291456,   7481844,   8897462,  10580926,  12582912,  14963688,  17794925,  21161851,
+ 25165824,  29927377,  35589850,  42323702,  50331648,  59854754,  71179699,  84647405,
+100663296, 119709508, 142359398, 169294809, 201326592, 239419016, 284718797, 338589619,
+402653184, 478838031, 569437594, 677179238, 805306368, 957676063,1138875188,1354358476,
+1610612736, 1915352125, /*2277750375, 2708716952, 3221225472, 3830704250,*/
 };
 
 /* magic number division by 3 from schroedinger */
@@ -601,7 +609,11 @@ static inline void codeblock(DiracContext *s, SubBand *b,
         b->quant = quant;
     }
 
-    b->quant = FFMIN(b->quant, MAX_QUANT);
+    if (b->quant > 115) {
+        av_log(s->avctx, AV_LOG_ERROR, "Unsupported quant %d\n", b->quant);
+        b->quant = 0;
+        return;
+    }
 
     qfactor = qscale_tab[b->quant];
     /* TODO: context pointer? */
@@ -781,12 +793,18 @@ static void decode_subband(DiracContext *s, GetBitContext *gb, int quant,
     int top    = b1->height * slice_y    / s->num_y;
     int bottom = b1->height *(slice_y+1) / s->num_y;
 
-    int qfactor = qscale_tab[quant & 0x7f];
-    int qoffset = qoffset_intra_tab[quant & 0x7f] + 2;
+    int qfactor, qoffset;
 
     uint8_t *buf1 =      b1->ibuf + top * b1->stride;
     uint8_t *buf2 = b2 ? b2->ibuf + top * b2->stride: NULL;
     int x, y;
+
+    if (quant > 115) {
+        av_log(s->avctx, AV_LOG_ERROR, "Unsupported quant %d\n", quant);
+        return;
+    }
+    qfactor = qscale_tab[quant & 0x7f];
+    qoffset = qoffset_intra_tab[quant & 0x7f] + 2;
     /* we have to constantly check for overread since the spec explicitly
        requires this, with the meaning that all remaining coeffs are set to 0 */
     if (get_bits_count(gb) >= bits_end)