avcodec/vp9dsp_template: Fix integer overflow(s) in iadst16_1d()

Fixes: signed integer overflow: 1080285923 - -1130879337 cannot be represented in type 'int'
Fixes: 22002/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_VP9_fuzzer-6260237310099456

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

1 files changed, 76 insertions, 76 deletions

diff --git a/libavcodec/vp9dsp_template.c b/libavcodec/vp9dsp_template.c
index c3273dd726..c6944f5ce3 100644
--- a/libavcodec/vp9dsp_template.c
+++ b/libavcodec/vp9dsp_template.c
@@ -1378,48 +1378,48 @@ static av_always_inline void iadst16_1d(const dctcoef *in, ptrdiff_t stride,
     dctint t0a, t1a, t2a, t3a, t4a, t5a, t6a, t7a;
     dctint t8a, t9a, t10a, t11a, t12a, t13a, t14a, t15a;
 
-    t0  = IN(15) * 16364 + IN(0)  *   804;
-    t1  = IN(15) *   804 - IN(0)  * 16364;
-    t2  = IN(13) * 15893 + IN(2)  *  3981;
-    t3  = IN(13) *  3981 - IN(2)  * 15893;
-    t4  = IN(11) * 14811 + IN(4)  *  7005;
-    t5  = IN(11) *  7005 - IN(4)  * 14811;
-    t6  = IN(9)  * 13160 + IN(6)  *  9760;
-    t7  = IN(9)  *  9760 - IN(6)  * 13160;
-    t8  = IN(7)  * 11003 + IN(8)  * 12140;
-    t9  = IN(7)  * 12140 - IN(8)  * 11003;
-    t10 = IN(5)  *  8423 + IN(10) * 14053;
-    t11 = IN(5)  * 14053 - IN(10) *  8423;
-    t12 = IN(3)  *  5520 + IN(12) * 15426;
-    t13 = IN(3)  * 15426 - IN(12) *  5520;
-    t14 = IN(1)  *  2404 + IN(14) * 16207;
-    t15 = IN(1)  * 16207 - IN(14) *  2404;
-
-    t0a  = (t0 + t8  + (1 << 13)) >> 14;
-    t1a  = (t1 + t9  + (1 << 13)) >> 14;
-    t2a  = (t2 + t10 + (1 << 13)) >> 14;
-    t3a  = (t3 + t11 + (1 << 13)) >> 14;
-    t4a  = (t4 + t12 + (1 << 13)) >> 14;
-    t5a  = (t5 + t13 + (1 << 13)) >> 14;
-    t6a  = (t6 + t14 + (1 << 13)) >> 14;
-    t7a  = (t7 + t15 + (1 << 13)) >> 14;
-    t8a  = (t0 - t8  + (1 << 13)) >> 14;
-    t9a  = (t1 - t9  + (1 << 13)) >> 14;
-    t10a = (t2 - t10 + (1 << 13)) >> 14;
-    t11a = (t3 - t11 + (1 << 13)) >> 14;
-    t12a = (t4 - t12 + (1 << 13)) >> 14;
-    t13a = (t5 - t13 + (1 << 13)) >> 14;
-    t14a = (t6 - t14 + (1 << 13)) >> 14;
-    t15a = (t7 - t15 + (1 << 13)) >> 14;
-
-    t8   = t8a  * 16069 + t9a  *  3196;
-    t9   = t8a  *  3196 - t9a  * 16069;
-    t10  = t10a *  9102 + t11a * 13623;
-    t11  = t10a * 13623 - t11a *  9102;
-    t12  = t13a * 16069 - t12a *  3196;
-    t13  = t13a *  3196 + t12a * 16069;
-    t14  = t15a *  9102 - t14a * 13623;
-    t15  = t15a * 13623 + t14a *  9102;
+    t0  = IN(15) * 16364U + IN(0)  *   804U;
+    t1  = IN(15) *   804U - IN(0)  * 16364U;
+    t2  = IN(13) * 15893U + IN(2)  *  3981U;
+    t3  = IN(13) *  3981U - IN(2)  * 15893U;
+    t4  = IN(11) * 14811U + IN(4)  *  7005U;
+    t5  = IN(11) *  7005U - IN(4)  * 14811U;
+    t6  = IN(9)  * 13160U + IN(6)  *  9760U;
+    t7  = IN(9)  *  9760U - IN(6)  * 13160U;
+    t8  = IN(7)  * 11003U + IN(8)  * 12140U;
+    t9  = IN(7)  * 12140U - IN(8)  * 11003U;
+    t10 = IN(5)  *  8423U + IN(10) * 14053U;
+    t11 = IN(5)  * 14053U - IN(10) *  8423U;
+    t12 = IN(3)  *  5520U + IN(12) * 15426U;
+    t13 = IN(3)  * 15426U - IN(12) *  5520U;
+    t14 = IN(1)  *  2404U + IN(14) * 16207U;
+    t15 = IN(1)  * 16207U - IN(14) *  2404U;
+
+    t0a  = (dctint)((1U << 13) + t0 + t8 ) >> 14;
+    t1a  = (dctint)((1U << 13) + t1 + t9 ) >> 14;
+    t2a  = (dctint)((1U << 13) + t2 + t10) >> 14;
+    t3a  = (dctint)((1U << 13) + t3 + t11) >> 14;
+    t4a  = (dctint)((1U << 13) + t4 + t12) >> 14;
+    t5a  = (dctint)((1U << 13) + t5 + t13) >> 14;
+    t6a  = (dctint)((1U << 13) + t6 + t14) >> 14;
+    t7a  = (dctint)((1U << 13) + t7 + t15) >> 14;
+    t8a  = (dctint)((1U << 13) + t0 - t8 ) >> 14;
+    t9a  = (dctint)((1U << 13) + t1 - t9 ) >> 14;
+    t10a = (dctint)((1U << 13) + t2 - t10) >> 14;
+    t11a = (dctint)((1U << 13) + t3 - t11) >> 14;
+    t12a = (dctint)((1U << 13) + t4 - t12) >> 14;
+    t13a = (dctint)((1U << 13) + t5 - t13) >> 14;
+    t14a = (dctint)((1U << 13) + t6 - t14) >> 14;
+    t15a = (dctint)((1U << 13) + t7 - t15) >> 14;
+
+    t8   = t8a  * 16069U + t9a  *  3196U;
+    t9   = t8a  *  3196U - t9a  * 16069U;
+    t10  = t10a *  9102U + t11a * 13623U;
+    t11  = t10a * 13623U - t11a *  9102U;
+    t12  = t13a * 16069U - t12a *  3196U;
+    t13  = t13a *  3196U + t12a * 16069U;
+    t14  = t15a *  9102U - t14a * 13623U;
+    t15  = t15a * 13623U + t14a *  9102U;
 
     t0   = t0a + t4a;
     t1   = t1a + t5a;
@@ -1429,49 +1429,49 @@ static av_always_inline void iadst16_1d(const dctcoef *in, ptrdiff_t stride,
     t5   = t1a - t5a;
     t6   = t2a - t6a;
     t7   = t3a - t7a;
-    t8a  = (t8  + t12 + (1 << 13)) >> 14;
-    t9a  = (t9  + t13 + (1 << 13)) >> 14;
-    t10a = (t10 + t14 + (1 << 13)) >> 14;
-    t11a = (t11 + t15 + (1 << 13)) >> 14;
-    t12a = (t8  - t12 + (1 << 13)) >> 14;
-    t13a = (t9  - t13 + (1 << 13)) >> 14;
-    t14a = (t10 - t14 + (1 << 13)) >> 14;
-    t15a = (t11 - t15 + (1 << 13)) >> 14;
-
-    t4a  = t4 * 15137 + t5 *  6270;
-    t5a  = t4 *  6270 - t5 * 15137;
-    t6a  = t7 * 15137 - t6 *  6270;
-    t7a  = t7 *  6270 + t6 * 15137;
-    t12  = t12a * 15137 + t13a *  6270;
-    t13  = t12a *  6270 - t13a * 15137;
-    t14  = t15a * 15137 - t14a *  6270;
-    t15  = t15a *  6270 + t14a * 15137;
+    t8a  = (dctint)((1U << 13) + t8  + t12) >> 14;
+    t9a  = (dctint)((1U << 13) + t9  + t13) >> 14;
+    t10a = (dctint)((1U << 13) + t10 + t14) >> 14;
+    t11a = (dctint)((1U << 13) + t11 + t15) >> 14;
+    t12a = (dctint)((1U << 13) + t8  - t12) >> 14;
+    t13a = (dctint)((1U << 13) + t9  - t13) >> 14;
+    t14a = (dctint)((1U << 13) + t10 - t14) >> 14;
+    t15a = (dctint)((1U << 13) + t11 - t15) >> 14;
+
+    t4a  = t4 * 15137U + t5 *  6270U;
+    t5a  = t4 *  6270U - t5 * 15137U;
+    t6a  = t7 * 15137U - t6 *  6270U;
+    t7a  = t7 *  6270U + t6 * 15137U;
+    t12  = t12a * 15137U + t13a *  6270U;
+    t13  = t12a *  6270U - t13a * 15137U;
+    t14  = t15a * 15137U - t14a *  6270U;
+    t15  = t15a *  6270U + t14a * 15137U;
 
     out[ 0] =   t0 + t2;
     out[15] = -(t1 + t3);
     t2a     =   t0 - t2;
     t3a     =   t1 - t3;
-    out[ 3] = -((t4a + t6a + (1 << 13)) >> 14);
-    out[12] =   (t5a + t7a + (1 << 13)) >> 14;
-    t6      =   (t4a - t6a + (1 << 13)) >> 14;
-    t7      =   (t5a - t7a + (1 << 13)) >> 14;
+    out[ 3] = -((dctint)((1U << 13) + t4a + t6a) >> 14);
+    out[12] =   (dctint)((1U << 13) + t5a + t7a) >> 14;
+    t6      =   (dctint)((1U << 13) + t4a - t6a) >> 14;
+    t7      =   (dctint)((1U << 13) + t5a - t7a) >> 14;
     out[ 1] = -(t8a + t10a);
     out[14] =   t9a + t11a;
     t10     =   t8a - t10a;
     t11     =   t9a - t11a;
-    out[ 2] =   (t12 + t14 + (1 << 13)) >> 14;
-    out[13] = -((t13 + t15 + (1 << 13)) >> 14);
-    t14a    =   (t12 - t14 + (1 << 13)) >> 14;
-    t15a    =   (t13 - t15 + (1 << 13)) >> 14;
-
-    out[ 7] = ((t2a  + t3a)  * -11585 + (1 << 13)) >> 14;
-    out[ 8] = ((t2a  - t3a)  *  11585 + (1 << 13)) >> 14;
-    out[ 4] = ((t7   + t6)   *  11585 + (1 << 13)) >> 14;
-    out[11] = ((t7   - t6)   *  11585 + (1 << 13)) >> 14;
-    out[ 6] = ((t11  + t10)  *  11585 + (1 << 13)) >> 14;
-    out[ 9] = ((t11  - t10)  *  11585 + (1 << 13)) >> 14;
-    out[ 5] = ((t14a + t15a) * -11585 + (1 << 13)) >> 14;
-    out[10] = ((t14a - t15a) *  11585 + (1 << 13)) >> 14;
+    out[ 2] =   (dctint)((1U << 13) + t12 + t14) >> 14;
+    out[13] = -((dctint)((1U << 13) + t13 + t15) >> 14);
+    t14a    =   (dctint)((1U << 13) + t12 - t14) >> 14;
+    t15a    =   (dctint)((1U << 13) + t13 - t15) >> 14;
+
+    out[ 7] = (dctint)(-(t2a  + t3a)  * 11585U  + (1 << 13)) >> 14;
+    out[ 8] = (dctint)( (t2a  - t3a)  * 11585U  + (1 << 13)) >> 14;
+    out[ 4] = (dctint)( (t7   + t6)   * 11585U  + (1 << 13)) >> 14;
+    out[11] = (dctint)( (t7   - t6)   * 11585U  + (1 << 13)) >> 14;
+    out[ 6] = (dctint)( (t11  + t10)  * 11585U  + (1 << 13)) >> 14;
+    out[ 9] = (dctint)( (t11  - t10)  * 11585U  + (1 << 13)) >> 14;
+    out[ 5] = (dctint)(-(t14a + t15a) * 11585U  + (1 << 13)) >> 14;
+    out[10] = (dctint)( (t14a - t15a) * 11585U  + (1 << 13)) >> 14;
 }
 
 itxfm_wrap(16, 6)