avcodec/vp6: select idct based (loosely) on number of coefficients decoded

The VP3/4/5/6 reference decoders all use three IDCT versions: one for the
DC-only case, another for blocks with more than 10 coefficients, and an
optimised one for blocks with up to 10 AC coefficents. VP6 relies on the
sparse 10 coefficient version, and without it, IDCT drift occurs.

Fixes: https://trac.ffmpeg.org/ticket/1282

Signed-off-by: Peter Ross <pross@xvid.org>

1 files changed, 14 insertions, 0 deletions

diff --git a/libavcodec/vp6.c b/libavcodec/vp6.c
index 645fc5c690..977fcb7076 100644
--- a/libavcodec/vp6.c
+++ b/libavcodec/vp6.c
@@ -194,6 +194,18 @@ static void vp6_coeff_order_table_init(VP56Context *s)
         for (pos=1; pos<64; pos++)
             if (s->modelp->coeff_reorder[pos] == i)
                 s->modelp->coeff_index_to_pos[idx++] = pos;
+
+    for (idx = 0; idx < 64; idx++) {
+        int max = 0;
+        for (i = 0; i <= idx; i++) {
+            int v = s->modelp->coeff_index_to_pos[i];
+            if (v > max)
+                max = v;
+        }
+        if (s->sub_version > 6)
+            max++;
+        s->modelp->coeff_index_to_idct_selector[idx] = max;
+    }
 }
 
 static void vp6_default_models_init(VP56Context *s)
@@ -446,6 +458,7 @@ static int vp6_parse_coeff_huffman(VP56Context *s)
             cg = FFMIN(vp6_coeff_groups[coeff_idx], 3);
             vlc_coeff = &s->ract_vlc[pt][ct][cg];
         }
+        s->idct_selector[b] = model->coeff_index_to_idct_selector[FFMIN(coeff_idx, 63)];
     }
     return 0;
 }
@@ -527,6 +540,7 @@ static int vp6_parse_coeff(VP56Context *s)
 
         s->left_block[ff_vp56_b6to4[b]].not_null_dc =
         s->above_blocks[s->above_block_idx[b]].not_null_dc = !!s->block_coeff[b][0];
+        s->idct_selector[b] = model->coeff_index_to_idct_selector[FFMIN(coeff_idx, 63)];
     }
     return 0;
 }