avcodec/vc1dec: Fixing is_intra, 2nd attempt

This also un-does the fate changes from a52f443714b5c2a40ed272d8445f4c39220a4b69,
leaving this fix without even small differences in the output, that is
a sample for which this makes a vissible difference is very welcome

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

4 files changed, 103 insertions, 99 deletions

diff --git a/libavcodec/vc1dec.c b/libavcodec/vc1dec.c
index 8368781385..adf1e0951e 100644
--- a/libavcodec/vc1dec.c
+++ b/libavcodec/vc1dec.c
@@ -3379,7 +3379,7 @@ static av_always_inline void vc1_apply_p_v_loop_filter(VC1Context *v, int block_
     int mb_cbp         = v->cbp[s->mb_x - s->mb_stride],
         block_cbp      = mb_cbp      >> (block_num * 4), bottom_cbp,
         mb_is_intra    = v->is_intra[s->mb_x - s->mb_stride],
-        block_is_intra = mb_is_intra, bottom_is_intra;
+        block_is_intra = mb_is_intra >> block_num, bottom_is_intra;
     int idx, linesize  = block_num > 3 ? s->uvlinesize : s->linesize, ttblk;
     uint8_t *dst;
 
@@ -3394,19 +3394,19 @@ static av_always_inline void vc1_apply_p_v_loop_filter(VC1Context *v, int block_
 
         if (block_num > 3) {
             bottom_cbp      = v->cbp[s->mb_x]      >> (block_num * 4);
-            bottom_is_intra = v->is_intra[s->mb_x];
+            bottom_is_intra = v->is_intra[s->mb_x] >> block_num;
             mv              = &v->luma_mv[s->mb_x - s->mb_stride];
             mv_stride       = s->mb_stride;
         } else {
             bottom_cbp      = (block_num < 2) ? (mb_cbp               >> ((block_num + 2) * 4))
                                               : (v->cbp[s->mb_x]      >> ((block_num - 2) * 4));
-            bottom_is_intra = (block_num < 2) ? mb_is_intra
-                                              : v->is_intra[s->mb_x];
+            bottom_is_intra = (block_num < 2) ? (mb_is_intra          >> (block_num + 2))
+                                              : (v->is_intra[s->mb_x] >> (block_num - 2));
             mv_stride       = s->b8_stride;
             mv              = &s->current_picture.motion_val[0][s->block_index[block_num] - 2 * mv_stride];
         }
 
-        if (bottom_is_intra || block_is_intra ||
+        if (bottom_is_intra & 1 || block_is_intra & 1 ||
             mv[0][0] != mv[mv_stride][0] || mv[0][1] != mv[mv_stride][1]) {
             v->vc1dsp.vc1_v_loop_filter8(dst, linesize, v->pq);
         } else {
@@ -3443,7 +3443,7 @@ static av_always_inline void vc1_apply_p_h_loop_filter(VC1Context *v, int block_
     int mb_cbp         = v->cbp[s->mb_x - 1 - s->mb_stride],
         block_cbp      = mb_cbp      >> (block_num * 4), right_cbp,
         mb_is_intra    = v->is_intra[s->mb_x - 1 - s->mb_stride],
-        block_is_intra = mb_is_intra, right_is_intra;
+        block_is_intra = mb_is_intra >> block_num, right_is_intra;
     int idx, linesize  = block_num > 3 ? s->uvlinesize : s->linesize, ttblk;
     uint8_t *dst;
 
@@ -3458,16 +3458,16 @@ static av_always_inline void vc1_apply_p_h_loop_filter(VC1Context *v, int block_
 
         if (block_num > 3) {
             right_cbp      = v->cbp[s->mb_x - s->mb_stride] >> (block_num * 4);
-            right_is_intra = v->is_intra[s->mb_x - s->mb_stride];
+            right_is_intra = v->is_intra[s->mb_x - s->mb_stride] >> block_num;
             mv             = &v->luma_mv[s->mb_x - s->mb_stride - 1];
         } else {
             right_cbp      = (block_num & 1) ? (v->cbp[s->mb_x - s->mb_stride]      >> ((block_num - 1) * 4))
                                              : (mb_cbp                              >> ((block_num + 1) * 4));
-            right_is_intra = (block_num & 1) ? v->is_intra[s->mb_x - s->mb_stride]
-                                             : mb_is_intra;
+            right_is_intra = (block_num & 1) ? (v->is_intra[s->mb_x - s->mb_stride] >> (block_num - 1))
+                                             : (mb_is_intra                         >> (block_num + 1));
             mv             = &s->current_picture.motion_val[0][s->block_index[block_num] - s->b8_stride * 2 - 2];
         }
-        if (block_is_intra || right_is_intra || mv[0][0] != mv[1][0] || mv[0][1] != mv[1][1]) {
+        if (block_is_intra & 1 || right_is_intra & 1 || mv[0][0] != mv[1][0] || mv[0][1] != mv[1][1]) {
             v->vc1dsp.vc1_h_loop_filter8(dst, linesize, v->pq);
         } else {
             idx = ((right_cbp >> 1) | block_cbp) & 5; // FIXME check
@@ -3619,7 +3619,7 @@ static int vc1_decode_p_mb(VC1Context *v)
                             v->vc1dsp.vc1_v_overlap(s->dest[dst_idx] + off, i & 4 ? s->uvlinesize : s->linesize);
                     }
                     block_cbp   |= 0xF << (i << 2);
-                    block_intra |= 1;
+                    block_intra |= 1 << i;
                 } else if (val) {
                     pat = vc1_decode_p_block(v, s->block[i], i, mquant, ttmb, first_block,
                                              s->dest[dst_idx] + off, (i & 4) ? s->uvlinesize : s->linesize,
@@ -3730,7 +3730,7 @@ static int vc1_decode_p_mb(VC1Context *v)
                             v->vc1dsp.vc1_v_overlap(s->dest[dst_idx] + off, i & 4 ? s->uvlinesize : s->linesize);
                     }
                     block_cbp   |= 0xF << (i << 2);
-                    block_intra |= 1;
+                    block_intra |= 1 << i;
                 } else if (is_coded[i]) {
                     pat = vc1_decode_p_block(v, s->block[i], i, mquant, ttmb,
                                              first_block, s->dest[dst_idx] + off,
@@ -3835,7 +3835,8 @@ static int vc1_decode_p_mb_intfr(VC1Context *v)
                 s->current_picture.motion_val[1][s->block_index[i]][1] = 0;
             }
             s->current_picture.mb_type[mb_pos]                     = MB_TYPE_INTRA;
-            s->mb_intra = v->is_intra[s->mb_x] = 1;
+            s->mb_intra = 1;
+            v->is_intra[s->mb_x] = 0x3F;
             for (i = 0; i < 6; i++)
                 v->mb_type[0][s->block_index[i]] = 1;
             fieldtx = v->fieldtx_plane[mb_pos] = get_bits1(gb);
@@ -4004,7 +4005,8 @@ static int vc1_decode_p_mb_intfi(VC1Context *v)
 
     idx_mbmode = get_vlc2(gb, v->mbmode_vlc->table, VC1_IF_MBMODE_VLC_BITS, 2);
     if (idx_mbmode <= 1) { // intra MB
-        s->mb_intra = v->is_intra[s->mb_x] = 1;
+        s->mb_intra = 1;
+        v->is_intra[s->mb_x] = 0x3F;
         s->current_picture.motion_val[1][s->block_index[0] + v->blocks_off][0] = 0;
         s->current_picture.motion_val[1][s->block_index[0] + v->blocks_off][1] = 0;
         s->current_picture.mb_type[mb_pos + v->mb_off] = MB_TYPE_INTRA;
@@ -4281,7 +4283,8 @@ static void vc1_decode_b_mb_intfi(VC1Context *v)
 
     idx_mbmode = get_vlc2(gb, v->mbmode_vlc->table, VC1_IF_MBMODE_VLC_BITS, 2);
     if (idx_mbmode <= 1) { // intra MB
-        s->mb_intra = v->is_intra[s->mb_x] = 1;
+        s->mb_intra = 1;
+        v->is_intra[s->mb_x] = 0x3F;
         s->current_picture.motion_val[1][s->block_index[0]][0] = 0;
         s->current_picture.motion_val[1][s->block_index[0]][1] = 0;
         s->current_picture.mb_type[mb_pos + v->mb_off]         = MB_TYPE_INTRA;
@@ -4507,7 +4510,8 @@ static int vc1_decode_b_mb_intfr(VC1Context *v)
             s->mv[1][i][1] = s->current_picture.motion_val[1][s->block_index[i]][1] = 0;
         }
         s->current_picture.mb_type[mb_pos] = MB_TYPE_INTRA;
-        s->mb_intra = v->is_intra[s->mb_x] = 1;
+        s->mb_intra = 1;
+        v->is_intra[s->mb_x] = 0x3F;
         for (i = 0; i < 6; i++)
             v->mb_type[0][s->block_index[i]] = 1;
         fieldtx = v->fieldtx_plane[mb_pos] = get_bits1(gb);
diff --git a/tests/ref/fate/vc1_sa00040 b/tests/ref/fate/vc1_sa00040
index bc0f99224b..79bff27943 100644
--- a/tests/ref/fate/vc1_sa00040
+++ b/tests/ref/fate/vc1_sa00040
@@ -9,8 +9,8 @@
 0,          8,          8,        1,    38016, 0xc15f4368
 0,          9,          9,        1,    38016, 0xd1bd47a8
 0,         10,         10,        1,    38016, 0xd1bd47a8
-0,         11,         11,        1,    38016, 0xe1e621ca
-0,         12,         12,        1,    38016, 0xe1e621ca
-0,         13,         13,        1,    38016, 0xe1e621ca
-0,         14,         14,        1,    38016, 0xe1e621ca
-0,         15,         15,        1,    38016, 0xe1e621ca
+0,         11,         11,        1,    38016, 0xe1e821ca
+0,         12,         12,        1,    38016, 0xe1e821ca
+0,         13,         13,        1,    38016, 0xe1e821ca
+0,         14,         14,        1,    38016, 0xe1e821ca
+0,         15,         15,        1,    38016, 0xe1e821ca
diff --git a/tests/ref/fate/vc1_sa10091 b/tests/ref/fate/vc1_sa10091
index 0f44e1da2f..33326d6a63 100644
--- a/tests/ref/fate/vc1_sa10091
+++ b/tests/ref/fate/vc1_sa10091
@@ -9,23 +9,23 @@
 0,          8,          8,        1,   518400, 0x70d9a891
 0,          9,          9,        1,   518400, 0x70d9a891
 0,         10,         10,        1,   518400, 0xa461ee86
-0,         11,         11,        1,   518400, 0x77cbc6e8
-0,         12,         12,        1,   518400, 0x77cbc6e8
-0,         13,         13,        1,   518400, 0x77cbc6e8
-0,         14,         14,        1,   518400, 0x514afd2a
-0,         15,         15,        1,   518400, 0x514afd2a
-0,         16,         16,        1,   518400, 0x8c0bcaca
-0,         17,         17,        1,   518400, 0x680727c3
-0,         18,         18,        1,   518400, 0x680727c3
-0,         19,         19,        1,   518400, 0x06012382
-0,         20,         20,        1,   518400, 0x06012382
-0,         21,         21,        1,   518400, 0xcbe29fe3
-0,         22,         22,        1,   518400, 0xcbe29fe3
-0,         23,         23,        1,   518400, 0x13e9bd17
-0,         24,         24,        1,   518400, 0xc86492fd
-0,         25,         25,        1,   518400, 0x38f9ad47
-0,         26,         26,        1,   518400, 0x38f9ad47
-0,         27,         27,        1,   518400, 0x38f9ad47
-0,         28,         28,        1,   518400, 0x5c00a10a
-0,         29,         29,        1,   518400, 0x5c00a10a
-0,         30,         30,        1,   518400, 0x5c00a10a
+0,         11,         11,        1,   518400, 0x722bc6e8
+0,         12,         12,        1,   518400, 0x722bc6e8
+0,         13,         13,        1,   518400, 0x722bc6e8
+0,         14,         14,        1,   518400, 0xf752fd2c
+0,         15,         15,        1,   518400, 0xf752fd2c
+0,         16,         16,        1,   518400, 0x91abcaca
+0,         17,         17,        1,   518400, 0x572727c3
+0,         18,         18,        1,   518400, 0x572727c3
+0,         19,         19,        1,   518400, 0x24c12382
+0,         20,         20,        1,   518400, 0x24c12382
+0,         21,         21,        1,   518400, 0x9aa39fe8
+0,         22,         22,        1,   518400, 0x9aa39fe8
+0,         23,         23,        1,   518400, 0x5cb6bd19
+0,         24,         24,        1,   518400, 0x704d9300
+0,         25,         25,        1,   518400, 0x590fad49
+0,         26,         26,        1,   518400, 0x590fad49
+0,         27,         27,        1,   518400, 0x590fad49
+0,         28,         28,        1,   518400, 0x46bea10b
+0,         29,         29,        1,   518400, 0x46bea10b
+0,         30,         30,        1,   518400, 0x46bea10b
diff --git a/tests/ref/fate/vc1_sa20021 b/tests/ref/fate/vc1_sa20021
index 1c11a4596a..22989a0aed 100644
--- a/tests/ref/fate/vc1_sa20021
+++ b/tests/ref/fate/vc1_sa20021
@@ -1,61 +1,61 @@
 #tb 0: 1/25
 0,          0,          0,        1,   506880, 0x884bc093
 0,          2,          2,        1,   506880, 0x4b09548f
-0,          3,          3,        1,   506880, 0x195bbee1
-0,          4,          4,        1,   506880, 0xcad11e28
-0,          5,          5,        1,   506880, 0x25a2c499
-0,          6,          6,        1,   506880, 0x239e2689
-0,          7,          7,        1,   506880, 0x239e2689
-0,          8,          8,        1,   506880, 0x239e2689
-0,          9,          9,        1,   506880, 0x239e2689
-0,         10,         10,        1,   506880, 0x17913b35
-0,         11,         11,        1,   506880, 0x17913b35
-0,         12,         12,        1,   506880, 0xfc4c9bb4
-0,         13,         13,        1,   506880, 0xfc4c9bb4
-0,         14,         14,        1,   506880, 0x0090cbcf
-0,         15,         15,        1,   506880, 0xccc12399
-0,         16,         16,        1,   506880, 0xc0a7af29
-0,         17,         17,        1,   506880, 0xc0a7af29
-0,         18,         18,        1,   506880, 0xc524bca8
-0,         19,         19,        1,   506880, 0xc62a1156
-0,         20,         20,        1,   506880, 0xc62a1156
-0,         21,         21,        1,   506880, 0xc62a1156
-0,         22,         22,        1,   506880, 0xb5c80b5d
-0,         23,         23,        1,   506880, 0xea4f9ebf
-0,         24,         24,        1,   506880, 0xd0dc8b34
-0,         25,         25,        1,   506880, 0xd0dc8b34
-0,         26,         26,        1,   506880, 0x83338b51
-0,         27,         27,        1,   506880, 0x83338b51
-0,         28,         28,        1,   506880, 0x83338b51
-0,         29,         29,        1,   506880, 0x478ed9b6
-0,         30,         30,        1,   506880, 0x478ed9b6
-0,         31,         31,        1,   506880, 0x478ed9b6
-0,         32,         32,        1,   506880, 0x478ed9b6
-0,         33,         33,        1,   506880, 0x45e44eba
-0,         34,         34,        1,   506880, 0x45e44eba
-0,         35,         35,        1,   506880, 0x39f87da3
-0,         36,         36,        1,   506880, 0x39f87da3
-0,         37,         37,        1,   506880, 0x39f87da3
-0,         38,         38,        1,   506880, 0x693e0646
-0,         39,         39,        1,   506880, 0x693e0646
-0,         40,         40,        1,   506880, 0xd113ce59
-0,         41,         41,        1,   506880, 0x26805371
-0,         42,         42,        1,   506880, 0x3b6dd731
-0,         43,         43,        1,   506880, 0x3b6dd731
-0,         44,         44,        1,   506880, 0xed544f56
-0,         45,         45,        1,   506880, 0xe06586d1
-0,         46,         46,        1,   506880, 0xd889d381
-0,         47,         47,        1,   506880, 0xd889d381
-0,         48,         48,        1,   506880, 0xd889d381
-0,         49,         49,        1,   506880, 0xedcdb7f0
-0,         50,         50,        1,   506880, 0xedcdb7f0
-0,         51,         51,        1,   506880, 0xedcdb7f0
-0,         52,         52,        1,   506880, 0xedcdb7f0
-0,         53,         53,        1,   506880, 0x72f4b21e
-0,         54,         54,        1,   506880, 0xd148430c
-0,         55,         55,        1,   506880, 0xd2a5ac39
-0,         56,         56,        1,   506880, 0xd2a5ac39
-0,         57,         57,        1,   506880, 0xd2a5ac39
-0,         58,         58,        1,   506880, 0xd2a5ac39
-0,         59,         59,        1,   506880, 0xd2a5ac39
-0,         60,         60,        1,   506880, 0x65c0f0e0
+0,          3,          3,        1,   506880, 0x195cbee1
+0,          4,          4,        1,   506880, 0xc8141e28
+0,          5,          5,        1,   506880, 0xb170c49b
+0,          6,          6,        1,   506880, 0x2782268a
+0,          7,          7,        1,   506880, 0x2782268a
+0,          8,          8,        1,   506880, 0x2782268a
+0,          9,          9,        1,   506880, 0x2782268a
+0,         10,         10,        1,   506880, 0xe6803b32
+0,         11,         11,        1,   506880, 0xe6803b32
+0,         12,         12,        1,   506880, 0xa5ef9baf
+0,         13,         13,        1,   506880, 0xa5ef9baf
+0,         14,         14,        1,   506880, 0x46e8cbcb
+0,         15,         15,        1,   506880, 0x28a2239b
+0,         16,         16,        1,   506880, 0x7667af2f
+0,         17,         17,        1,   506880, 0x7667af2f
+0,         18,         18,        1,   506880, 0x8011bcaf
+0,         19,         19,        1,   506880, 0xd422115b
+0,         20,         20,        1,   506880, 0xd422115b
+0,         21,         21,        1,   506880, 0xd422115b
+0,         22,         22,        1,   506880, 0xbcee0b5b
+0,         23,         23,        1,   506880, 0x08fe9ec8
+0,         24,         24,        1,   506880, 0xc8fb8b37
+0,         25,         25,        1,   506880, 0xc8fb8b37
+0,         26,         26,        1,   506880, 0x2c698b52
+0,         27,         27,        1,   506880, 0x2c698b52
+0,         28,         28,        1,   506880, 0x2c698b52
+0,         29,         29,        1,   506880, 0x2b4ad9bc
+0,         30,         30,        1,   506880, 0x2b4ad9bc
+0,         31,         31,        1,   506880, 0x2b4ad9bc
+0,         32,         32,        1,   506880, 0x2b4ad9bc
+0,         33,         33,        1,   506880, 0x92e84ebb
+0,         34,         34,        1,   506880, 0x92e84ebb
+0,         35,         35,        1,   506880, 0xdb877da3
+0,         36,         36,        1,   506880, 0xdb877da3
+0,         37,         37,        1,   506880, 0xdb877da3
+0,         38,         38,        1,   506880, 0x44610654
+0,         39,         39,        1,   506880, 0x44610654
+0,         40,         40,        1,   506880, 0xe254ce67
+0,         41,         41,        1,   506880, 0xa6085385
+0,         42,         42,        1,   506880, 0x2d45d744
+0,         43,         43,        1,   506880, 0x2d45d744
+0,         44,         44,        1,   506880, 0x6e684f51
+0,         45,         45,        1,   506880, 0xe96186cf
+0,         46,         46,        1,   506880, 0xb535d369
+0,         47,         47,        1,   506880, 0xb535d369
+0,         48,         48,        1,   506880, 0xb535d369
+0,         49,         49,        1,   506880, 0xeed0b7e0
+0,         50,         50,        1,   506880, 0xeed0b7e0
+0,         51,         51,        1,   506880, 0xeed0b7e0
+0,         52,         52,        1,   506880, 0xeed0b7e0
+0,         53,         53,        1,   506880, 0x8789b20b
+0,         54,         54,        1,   506880, 0x0a0f42fb
+0,         55,         55,        1,   506880, 0x09bbac2d
+0,         56,         56,        1,   506880, 0x09bbac2d
+0,         57,         57,        1,   506880, 0x09bbac2d
+0,         58,         58,        1,   506880, 0x09bbac2d
+0,         59,         59,        1,   506880, 0x09bbac2d
+0,         60,         60,        1,   506880, 0xda77f0df