avcodec/h261enc: Fix ac_vlc_length tables

These tables are supposed to contain the number of bits needed
to encode a given (run, level) pair. Yet the number of bits
for pairs needing the escape code was wrong (it only contained
the escape code and not the bits needed for run and level).

Furthermore, H.261 (a format with explicit end-of-block codes)
does not work well together with the RLTable API from rl.c:
The EOB code is the first one in ff_h261_rl_tcoeff's VLC table
and has a run value of zero. Therefore the result of get_rl_index()
is off by one for run == 0 and level values with explicit
(run, level) pair.

Fixing this necessitated changing the ref files of the
vsynth*-h261-trellis tests. Both filesizes as well as PSNR
decreased. If one used a qscale value of 11 for this test,
one would have received files with about the same size as
before this patch (with qscale 12), but with better PSNR.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>

4 files changed, 24 insertions, 59 deletions

diff --git a/libavcodec/h261enc.c b/libavcodec/h261enc.c
index b19830d578..a901c32e42 100644
--- a/libavcodec/h261enc.c
+++ b/libavcodec/h261enc.c
@@ -38,14 +38,15 @@
 
 #define H261_MAX_RUN   26
 #define H261_MAX_LEVEL 15
+#define H261_ESC_LEN   (6 + 6 + 8)
 
 static struct VLCLUT {
     uint8_t len;
     uint16_t code;
 } vlc_lut[H261_MAX_RUN + 1][32 /* 0..2 * H261_MAX_LEN are used */];
 
-static uint8_t uni_h261_rl_len [64*64*2*2];
-#define UNI_ENC_INDEX(last,run,level) ((last)*128*64 + (run)*128 + (level))
+static uint8_t uni_h261_rl_len     [64 * 128];
+static uint8_t uni_h261_rl_len_last[64 * 128];
 
 typedef struct H261EncContext {
     MpegEncContext s;
@@ -320,51 +321,10 @@ void ff_h261_encode_mb(MpegEncContext *s, int16_t block[6][64],
     }
 }
 
-static av_cold void init_uni_h261_rl_tab(const RLTable *rl, uint8_t *len_tab)
-{
-    int slevel, run, last;
-
-    av_assert0(MAX_LEVEL >= 64);
-    av_assert0(MAX_RUN   >= 63);
-
-    for(slevel=-64; slevel<64; slevel++){
-        if(slevel==0) continue;
-        for(run=0; run<64; run++){
-            for(last=0; last<=1; last++){
-                const int index= UNI_ENC_INDEX(last, run, slevel+64);
-                int level= slevel < 0 ? -slevel : slevel;
-                int len, code;
-
-                len_tab[index]= 100;
-
-                /* ESC0 */
-                code= get_rl_index(rl, 0, run, level);
-                len=  rl->table_vlc[code][1] + 1;
-                if(last)
-                    len += 2;
-
-                if(code!=rl->n && len < len_tab[index]){
-                    len_tab [index]= len;
-                }
-                /* ESC */
-                len = rl->table_vlc[rl->n][1];
-                if(last)
-                    len += 2;
-
-                if(len < len_tab[index]){
-                    len_tab [index]= len;
-                }
-            }
-        }
-    }
-}
-
 static av_cold void h261_encode_init_static(void)
 {
-    static uint8_t h261_rl_table_store[2][2 * MAX_RUN + MAX_LEVEL + 3];
-
-    ff_rl_init(&ff_h261_rl_tcoeff, h261_rl_table_store);
-    init_uni_h261_rl_tab(&ff_h261_rl_tcoeff, uni_h261_rl_len);
+    memset(uni_h261_rl_len,      H261_ESC_LEN, sizeof(uni_h261_rl_len));
+    memset(uni_h261_rl_len_last, H261_ESC_LEN + 2 /* EOB */, sizeof(uni_h261_rl_len_last));
 
     // The following loop is over the ordinary elements, not EOB or escape.
     for (size_t i = 1; i < FF_ARRAY_ELEMS(ff_h261_tcoeff_vlc) - 1; i++) {
@@ -375,6 +335,11 @@ static av_cold void h261_encode_init_static(void)
 
         vlc_lut[run][H261_MAX_LEVEL + level] = (struct VLCLUT){ len, code << 1 };
         vlc_lut[run][H261_MAX_LEVEL - level] = (struct VLCLUT){ len, (code << 1) | 1 };
+
+        uni_h261_rl_len     [UNI_AC_ENC_INDEX(run, 64 + level)] = len;
+        uni_h261_rl_len     [UNI_AC_ENC_INDEX(run, 64 - level)] = len;
+        uni_h261_rl_len_last[UNI_AC_ENC_INDEX(run, 64 + level)] = len + 2;
+        uni_h261_rl_len_last[UNI_AC_ENC_INDEX(run, 64 - level)] = len + 2;
     }
 }
 
@@ -398,10 +363,10 @@ av_cold int ff_h261_encode_init(MpegEncContext *s)
 
     s->min_qcoeff       = -127;
     s->max_qcoeff       = 127;
-    s->ac_esc_length    = 6+6+8;
+    s->ac_esc_length    = H261_ESC_LEN;
 
     s->intra_ac_vlc_length      = s->inter_ac_vlc_length      = uni_h261_rl_len;
-    s->intra_ac_vlc_last_length = s->inter_ac_vlc_last_length = uni_h261_rl_len + 128*64;
+    s->intra_ac_vlc_last_length = s->inter_ac_vlc_last_length = uni_h261_rl_len_last;
     ff_thread_once(&init_static_once, h261_encode_init_static);
 
     return 0;
diff --git a/tests/ref/vsynth/vsynth1-h261-trellis b/tests/ref/vsynth/vsynth1-h261-trellis
index 87b078b0d5..0cbb9b0e18 100644
--- a/tests/ref/vsynth/vsynth1-h261-trellis
+++ b/tests/ref/vsynth/vsynth1-h261-trellis
@@ -1,4 +1,4 @@
-02b4109ce5343b7ef24fb11c2635498a *tests/data/fate/vsynth1-h261-trellis.avi
-655416 tests/data/fate/vsynth1-h261-trellis.avi
-70ceba944548ba680b1101c91707ea25 *tests/data/fate/vsynth1-h261-trellis.out.rawvideo
-stddev:    8.75 PSNR: 29.28 MAXDIFF:   90 bytes:  7603200/  7603200
+9980463214cb744a7780737a16408540 *tests/data/fate/vsynth1-h261-trellis.avi
+590384 tests/data/fate/vsynth1-h261-trellis.avi
+7bbb0520a5ca26cdcea148f9997eae27 *tests/data/fate/vsynth1-h261-trellis.out.rawvideo
+stddev:    9.09 PSNR: 28.96 MAXDIFF:   93 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth2-h261-trellis b/tests/ref/vsynth/vsynth2-h261-trellis
index b9c694f016..03d17e194d 100644
--- a/tests/ref/vsynth/vsynth2-h261-trellis
+++ b/tests/ref/vsynth/vsynth2-h261-trellis
@@ -1,4 +1,4 @@
-f5e0cfc70bbe4f4048c15be88dea4378 *tests/data/fate/vsynth2-h261-trellis.avi
-249856 tests/data/fate/vsynth2-h261-trellis.avi
-15452237f6c333690d3e05f354f63196 *tests/data/fate/vsynth2-h261-trellis.out.rawvideo
-stddev:    7.10 PSNR: 31.10 MAXDIFF:   96 bytes:  7603200/  7603200
+cdf6013013b8de3fe56476178b5adc89 *tests/data/fate/vsynth2-h261-trellis.avi
+227156 tests/data/fate/vsynth2-h261-trellis.avi
+d1107f161a7d923e0ffb6aeb9c713633 *tests/data/fate/vsynth2-h261-trellis.out.rawvideo
+stddev:    7.24 PSNR: 30.93 MAXDIFF:   87 bytes:  7603200/  7603200
diff --git a/tests/ref/vsynth/vsynth_lena-h261-trellis b/tests/ref/vsynth/vsynth_lena-h261-trellis
index a306508032..75558679ec 100644
--- a/tests/ref/vsynth/vsynth_lena-h261-trellis
+++ b/tests/ref/vsynth/vsynth_lena-h261-trellis
@@ -1,4 +1,4 @@
-41522be50f14b7fff6b1fb8d10b1ff00 *tests/data/fate/vsynth_lena-h261-trellis.avi
-184586 tests/data/fate/vsynth_lena-h261-trellis.avi
-f9df8cd110a2f3d9706dd2f29a1d0a89 *tests/data/fate/vsynth_lena-h261-trellis.out.rawvideo
-stddev:    6.32 PSNR: 32.11 MAXDIFF:   89 bytes:  7603200/  7603200
+aa3e00ffeb48a23452bb1c5b0dde9e77 *tests/data/fate/vsynth_lena-h261-trellis.avi
+167808 tests/data/fate/vsynth_lena-h261-trellis.avi
+055f8805df2bb21ef6752dc944e7828d *tests/data/fate/vsynth_lena-h261-trellis.out.rawvideo
+stddev:    6.43 PSNR: 31.96 MAXDIFF:   87 bytes:  7603200/  7603200