avcodec/huffman/ff_huff_gen_len_table: support skiping stat=0 entries

This is probably not the simplest solution but as this is needed for a bugfix,
simplification is left for later.

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

4 files changed, 19 insertions, 10 deletions

diff --git a/libavcodec/huffman.c b/libavcodec/huffman.c
index 0fc2055dbe..2866eef310 100644
--- a/libavcodec/huffman.c
+++ b/libavcodec/huffman.c
@@ -52,12 +52,14 @@ static void heap_sift(HeapElem *h, int root, int size)
     }
 }
 
-int ff_huff_gen_len_table(uint8_t *dst, const uint64_t *stats, int size)
+int ff_huff_gen_len_table(uint8_t *dst, const uint64_t *stats, int stats_size, int skip0)
 {
-    HeapElem *h  = av_malloc_array(sizeof(*h), size);
-    int *up      = av_malloc_array(sizeof(*up) * 2, size);
-    uint8_t *len = av_malloc_array(sizeof(*len) * 2, size);
+    HeapElem *h  = av_malloc_array(sizeof(*h), stats_size);
+    int *up      = av_malloc_array(sizeof(*up) * 2, stats_size);
+    uint8_t *len = av_malloc_array(sizeof(*len) * 2, stats_size);
+    uint16_t *map= av_malloc_array(sizeof(*map), stats_size);
     int offset, i, next;
+    int size = 0;
     int ret = 0;
 
     if (!h || !up || !len) {
@@ -65,10 +67,16 @@ int ff_huff_gen_len_table(uint8_t *dst, const uint64_t *stats, int size)
         goto end;
     }
 
+    for (i = 0; i<stats_size; i++) {
+        dst[i] = 255;
+        if (stats[i] || !skip0)
+            map[size++] = i;
+    }
+
     for (offset = 1; ; offset <<= 1) {
         for (i=0; i < size; i++) {
             h[i].name = i;
-            h[i].val = (stats[i] << 14) + offset;
+            h[i].val = (stats[map[i]] << 14) + offset;
         }
         for (i = size / 2 - 1; i >= 0; i--)
             heap_sift(h, i, size);
@@ -89,8 +97,8 @@ int ff_huff_gen_len_table(uint8_t *dst, const uint64_t *stats, int size)
         for (i = 2 * size - 3; i >= size; i--)
             len[i] = len[up[i]] + 1;
         for (i = 0; i < size; i++) {
-            dst[i] = len[up[i]] + 1;
-            if (dst[i] >= 32) break;
+            dst[map[i]] = len[up[i]] + 1;
+            if (dst[map[i]] >= 32) break;
         }
         if (i==size) break;
     }
@@ -98,6 +106,7 @@ end:
     av_free(h);
     av_free(up);
     av_free(len);
+    av_free(map);
     return ret;
 }
 
diff --git a/libavcodec/huffman.h b/libavcodec/huffman.h
index b1ace62201..6ab23ae216 100644
--- a/libavcodec/huffman.h
+++ b/libavcodec/huffman.h
@@ -43,6 +43,6 @@ typedef int (*HuffCmp)(const void *va, const void *vb);
 int ff_huff_build_tree(AVCodecContext *avctx, VLC *vlc, int nb_codes, int nb_bits,
                        Node *nodes, HuffCmp cmp, int flags);
 
-int ff_huff_gen_len_table(uint8_t *dst, const uint64_t *stats, int n);
+int ff_huff_gen_len_table(uint8_t *dst, const uint64_t *stats, int n, int skip0);
 
 #endif /* AVCODEC_HUFFMAN_H */
diff --git a/libavcodec/huffyuvenc.c b/libavcodec/huffyuvenc.c
index e650dee7d5..0ed2c1d127 100644
--- a/libavcodec/huffyuvenc.c
+++ b/libavcodec/huffyuvenc.c
@@ -196,7 +196,7 @@ static int store_huffman_tables(HYuvContext *s, uint8_t *buf)
         count = 1 + s->alpha + 2*s->chroma;
 
     for (i = 0; i < count; i++) {
-        if ((ret = ff_huff_gen_len_table(s->len[i], s->stats[i], s->vlc_n)) < 0)
+        if ((ret = ff_huff_gen_len_table(s->len[i], s->stats[i], s->vlc_n, 0)) < 0)
             return ret;
 
         if (ff_huffyuv_generate_bits_table(s->bits[i], s->len[i], s->vlc_n) < 0) {
diff --git a/libavcodec/utvideoenc.c b/libavcodec/utvideoenc.c
index 0a96fed0ac..23ca28c437 100644
--- a/libavcodec/utvideoenc.c
+++ b/libavcodec/utvideoenc.c
@@ -468,7 +468,7 @@ static int encode_plane(AVCodecContext *avctx, uint8_t *src,
     }
 
     /* Calculate huffman lengths */
-    if ((ret = ff_huff_gen_len_table(lengths, counts, 256)) < 0)
+    if ((ret = ff_huff_gen_len_table(lengths, counts, 256, 0)) < 0)
         return ret;
 
     /*