avcodec/hevc: Simplify get_qPy_pred()

Fixes use of uninitialized memory
Fixes: 93728afd9aa074ba14a09bfd93a632fd-asan_static-oob_124a17d_1445_cov_1021181966_DBLK_D_VIXS_1.bit
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <[email protected]>
(cherry picked from commit 64278039e55ffc88d231a8d760ecc257a120760a)

Conflicts:

	libavcodec/hevc_filter.c

3 files changed, 18 insertions, 40 deletions

diff --git a/libavcodec/hevc.c b/libavcodec/hevc.c
index f8438d4bc3..f27eddbdc5 100644
--- a/libavcodec/hevc.c
+++ b/libavcodec/hevc.c
@@ -1549,6 +1549,7 @@ static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size)
     int x_cb             = x0 >> log2_min_cb_size;
     int y_cb             = y0 >> log2_min_cb_size;
     int x, y, ret;
+    int qp_block_mask = (1<<(s->sps->log2_ctb_size - s->pps->diff_cu_qp_delta_depth)) - 1;
 
     lc->cu.x            = x0;
     lc->cu.y            = y0;
@@ -1686,6 +1687,11 @@ static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size)
         x += min_cb_width;
     }
 
+    if(((x0 + (1<<log2_cb_size)) & qp_block_mask) == 0 &&
+       ((y0 + (1<<log2_cb_size)) & qp_block_mask) == 0) {
+        lc->qPy_pred = lc->qp_y;
+    }
+
     set_ct_depth(s, x0, y0, log2_cb_size, lc->ct.depth);
 
     return 0;
@@ -1697,6 +1703,7 @@ static int hls_coding_quadtree(HEVCContext *s, int x0, int y0,
     HEVCLocalContext *lc = s->HEVClc;
     const int cb_size    = 1 << log2_cb_size;
     int ret;
+    int qp_block_mask = (1<<(s->sps->log2_ctb_size - s->pps->diff_cu_qp_delta_depth)) - 1;
 
     lc->ct.depth = cb_depth;
     if ((x0 + cb_size <= s->sps->width) &&
@@ -1736,8 +1743,15 @@ static int hls_coding_quadtree(HEVCContext *s, int x0, int y0,
         }
         if (more_data && x1 < s->sps->width &&
             y1 < s->sps->height) {
-            return hls_coding_quadtree(s, x1, y1, log2_cb_size - 1, cb_depth + 1);
+            more_data = hls_coding_quadtree(s, x1, y1, log2_cb_size - 1, cb_depth + 1);
+            if (more_data < 0)
+                return more_data;
         }
+
+        if(((x0 + (1<<log2_cb_size)) & qp_block_mask) == 0 &&
+            ((y0 + (1<<log2_cb_size)) & qp_block_mask) == 0)
+            lc->qPy_pred = lc->qp_y;
+
         if (more_data)
             return ((x1 + cb_size_split) < s->sps->width ||
                     (y1 + cb_size_split) < s->sps->height);
diff --git a/libavcodec/hevc.h b/libavcodec/hevc.h
index 0d4f3e6373..1a115cf40e 100644
--- a/libavcodec/hevc.h
+++ b/libavcodec/hevc.h
@@ -733,6 +733,8 @@ typedef struct HEVCLocalContext {
     int8_t qp_y;
     int8_t curr_qp_y;
 
+    int qPy_pred;
+
     TransformUnit tu;
 
     uint8_t ctb_left_flag;
diff --git a/libavcodec/hevc_filter.c b/libavcodec/hevc_filter.c
index d040406008..bad5359e7d 100644
--- a/libavcodec/hevc_filter.c
+++ b/libavcodec/hevc_filter.c
@@ -93,45 +93,7 @@ static int get_qPy_pred(HEVCContext *s, int xC, int yC, int xBase, int yBase, in
         lc->first_qp_group = !lc->tu.is_cu_qp_delta_coded;
         qPy_pred = s->sh.slice_qp;
     } else {
-        qPy_pred = lc->qp_y;
-        if (log2_cb_size < s->sps->log2_ctb_size - s->pps->diff_cu_qp_delta_depth) {
-            static const int offsetX[8][8] = {
-                    {-1, 1, 3, 1, 7, 1, 3, 1},
-                    { 0, 0, 0, 0, 0, 0, 0, 0},
-                    { 1, 3, 1, 3, 1, 3, 1, 3},
-                    { 2, 2, 2, 2, 2, 2, 2, 2},
-                    { 3, 5, 7, 5, 3, 5, 7, 5},
-                    { 4, 4, 4, 4, 4, 4, 4, 4},
-                    { 5, 7, 5, 7, 5, 7, 5, 7},
-                    { 6, 6, 6, 6, 6, 6, 6, 6}
-            };
-            static const int offsetY[8][8] = {
-                    { 7, 0, 1, 2, 3, 4, 5, 6},
-                    { 0, 1, 2, 3, 4, 5, 6, 7},
-                    { 1, 0, 3, 2, 5, 4, 7, 6},
-                    { 0, 1, 2, 3, 4, 5, 6, 7},
-                    { 3, 0, 1, 2, 7, 4, 5, 6},
-                    { 0, 1, 2, 3, 4, 5, 6, 7},
-                    { 1, 0, 3, 2, 5, 4, 7, 6},
-                    { 0, 1, 2, 3, 4, 5, 6, 7}
-            };
-            int xC0b = (xC - (xC & ctb_size_mask)) >> s->sps->log2_min_cb_size;
-            int yC0b = (yC - (yC & ctb_size_mask)) >> s->sps->log2_min_cb_size;
-            int idxX = (xQgBase & ctb_size_mask)   >> s->sps->log2_min_cb_size;
-            int idxY = (yQgBase & ctb_size_mask)   >> s->sps->log2_min_cb_size;
-            int idx_mask = ctb_size_mask >> s->sps->log2_min_cb_size;
-            int x, y;
-
-            x = FFMIN(xC0b +  offsetX[idxX][idxY],             min_cb_width  - 1);
-            y = FFMIN(yC0b + (offsetY[idxX][idxY] & idx_mask), min_cb_height - 1);
-
-            if (xC0b == (lc->start_of_tiles_x >> s->sps->log2_min_cb_size) &&
-                offsetX[idxX][idxY] == -1) {
-                x = (lc->end_of_tiles_x >> s->sps->log2_min_cb_size) - 1;
-                y = yC0b - 1;
-            }
-            qPy_pred = s->qp_y_tab[y * min_cb_width + x];
-        }
+        qPy_pred = lc->qPy_pred;
     }
 
     // qPy_a