avcodec/vvcdec: inter, wait reference with a different resolution

For RPR, the current frame may reference a frame with a different resolution.
Therefore, we need to consider frame scaling when we wait for reference pixels.

4 files changed, 68 insertions, 3 deletions

diff --git a/libavcodec/vvc/dec.c b/libavcodec/vvc/dec.c
index 4e882f93c9..e53ad4e607 100644
--- a/libavcodec/vvc/dec.c
+++ b/libavcodec/vvc/dec.c
@@ -576,6 +576,11 @@ static int ref_frame(VVCFrame *dst, const VVCFrame *src)
 
     dst->poc = src->poc;
     dst->ctb_count = src->ctb_count;
+
+    dst->scaling_win = src->scaling_win;
+    dst->ref_width   = src->ref_width;
+    dst->ref_height  = src->ref_height;
+
     dst->flags = src->flags;
     dst->sequence = src->sequence;
 
diff --git a/libavcodec/vvc/dec.h b/libavcodec/vvc/dec.h
index 6f14cc1860..1e0b76f283 100644
--- a/libavcodec/vvc/dec.h
+++ b/libavcodec/vvc/dec.h
@@ -46,6 +46,10 @@ typedef struct VVCRefPic {
     struct VVCFrame *ref;
     int poc;
     int is_lt;                  // is long term reference
+
+    // for RPR
+    int is_scaled;              ///< RprConstraintsActiveFlag
+    int scale[2];               ///< RefPicScale[]
 } VVCRefPic;
 
 typedef struct RefPicList {
@@ -57,6 +61,13 @@ typedef struct RefPicListTab {
     RefPicList refPicList[2];
 } RefPicListTab;
 
+typedef struct VVCWindow {
+    int16_t left_offset;
+    int16_t right_offset;
+    int16_t top_offset;
+    int16_t bottom_offset;
+} VVCWindow;
+
 typedef struct VVCFrame {
     struct AVFrame *frame;
 
@@ -71,6 +82,12 @@ typedef struct VVCFrame {
 
     int poc;
 
+    //for RPR
+    VVCWindow scaling_win;                      ///< pps_scaling_win_left_offset * SubWithC,  pps_scaling_win_right_offset  * SubWithC,
+                                                ///< pps_scaling_win_top_offset  * SubHeigtC, pps_scaling_win_bottom_offset * SubHiehgtC
+    int ref_width;                              ///< CurrPicScalWinWidthL
+    int ref_height;                             ///< CurrPicScalWinHeightL
+
     struct VVCFrame *collocated_ref;
 
     struct FrameProgress *progress;             ///< RefStruct reference
diff --git a/libavcodec/vvc/refs.c b/libavcodec/vvc/refs.c
index 954db4a8c8..fb42963034 100644
--- a/libavcodec/vvc/refs.c
+++ b/libavcodec/vvc/refs.c
@@ -114,10 +114,12 @@ static FrameProgress *alloc_progress(void)
 
 static VVCFrame *alloc_frame(VVCContext *s, VVCFrameContext *fc)
 {
+    const VVCSPS *sps = fc->ps.sps;
     const VVCPPS *pps = fc->ps.pps;
     for (int i = 0; i < FF_ARRAY_ELEMS(fc->DPB); i++) {
         int ret;
         VVCFrame *frame = &fc->DPB[i];
+        VVCWindow *win = &frame->scaling_win;
         if (frame->frame->buf[0])
             continue;
 
@@ -144,6 +146,13 @@ static VVCFrame *alloc_frame(VVCContext *s, VVCFrameContext *fc)
         for (int j = 0; j < frame->ctb_count; j++)
             frame->rpl_tab[j] = frame->rpl;
 
+        win->left_offset   = pps->r->pps_scaling_win_left_offset   << sps->hshift[CHROMA];
+        win->right_offset  = pps->r->pps_scaling_win_right_offset  << sps->hshift[CHROMA];
+        win->top_offset    = pps->r->pps_scaling_win_top_offset    << sps->vshift[CHROMA];
+        win->bottom_offset = pps->r->pps_scaling_win_bottom_offset << sps->vshift[CHROMA];
+        frame->ref_width   = pps->r->pps_pic_width_in_luma_samples  - win->left_offset   - win->right_offset;
+        frame->ref_height  = pps->r->pps_pic_height_in_luma_samples - win->bottom_offset - win->top_offset;
+
         frame->progress = alloc_progress();
         if (!frame->progress)
             goto fail;
@@ -353,6 +362,24 @@ static VVCFrame *generate_missing_ref(VVCContext *s, VVCFrameContext *fc, int po
     return frame;
 }
 
+#define CHECK_MAX(d) (frame->ref_##d * frame->sps->r->sps_pic_##d##_max_in_luma_samples >= ref->ref_##d * (frame->pps->r->pps_pic_##d##_in_luma_samples - max))
+#define CHECK_SAMPLES(d) (frame->pps->r->pps_pic_##d##_in_luma_samples == ref->pps->r->pps_pic_##d##_in_luma_samples)
+static int check_candidate_ref(const VVCFrame *frame, const VVCRefPic *refp)
+{
+    const VVCFrame *ref = refp->ref;
+
+    if (refp->is_scaled) {
+        const int max = FFMAX(8, frame->sps->min_cb_size_y);
+        return frame->ref_width * 2 >= ref->ref_width &&
+            frame->ref_height * 2 >= ref->ref_height &&
+            frame->ref_width <= ref->ref_width * 8 &&
+            frame->ref_height <= ref->ref_height * 8 &&
+            CHECK_MAX(width) && CHECK_MAX(height);
+    }
+    return CHECK_SAMPLES(width) && CHECK_SAMPLES(height);
+}
+
+#define RPR_SCALE(f) (((ref->f << 14) + (fc->ref->f >> 1)) / fc->ref->f)
 /* add a reference with the given poc to the list and mark it as used in DPB */
 static int add_candidate_ref(VVCContext *s, VVCFrameContext *fc, RefPicList *list,
                              int poc, int ref_flag, uint8_t use_msb)
@@ -372,6 +399,18 @@ static int add_candidate_ref(VVCContext *s, VVCFrameContext *fc, RefPicList *lis
     refp->poc = poc;
     refp->ref = ref;
     refp->is_lt = ref_flag & VVC_FRAME_FLAG_LONG_REF;
+    refp->is_scaled = ref->sps->r->sps_num_subpics_minus1 != fc->ref->sps->r->sps_num_subpics_minus1||
+        memcmp(&ref->scaling_win, &fc->ref->scaling_win, sizeof(ref->scaling_win)) ||
+        ref->pps->r->pps_pic_width_in_luma_samples != fc->ref->pps->r->pps_pic_width_in_luma_samples ||
+        ref->pps->r->pps_pic_height_in_luma_samples != fc->ref->pps->r->pps_pic_height_in_luma_samples;
+
+    if (!check_candidate_ref(fc->ref, refp))
+        return AVERROR_INVALIDDATA;
+
+    if (refp->is_scaled) {
+        refp->scale[0] = RPR_SCALE(ref_width);
+        refp->scale[1] = RPR_SCALE(ref_height);
+    }
     list->nb_refs++;
 
     mark_ref(ref, ref_flag);
diff --git a/libavcodec/vvc/thread.c b/libavcodec/vvc/thread.c
index 2654b40058..8777d380bf 100644
--- a/libavcodec/vvc/thread.c
+++ b/libavcodec/vvc/thread.c
@@ -293,10 +293,14 @@ static void schedule_inter(VVCContext *s, VVCFrameContext *fc, const SliceContex
         CTU *ctu = fc->tab.ctus + rs;
         for (int lx = 0; lx < 2; lx++) {
             for (int i = 0; i < sh->r->num_ref_idx_active[lx]; i++) {
-                const int y = ctu->max_y[lx][i];
-                VVCFrame *ref = sc->rpl[lx].refs[i].ref;
-                if (ref && y >= 0)
+                int y = ctu->max_y[lx][i];
+                VVCRefPic *refp = sc->rpl[lx].refs + i;
+                VVCFrame *ref   = refp->ref;
+                if (ref && y >= 0) {
+                    if (refp->is_scaled)
+                        y = y * refp->scale[1] >> 14;
                     add_progress_listener(ref, &t->listener[lx][i], t, s, VVC_PROGRESS_PIXEL, y + LUMA_EXTRA_AFTER);
+                }
             }
         }
     }