h264: use a smaller struct for the ref lists

There is no need to store a whole H264Picture, with a full AVFrame
embedded in it. This should allow getting rid of the embedded AVFrame
later.

8 files changed, 126 insertions, 107 deletions

diff --git a/libavcodec/dxva2_h264.c b/libavcodec/dxva2_h264.c
index 4e8c4ea552..03af474892 100644
--- a/libavcodec/dxva2_h264.c
+++ b/libavcodec/dxva2_h264.c
@@ -240,7 +240,7 @@ static void fill_slice_long(AVCodecContext *avctx, DXVA_Slice_H264_Long *slice,
         unsigned i;
         for (i = 0; i < FF_ARRAY_ELEMS(slice->RefPicList[list]); i++) {
             if (list < sl->list_count && i < sl->ref_count[list]) {
-                const H264Picture *r = &sl->ref_list[list][i];
+                const H264Picture *r = sl->ref_list[list][i].parent;
                 unsigned plane;
                 unsigned index;
                 if (ctx->workaround & FF_DXVA2_WORKAROUND_INTEL_CLEARVIDEO)
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 8b73b752ad..9fc5c12fee 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -296,6 +296,17 @@ typedef struct H264Picture {
     int recovered;          ///< picture at IDR or recovery point + recovery count
 } H264Picture;
 
+typedef struct H264Ref {
+    uint8_t *data[3];
+    int linesize[3];
+
+    int reference;
+    int poc;
+    int pic_id;
+
+    H264Picture *parent;
+} H264Ref;
+
 typedef struct H264SliceContext {
     struct H264Context *h264;
     GetBitContext gb;
@@ -393,7 +404,7 @@ typedef struct H264SliceContext {
      */
     unsigned int ref_count[2];          ///< counts frames or fields, depending on current mb mode
     unsigned int list_count;
-    H264Picture ref_list[2][48];        /**< 0..15: frame refs, 16..47: mbaff field refs.
+    H264Ref ref_list[2][48];        /**< 0..15: frame refs, 16..47: mbaff field refs.
                                          *   Reordered version of default_ref_list
                                          *   according to picture reordering in slice header */
     int ref2frm[MAX_SLICES][2][64];     ///< reference to frame number lists, used in the loop filter, the first 2 are for -2,-1
@@ -585,7 +596,7 @@ typedef struct H264Context {
      */
     int max_pic_num;
 
-    H264Picture default_ref_list[2][32]; ///< base reference list for all slices of a coded picture
+    H264Ref default_ref_list[2][32]; ///< base reference list for all slices of a coded picture
     H264Picture *short_ref[32];
     H264Picture *long_ref[32];
     H264Picture *delayed_pic[MAX_DELAYED_PIC_COUNT + 2]; // FIXME size?
diff --git a/libavcodec/h264_direct.c b/libavcodec/h264_direct.c
index ae24b02f23..a7966e15e9 100644
--- a/libavcodec/h264_direct.c
+++ b/libavcodec/h264_direct.c
@@ -39,7 +39,7 @@ static int get_scale_factor(H264SliceContext *sl,
 {
     int poc0 = sl->ref_list[0][i].poc;
     int td = av_clip_int8(poc1 - poc0);
-    if (td == 0 || sl->ref_list[0][i].long_ref) {
+    if (td == 0 || sl->ref_list[0][i].parent->long_ref) {
         return 256;
     } else {
         int tb = av_clip_int8(poc - poc0);
@@ -59,7 +59,7 @@ void ff_h264_direct_dist_scale_factor(const H264Context *const h,
     if (FRAME_MBAFF(h))
         for (field = 0; field < 2; field++) {
             const int poc  = h->cur_pic_ptr->field_poc[field];
-            const int poc1 = sl->ref_list[1][0].field_poc[field];
+            const int poc1 = sl->ref_list[1][0].parent->field_poc[field];
             for (i = 0; i < 2 * sl->ref_count[0]; i++)
                 sl->dist_scale_factor_field[field][i ^ field] =
                     get_scale_factor(sl, poc, poc1, i + 16);
@@ -73,7 +73,7 @@ static void fill_colmap(const H264Context *h, H264SliceContext *sl,
                         int map[2][16 + 32], int list,
                         int field, int colfield, int mbafi)
 {
-    H264Picture *const ref1 = &sl->ref_list[1][0];
+    H264Picture *const ref1 = sl->ref_list[1][0].parent;
     int j, old_ref, rfield;
     int start  = mbafi ? 16                       : 0;
     int end    = mbafi ? 16 + 2 * sl->ref_count[0] : sl->ref_count[0];
@@ -93,7 +93,7 @@ static void fill_colmap(const H264Context *h, H264SliceContext *sl,
                 poc = (poc & ~3) + rfield + 1;
 
             for (j = start; j < end; j++) {
-                if (4 * sl->ref_list[0][j].frame_num +
+                if (4 * sl->ref_list[0][j].parent->frame_num +
                     (sl->ref_list[0][j].reference & 3) == poc) {
                     int cur_ref = mbafi ? (j - 16) ^ field : j;
                     if (ref1->mbaff)
@@ -109,16 +109,16 @@ static void fill_colmap(const H264Context *h, H264SliceContext *sl,
 
 void ff_h264_direct_ref_list_init(const H264Context *const h, H264SliceContext *sl)
 {
-    H264Picture *const ref1 = &sl->ref_list[1][0];
+    H264Ref *const ref1 = &sl->ref_list[1][0];
     H264Picture *const cur = h->cur_pic_ptr;
     int list, j, field;
     int sidx     = (h->picture_structure & 1) ^ 1;
     int ref1sidx = (ref1->reference      & 1) ^ 1;
 
-    for (list = 0; list < 2; list++) {
+    for (list = 0; list < sl->list_count; list++) {
         cur->ref_count[sidx][list] = sl->ref_count[list];
         for (j = 0; j < sl->ref_count[list]; j++)
-            cur->ref_poc[sidx][list][j] = 4 * sl->ref_list[list][j].frame_num +
+            cur->ref_poc[sidx][list][j] = 4 * sl->ref_list[list][j].parent->frame_num +
                                           (sl->ref_list[list][j].reference & 3);
     }
 
@@ -130,16 +130,20 @@ void ff_h264_direct_ref_list_init(const H264Context *const h, H264SliceContext *
     cur->mbaff = FRAME_MBAFF(h);
 
     sl->col_fieldoff = 0;
+
+    if (sl->list_count != 2 || !sl->ref_count[1])
+        return;
+
     if (h->picture_structure == PICT_FRAME) {
         int cur_poc  = h->cur_pic_ptr->poc;
-        int *col_poc = sl->ref_list[1]->field_poc;
+        int *col_poc = sl->ref_list[1][0].parent->field_poc;
         sl->col_parity = (FFABS(col_poc[0] - cur_poc) >=
                           FFABS(col_poc[1] - cur_poc));
         ref1sidx =
         sidx     = sl->col_parity;
     // FL -> FL & differ parity
     } else if (!(h->picture_structure & sl->ref_list[1][0].reference) &&
-               !sl->ref_list[1][0].mbaff) {
+               !sl->ref_list[1][0].parent->mbaff) {
         sl->col_fieldoff = 2 * sl->ref_list[1][0].reference - 3;
     }
 
@@ -192,7 +196,7 @@ static void pred_spatial_direct_motion(const H264Context *const h, H264SliceCont
 
     assert(sl->ref_list[1][0].reference & 3);
 
-    await_reference_mb_row(h, &sl->ref_list[1][0],
+    await_reference_mb_row(h, sl->ref_list[1][0].parent,
                            sl->mb_y + !!IS_INTERLACED(*mb_type));
 
 #define MB_TYPE_16x16_OR_INTRA (MB_TYPE_16x16 | MB_TYPE_INTRA4x4 | \
@@ -260,7 +264,7 @@ static void pred_spatial_direct_motion(const H264Context *const h, H264SliceCont
         return;
     }
 
-    if (IS_INTERLACED(sl->ref_list[1][0].mb_type[mb_xy])) { // AFL/AFR/FR/FL -> AFL/FL
+    if (IS_INTERLACED(sl->ref_list[1][0].parent->mb_type[mb_xy])) { // AFL/AFR/FR/FL -> AFL/FL
         if (!IS_INTERLACED(*mb_type)) {                    //     AFR/FR    -> AFL/FL
             mb_y  = (sl->mb_y & ~1) + sl->col_parity;
             mb_xy = sl->mb_x +
@@ -275,8 +279,8 @@ static void pred_spatial_direct_motion(const H264Context *const h, H264SliceCont
         if (IS_INTERLACED(*mb_type)) {                   // AFL       /FL -> AFR/FR
             mb_y           =  sl->mb_y & ~1;
             mb_xy          = (sl->mb_y & ~1) * h->mb_stride + sl->mb_x;
-            mb_type_col[0] = sl->ref_list[1][0].mb_type[mb_xy];
-            mb_type_col[1] = sl->ref_list[1][0].mb_type[mb_xy + h->mb_stride];
+            mb_type_col[0] = sl->ref_list[1][0].parent->mb_type[mb_xy];
+            mb_type_col[1] = sl->ref_list[1][0].parent->mb_type[mb_xy + h->mb_stride];
             b8_stride      = 2 + 4 * h->mb_stride;
             b4_stride     *= 6;
             if (IS_INTERLACED(mb_type_col[0]) !=
@@ -296,7 +300,7 @@ static void pred_spatial_direct_motion(const H264Context *const h, H264SliceCont
         } else {                                         //     AFR/FR    -> AFR/FR
 single_col:
             mb_type_col[0] =
-            mb_type_col[1] = sl->ref_list[1][0].mb_type[mb_xy];
+            mb_type_col[1] = sl->ref_list[1][0].parent->mb_type[mb_xy];
 
             sub_mb_type |= MB_TYPE_16x16 | MB_TYPE_DIRECT2; /* B_SUB_8x8 */
             if (!is_b8x8 && (mb_type_col[0] & MB_TYPE_16x16_OR_INTRA)) {
@@ -316,12 +320,12 @@ single_col:
         }
     }
 
-    await_reference_mb_row(h, &sl->ref_list[1][0], mb_y);
+    await_reference_mb_row(h, sl->ref_list[1][0].parent, mb_y);
 
-    l1mv0  = &sl->ref_list[1][0].motion_val[0][h->mb2b_xy[mb_xy]];
-    l1mv1  = &sl->ref_list[1][0].motion_val[1][h->mb2b_xy[mb_xy]];
-    l1ref0 = &sl->ref_list[1][0].ref_index[0][4 * mb_xy];
-    l1ref1 = &sl->ref_list[1][0].ref_index[1][4 * mb_xy];
+    l1mv0  = &sl->ref_list[1][0].parent->motion_val[0][h->mb2b_xy[mb_xy]];
+    l1mv1  = &sl->ref_list[1][0].parent->motion_val[1][h->mb2b_xy[mb_xy]];
+    l1ref0 = &sl->ref_list[1][0].parent->ref_index[0][4 * mb_xy];
+    l1ref1 = &sl->ref_list[1][0].parent->ref_index[1][4 * mb_xy];
     if (!b8_stride) {
         if (sl->mb_y & 1) {
             l1ref0 += 2;
@@ -348,7 +352,7 @@ single_col:
                            (uint8_t)ref[0], 1);
             fill_rectangle(&sl->ref_cache[1][scan8[i8 * 4]], 2, 2, 8,
                            (uint8_t)ref[1], 1);
-            if (!IS_INTRA(mb_type_col[y8]) && !sl->ref_list[1][0].long_ref &&
+            if (!IS_INTRA(mb_type_col[y8]) && !sl->ref_list[1][0].parent->long_ref &&
                 ((l1ref0[xy8] == 0 &&
                   FFABS(l1mv0[xy4][0]) <= 1 &&
                   FFABS(l1mv0[xy4][1]) <= 1) ||
@@ -379,7 +383,7 @@ single_col:
 
         fill_rectangle(&sl->ref_cache[0][scan8[0]], 4, 4, 8, (uint8_t)ref[0], 1);
         fill_rectangle(&sl->ref_cache[1][scan8[0]], 4, 4, 8, (uint8_t)ref[1], 1);
-        if (!IS_INTRA(mb_type_col[0]) && !sl->ref_list[1][0].long_ref &&
+        if (!IS_INTRA(mb_type_col[0]) && !sl->ref_list[1][0].parent->long_ref &&
             ((l1ref0[0] == 0 &&
               FFABS(l1mv0[0][0]) <= 1 &&
               FFABS(l1mv0[0][1]) <= 1) ||
@@ -417,7 +421,7 @@ single_col:
 
             assert(b8_stride == 2);
             /* col_zero_flag */
-            if (!IS_INTRA(mb_type_col[0]) && !sl->ref_list[1][0].long_ref &&
+            if (!IS_INTRA(mb_type_col[0]) && !sl->ref_list[1][0].parent->long_ref &&
                 (l1ref0[i8] == 0 ||
                  (l1ref0[i8] < 0 &&
                   l1ref1[i8] == 0 &&
@@ -475,10 +479,10 @@ static void pred_temp_direct_motion(const H264Context *const h, H264SliceContext
 
     assert(sl->ref_list[1][0].reference & 3);
 
-    await_reference_mb_row(h, &sl->ref_list[1][0],
+    await_reference_mb_row(h, sl->ref_list[1][0].parent,
                            sl->mb_y + !!IS_INTERLACED(*mb_type));
 
-    if (IS_INTERLACED(sl->ref_list[1][0].mb_type[mb_xy])) { // AFL/AFR/FR/FL -> AFL/FL
+    if (IS_INTERLACED(sl->ref_list[1][0].parent->mb_type[mb_xy])) { // AFL/AFR/FR/FL -> AFL/FL
         if (!IS_INTERLACED(*mb_type)) {                    //     AFR/FR    -> AFL/FL
             mb_y  = (sl->mb_y & ~1) + sl->col_parity;
             mb_xy = sl->mb_x +
@@ -493,8 +497,8 @@ static void pred_temp_direct_motion(const H264Context *const h, H264SliceContext
         if (IS_INTERLACED(*mb_type)) {              // AFL       /FL -> AFR/FR
             mb_y           = sl->mb_y & ~1;
             mb_xy          = sl->mb_x + (sl->mb_y & ~1) * h->mb_stride;
-            mb_type_col[0] = sl->ref_list[1][0].mb_type[mb_xy];
-            mb_type_col[1] = sl->ref_list[1][0].mb_type[mb_xy + h->mb_stride];
+            mb_type_col[0] = sl->ref_list[1][0].parent->mb_type[mb_xy];
+            mb_type_col[1] = sl->ref_list[1][0].parent->mb_type[mb_xy + h->mb_stride];
             b8_stride      = 2 + 4 * h->mb_stride;
             b4_stride     *= 6;
             if (IS_INTERLACED(mb_type_col[0]) !=
@@ -517,7 +521,7 @@ static void pred_temp_direct_motion(const H264Context *const h, H264SliceContext
         } else {                                    //     AFR/FR    -> AFR/FR
 single_col:
             mb_type_col[0]     =
-                mb_type_col[1] = sl->ref_list[1][0].mb_type[mb_xy];
+                mb_type_col[1] = sl->ref_list[1][0].parent->mb_type[mb_xy];
 
             sub_mb_type = MB_TYPE_16x16 | MB_TYPE_P0L0 | MB_TYPE_P0L1 |
                           MB_TYPE_DIRECT2;                  /* B_SUB_8x8 */
@@ -540,12 +544,12 @@ single_col:
         }
     }
 
-    await_reference_mb_row(h, &sl->ref_list[1][0], mb_y);
+    await_reference_mb_row(h, sl->ref_list[1][0].parent, mb_y);
 
-    l1mv0  = &sl->ref_list[1][0].motion_val[0][h->mb2b_xy[mb_xy]];
-    l1mv1  = &sl->ref_list[1][0].motion_val[1][h->mb2b_xy[mb_xy]];
-    l1ref0 = &sl->ref_list[1][0].ref_index[0][4 * mb_xy];
-    l1ref1 = &sl->ref_list[1][0].ref_index[1][4 * mb_xy];
+    l1mv0  = &sl->ref_list[1][0].parent->motion_val[0][h->mb2b_xy[mb_xy]];
+    l1mv1  = &sl->ref_list[1][0].parent->motion_val[1][h->mb2b_xy[mb_xy]];
+    l1ref0 = &sl->ref_list[1][0].parent->ref_index[0][4 * mb_xy];
+    l1ref1 = &sl->ref_list[1][0].parent->ref_index[1][4 * mb_xy];
     if (!b8_stride) {
         if (sl->mb_y & 1) {
             l1ref0 += 2;
@@ -566,7 +570,7 @@ single_col:
             map_col_to_list0[1] = sl->map_col_to_list0_field[sl->mb_y & 1][1];
             dist_scale_factor   = sl->dist_scale_factor_field[sl->mb_y & 1];
         }
-        ref_offset = (sl->ref_list[1][0].mbaff << 4) & (mb_type_col[0] >> 3);
+        ref_offset = (sl->ref_list[1][0].parent->mbaff << 4) & (mb_type_col[0] >> 3);
 
         if (IS_INTERLACED(*mb_type) != IS_INTERLACED(mb_type_col[0])) {
             int y_shift = 2 * !IS_INTERLACED(*mb_type);
diff --git a/libavcodec/h264_mb.c b/libavcodec/h264_mb.c
index 832ba90fd8..57d429ab02 100644
--- a/libavcodec/h264_mb.c
+++ b/libavcodec/h264_mb.c
@@ -60,12 +60,12 @@ static inline void get_lowest_part_y(const H264Context *h, H264SliceContext *sl,
 
     if (list0) {
         int ref_n = sl->ref_cache[0][scan8[n]];
-        H264Picture *ref = &sl->ref_list[0][ref_n];
+        H264Ref *ref = &sl->ref_list[0][ref_n];
 
         // Error resilience puts the current picture in the ref list.
         // Don't try to wait on these as it will cause a deadlock.
         // Fields can wait on each other, though.
-        if (ref->tf.progress->data != h->cur_pic.tf.progress->data ||
+        if (ref->parent->tf.progress->data != h->cur_pic.tf.progress->data ||
             (ref->reference & 3) != h->picture_structure) {
             my = get_lowest_part_list_y(sl, n, height, y_offset, 0);
             if (refs[0][ref_n] < 0)
@@ -76,9 +76,9 @@ static inline void get_lowest_part_y(const H264Context *h, H264SliceContext *sl,
 
     if (list1) {
         int ref_n    = sl->ref_cache[1][scan8[n]];
-        H264Picture *ref = &sl->ref_list[1][ref_n];
+        H264Ref *ref = &sl->ref_list[1][ref_n];
 
-        if (ref->tf.progress->data != h->cur_pic.tf.progress->data ||
+        if (ref->parent->tf.progress->data != h->cur_pic.tf.progress->data ||
             (ref->reference & 3) != h->picture_structure) {
             my = get_lowest_part_list_y(sl, n, height, y_offset, 1);
             if (refs[1][ref_n] < 0)
@@ -167,33 +167,33 @@ static void await_references(const H264Context *h, H264SliceContext *sl)
         for (ref = 0; ref < 48 && nrefs[list]; ref++) {
             int row = refs[list][ref];
             if (row >= 0) {
-                H264Picture *ref_pic  = &sl->ref_list[list][ref];
+                H264Ref *ref_pic  = &sl->ref_list[list][ref];
                 int ref_field         = ref_pic->reference - 1;
-                int ref_field_picture = ref_pic->field_picture;
+                int ref_field_picture = ref_pic->parent->field_picture;
                 int pic_height        = 16 * h->mb_height >> ref_field_picture;
 
                 row <<= MB_MBAFF(sl);
                 nrefs[list]--;
 
                 if (!FIELD_PICTURE(h) && ref_field_picture) { // frame referencing two fields
-                    ff_thread_await_progress(&ref_pic->tf,
+                    ff_thread_await_progress(&ref_pic->parent->tf,
                                              FFMIN((row >> 1) - !(row & 1),
                                                    pic_height - 1),
                                              1);
-                    ff_thread_await_progress(&ref_pic->tf,
+                    ff_thread_await_progress(&ref_pic->parent->tf,
                                              FFMIN((row >> 1), pic_height - 1),
                                              0);
                 } else if (FIELD_PICTURE(h) && !ref_field_picture) { // field referencing one field of a frame
-                    ff_thread_await_progress(&ref_pic->tf,
+                    ff_thread_await_progress(&ref_pic->parent->tf,
                                              FFMIN(row * 2 + ref_field,
                                                    pic_height - 1),
                                              0);
                 } else if (FIELD_PICTURE(h)) {
-                    ff_thread_await_progress(&ref_pic->tf,
+                    ff_thread_await_progress(&ref_pic->parent->tf,
                                              FFMIN(row, pic_height - 1),
                                              ref_field);
                 } else {
-                    ff_thread_await_progress(&ref_pic->tf,
+                    ff_thread_await_progress(&ref_pic->parent->tf,
                                              FFMIN(row, pic_height - 1),
                                              0);
                 }
@@ -202,7 +202,7 @@ static void await_references(const H264Context *h, H264SliceContext *sl)
 }
 
 static av_always_inline void mc_dir_part(const H264Context *h, H264SliceContext *sl,
-                                         H264Picture *pic,
+                                         H264Ref *pic,
                                          int n, int square, int height,
                                          int delta, int list,
                                          uint8_t *dest_y, uint8_t *dest_cb,
@@ -216,7 +216,7 @@ static av_always_inline void mc_dir_part(const H264Context *h, H264SliceContext
     int my            = sl->mv_cache[list][scan8[n]][1] + src_y_offset * 8;
     const int luma_xy = (mx & 3) + ((my & 3) << 2);
     ptrdiff_t offset  = ((mx >> 2) << pixel_shift) + (my >> 2) * sl->mb_linesize;
-    uint8_t *src_y    = pic->f.data[0] + offset;
+    uint8_t *src_y    = pic->data[0] + offset;
     uint8_t *src_cb, *src_cr;
     int extra_width  = 0;
     int extra_height = 0;
@@ -253,7 +253,7 @@ static av_always_inline void mc_dir_part(const H264Context *h, H264SliceContext
         return;
 
     if (chroma_idc == 3 /* yuv444 */) {
-        src_cb = pic->f.data[1] + offset;
+        src_cb = pic->data[1] + offset;
         if (emu) {
             h->vdsp.emulated_edge_mc(sl->edge_emu_buffer,
                                      src_cb - (2 << pixel_shift) - 2 * sl->mb_linesize,
@@ -267,7 +267,7 @@ static av_always_inline void mc_dir_part(const H264Context *h, H264SliceContext
         if (!square)
             qpix_op[luma_xy](dest_cb + delta, src_cb + delta, sl->mb_linesize);
 
-        src_cr = pic->f.data[2] + offset;
+        src_cr = pic->data[2] + offset;
         if (emu) {
             h->vdsp.emulated_edge_mc(sl->edge_emu_buffer,
                                      src_cr - (2 << pixel_shift) - 2 * sl->mb_linesize,
@@ -290,9 +290,9 @@ static av_always_inline void mc_dir_part(const H264Context *h, H264SliceContext
         emu |= (my >> 3) < 0 || (my >> 3) + 8 >= (pic_height >> 1);
     }
 
-    src_cb = pic->f.data[1] + ((mx >> 3) << pixel_shift) +
+    src_cb = pic->data[1] + ((mx >> 3) << pixel_shift) +
              (my >> ysh) * sl->mb_uvlinesize;
-    src_cr = pic->f.data[2] + ((mx >> 3) << pixel_shift) +
+    src_cr = pic->data[2] + ((mx >> 3) << pixel_shift) +
              (my >> ysh) * sl->mb_uvlinesize;
 
     if (emu) {
@@ -348,7 +348,7 @@ static av_always_inline void mc_part_std(const H264Context *h, H264SliceContext
     y_offset += 8 * (sl->mb_y >> MB_FIELD(sl));
 
     if (list0) {
-        H264Picture *ref = &sl->ref_list[0][sl->ref_cache[0][scan8[n]]];
+        H264Ref *ref = &sl->ref_list[0][sl->ref_cache[0][scan8[n]]];
         mc_dir_part(h, sl, ref, n, square, height, delta, 0,
                     dest_y, dest_cb, dest_cr, x_offset, y_offset,
                     qpix_op, chroma_op, pixel_shift, chroma_idc);
@@ -358,7 +358,7 @@ static av_always_inline void mc_part_std(const H264Context *h, H264SliceContext
     }
 
     if (list1) {
-        H264Picture *ref = &sl->ref_list[1][sl->ref_cache[1][scan8[n]]];
+        H264Ref *ref = &sl->ref_list[1][sl->ref_cache[1][scan8[n]]];
         mc_dir_part(h, sl, ref, n, square, height, delta, 1,
                     dest_y, dest_cb, dest_cr, x_offset, y_offset,
                     qpix_op, chroma_op, pixel_shift, chroma_idc);
@@ -451,7 +451,7 @@ static av_always_inline void mc_part_weighted(const H264Context *h, H264SliceCon
     } else {
         int list     = list1 ? 1 : 0;
         int refn     = sl->ref_cache[list][scan8[n]];
-        H264Picture *ref = &sl->ref_list[list][refn];
+        H264Ref *ref = &sl->ref_list[list][refn];
         mc_dir_part(h, sl, ref, n, square, height, delta, list,
                     dest_y, dest_cb, dest_cr, x_offset, y_offset,
                     qpix_put, chroma_put, pixel_shift, chroma_idc);
@@ -483,7 +483,7 @@ static av_always_inline void prefetch_motion(const H264Context *h, H264SliceCont
     if (refn >= 0) {
         const int mx  = (sl->mv_cache[list][scan8[0]][0] >> 2) + 16 * sl->mb_x + 8;
         const int my  = (sl->mv_cache[list][scan8[0]][1] >> 2) + 16 * sl->mb_y;
-        uint8_t **src = sl->ref_list[list][refn].f.data;
+        uint8_t **src = sl->ref_list[list][refn].data;
         int off       = (mx << pixel_shift) +
                         (my + (sl->mb_x & 3) * 4) * sl->mb_linesize +
                         (64 << pixel_shift);
diff --git a/libavcodec/h264_picture.c b/libavcodec/h264_picture.c
index 7217f7f953..5792f77c7c 100644
--- a/libavcodec/h264_picture.c
+++ b/libavcodec/h264_picture.c
@@ -187,9 +187,9 @@ int ff_h264_field_end(H264Context *h, H264SliceContext *sl, int in_setup)
     if (!FIELD_PICTURE(h)) {
         h264_set_erpic(&sl->er.cur_pic, h->cur_pic_ptr);
         h264_set_erpic(&sl->er.last_pic,
-                       sl->ref_count[0] ? &sl->ref_list[0][0] : NULL);
+                       sl->ref_count[0] ? sl->ref_list[0][0].parent : NULL);
         h264_set_erpic(&sl->er.next_pic,
-                       sl->ref_count[1] ? &sl->ref_list[1][0] : NULL);
+                       sl->ref_count[1] ? sl->ref_list[1][0].parent : NULL);
         ff_er_frame_end(&sl->er);
     }
 #endif /* CONFIG_ERROR_RESILIENCE */
diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index 04cfa9eec3..5782c8e271 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -35,31 +35,34 @@
 
 #include <assert.h>
 
-#define COPY_PICTURE(dst, src) \
-do {\
-    *(dst) = *(src);\
-    (dst)->f.extended_data = (dst)->f.data;\
-    (dst)->tf.f = &(dst)->f;\
-} while (0)
-
-
-static void pic_as_field(H264Picture *pic, const int parity){
+static void pic_as_field(H264Ref *pic, const int parity)
+{
     int i;
-    for (i = 0; i < 4; ++i) {
+    for (i = 0; i < FF_ARRAY_ELEMS(pic->data); ++i) {
         if (parity == PICT_BOTTOM_FIELD)
-            pic->f.data[i] += pic->f.linesize[i];
+            pic->data[i]   += pic->linesize[i];
         pic->reference      = parity;
-        pic->f.linesize[i] *= 2;
+        pic->linesize[i] *= 2;
     }
-    pic->poc= pic->field_poc[parity == PICT_BOTTOM_FIELD];
+    pic->poc = pic->parent->field_poc[parity == PICT_BOTTOM_FIELD];
+}
+
+static void ref_from_h264pic(H264Ref *dst, H264Picture *src)
+{
+    memcpy(dst->data,     src->f.data,     sizeof(dst->data));
+    memcpy(dst->linesize, src->f.linesize, sizeof(dst->linesize));
+    dst->reference = src->reference;
+    dst->poc       = src->poc;
+    dst->pic_id    = src->pic_id;
+    dst->parent = src;
 }
 
-static int split_field_copy(H264Picture *dest, H264Picture *src, int parity, int id_add)
+static int split_field_copy(H264Ref *dest, H264Picture *src, int parity, int id_add)
 {
     int match = !!(src->reference & parity);
 
     if (match) {
-        COPY_PICTURE(dest, src);
+        ref_from_h264pic(dest, src);
         if (parity != PICT_FRAME) {
             pic_as_field(dest, parity);
             dest->pic_id *= 2;
@@ -70,7 +73,7 @@ static int split_field_copy(H264Picture *dest, H264Picture *src, int parity, int
     return match;
 }
 
-static int build_def_list(H264Picture *def, int def_len,
+static int build_def_list(H264Ref *def, int def_len,
                           H264Picture **in, int len, int is_long, int sel)
 {
     int  i[2] = { 0 };
@@ -142,19 +145,16 @@ int ff_h264_fill_default_ref_list(H264Context *h, H264SliceContext *sl)
                                   h->long_ref, 16, 1, h->picture_structure);
 
             if (len < sl->ref_count[list])
-                memset(&h->default_ref_list[list][len], 0, sizeof(H264Picture) * (sl->ref_count[list] - len));
+                memset(&h->default_ref_list[list][len], 0, sizeof(H264Ref) * (sl->ref_count[list] - len));
             lens[list] = len;
         }
 
         if (lens[0] == lens[1] && lens[1] > 1) {
             for (i = 0; i < lens[0] &&
-                        h->default_ref_list[0][i].f.buf[0]->buffer ==
-                        h->default_ref_list[1][i].f.buf[0]->buffer; i++);
+                        h->default_ref_list[0][i].parent->f.buf[0]->buffer ==
+                        h->default_ref_list[1][i].parent->f.buf[0]->buffer; i++);
             if (i == lens[0]) {
-                H264Picture tmp;
-                COPY_PICTURE(&tmp, &h->default_ref_list[1][0]);
-                COPY_PICTURE(&h->default_ref_list[1][0], &h->default_ref_list[1][1]);
-                COPY_PICTURE(&h->default_ref_list[1][1], &tmp);
+                FFSWAP(H264Ref, h->default_ref_list[1][0], h->default_ref_list[1][1]);
             }
         }
     } else {
@@ -165,7 +165,7 @@ int ff_h264_fill_default_ref_list(H264Context *h, H264SliceContext *sl)
                               h-> long_ref, 16, 1, h->picture_structure);
 
         if (len < sl->ref_count[0])
-            memset(&h->default_ref_list[0][len], 0, sizeof(H264Picture) * (sl->ref_count[0] - len));
+            memset(&h->default_ref_list[0][len], 0, sizeof(H264Ref) * (sl->ref_count[0] - len));
     }
 #ifdef TRACE
     for (i = 0; i < sl->ref_count[0]; i++) {
@@ -214,14 +214,13 @@ static int pic_num_extract(H264Context *h, int pic_num, int *structure)
 
 int ff_h264_decode_ref_pic_list_reordering(H264Context *h, H264SliceContext *sl)
 {
-    int list, index, pic_structure, i;
+    int list, index, pic_structure;
 
     print_short_term(h);
     print_long_term(h);
 
     for (list = 0; list < sl->list_count; list++) {
-        for (i = 0; i < sl->ref_count[list]; i++)
-            COPY_PICTURE(&sl->ref_list[list][i], &h->default_ref_list[list][i]);
+        memcpy(sl->ref_list[list], h->default_ref_list[list], sl->ref_count[list] * sizeof(sl->ref_list[0][0]));
 
         if (get_bits1(&sl->gb)) {    // ref_pic_list_modification_flag_l[01]
             int pred = h->curr_pic_num;
@@ -304,17 +303,18 @@ int ff_h264_decode_ref_pic_list_reordering(H264Context *h, H264SliceContext *sl)
                 if (i < 0) {
                     av_log(h->avctx, AV_LOG_ERROR,
                            "reference picture missing during reorder\n");
-                    memset(&sl->ref_list[list][index], 0, sizeof(H264Picture)); // FIXME
+                    memset(&sl->ref_list[list][index], 0, sizeof(sl->ref_list[0][0])); // FIXME
                 } else {
                     for (i = index; i + 1 < sl->ref_count[list]; i++) {
-                        if (ref->long_ref == sl->ref_list[list][i].long_ref &&
+                        if (sl->ref_list[list][i].parent &&
+                            ref->long_ref == sl->ref_list[list][i].parent->long_ref &&
                             ref->pic_id   == sl->ref_list[list][i].pic_id)
                             break;
                     }
                     for (; i > index; i--) {
-                        COPY_PICTURE(&sl->ref_list[list][i], &sl->ref_list[list][i - 1]);
+                        sl->ref_list[list][i] = sl->ref_list[list][i - 1];
                     }
-                    COPY_PICTURE(&sl->ref_list[list][index], ref);
+                    ref_from_h264pic(&sl->ref_list[list][index], ref);
                     if (FIELD_PICTURE(h)) {
                         pic_as_field(&sl->ref_list[list][index], pic_structure);
                     }
@@ -324,10 +324,10 @@ int ff_h264_decode_ref_pic_list_reordering(H264Context *h, H264SliceContext *sl)
     }
     for (list = 0; list < sl->list_count; list++) {
         for (index = 0; index < sl->ref_count[list]; index++) {
-            if (!sl->ref_list[list][index].f.buf[0]) {
+            if (!sl->ref_list[list][index].parent) {
                 av_log(h->avctx, AV_LOG_ERROR, "Missing reference picture\n");
-                if (h->default_ref_list[list][0].f.buf[0])
-                    COPY_PICTURE(&sl->ref_list[list][index], &h->default_ref_list[list][0]);
+                if (h->default_ref_list[list][0].parent)
+                    sl->ref_list[list][index] = h->default_ref_list[list][0];
                 else
                     return -1;
             }
@@ -340,20 +340,24 @@ int ff_h264_decode_ref_pic_list_reordering(H264Context *h, H264SliceContext *sl)
 void ff_h264_fill_mbaff_ref_list(H264Context *h, H264SliceContext *sl)
 {
     int list, i, j;
-    for (list = 0; list < 2; list++) { //FIXME try list_count
+    for (list = 0; list < sl->list_count; list++) { //FIXME try list_count
         for (i = 0; i < sl->ref_count[list]; i++) {
-            H264Picture *frame = &sl->ref_list[list][i];
-            H264Picture *field = &sl->ref_list[list][16 + 2 * i];
-            COPY_PICTURE(field, frame);
+            H264Ref *frame = &sl->ref_list[list][i];
+            H264Ref *field = &sl->ref_list[list][16 + 2 * i];
+
+            field[0] = *frame;
+
             for (j = 0; j < 3; j++)
-                field[0].f.linesize[j] <<= 1;
+                field[0].linesize[j] <<= 1;
             field[0].reference = PICT_TOP_FIELD;
-            field[0].poc       = field[0].field_poc[0];
-            COPY_PICTURE(field + 1, field);
+            field[0].poc       = field[0].parent->field_poc[0];
+
+            field[1] = field[0];
+
             for (j = 0; j < 3; j++)
-                field[1].f.data[j] += frame->f.linesize[j];
+                field[1].data[j] += frame->parent->f.linesize[j];
             field[1].reference = PICT_BOTTOM_FIELD;
-            field[1].poc       = field[1].field_poc[1];
+            field[1].poc       = field[1].parent->field_poc[1];
 
             sl->luma_weight[16 + 2 * i][list][0] = sl->luma_weight[16 + 2 * i + 1][list][0] = sl->luma_weight[i][list][0];
             sl->luma_weight[16 + 2 * i][list][1] = sl->luma_weight[16 + 2 * i + 1][list][1] = sl->luma_weight[i][list][1];
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index 4044ba2ead..d0438fd00c 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -819,7 +819,7 @@ static void implicit_weight_table(const H264Context *h, H264SliceContext *sl, in
         int poc0 = sl->ref_list[0][ref0].poc;
         for (ref1 = ref_start; ref1 < ref_count1; ref1++) {
             int w = 32;
-            if (!sl->ref_list[0][ref0].long_ref && !sl->ref_list[1][ref1].long_ref) {
+            if (!sl->ref_list[0][ref0].parent->long_ref && !sl->ref_list[1][ref1].parent->long_ref) {
                 int poc1 = sl->ref_list[1][ref1].poc;
                 int td   = av_clip_int8(poc1 - poc0);
                 if (td) {
@@ -1652,9 +1652,9 @@ int ff_h264_decode_slice_header(H264Context *h, H264SliceContext *sl)
         for (i = 0; i < 16; i++) {
             id_list[i] = 60;
             if (j < sl->list_count && i < sl->ref_count[j] &&
-                sl->ref_list[j][i].f.buf[0]) {
+                sl->ref_list[j][i].parent->f.buf[0]) {
                 int k;
-                AVBuffer *buf = sl->ref_list[j][i].f.buf[0]->buffer;
+                AVBuffer *buf = sl->ref_list[j][i].parent->f.buf[0]->buffer;
                 for (k = 0; k < h->short_ref_count; k++)
                     if (h->short_ref[k]->f.buf[0]->buffer == buf) {
                         id_list[i] = k;
diff --git a/libavcodec/vaapi_h264.c b/libavcodec/vaapi_h264.c
index 481d7d17f5..0fb3876177 100644
--- a/libavcodec/vaapi_h264.c
+++ b/libavcodec/vaapi_h264.c
@@ -156,13 +156,13 @@ static int fill_vaapi_ReferenceFrames(VAPictureParameterBufferH264 *pic_param,
  * @param[in]  ref_count   The number of reference pictures in ref_list
  */
 static void fill_vaapi_RefPicList(VAPictureH264 RefPicList[32],
-                                  H264Picture  *ref_list,
+                                  H264Ref  *ref_list,
                                   unsigned int  ref_count)
 {
     unsigned int i, n = 0;
     for (i = 0; i < ref_count; i++)
         if (ref_list[i].reference)
-            fill_vaapi_pic(&RefPicList[n++], &ref_list[i], 0);
+            fill_vaapi_pic(&RefPicList[n++], ref_list[i].parent, 0);
 
     for (; n < 32; n++)
         init_vaapi_pic(&RefPicList[n]);