avcodec/h2645_sei: Factor out applying SEIs to frames

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>

4 files changed, 68 insertions, 97 deletions

diff --git a/libavcodec/h2645_sei.c b/libavcodec/h2645_sei.c
index f37a43ed77..3d551d900f 100644
--- a/libavcodec/h2645_sei.c
+++ b/libavcodec/h2645_sei.c
@@ -419,6 +419,63 @@ int ff_h2645_sei_ctx_replace(H2645SEI *dst, const H2645SEI *src)
     return 0;
 }
 
+int ff_h2645_sei_to_frame(AVFrame *frame, H2645SEI *sei,
+                          enum AVCodecID codec_id,
+                          AVCodecContext *avctx)
+{
+    if (sei->display_orientation.present &&
+        (sei->display_orientation.anticlockwise_rotation ||
+         sei->display_orientation.hflip ||
+         sei->display_orientation.vflip)) {
+        H2645SEIDisplayOrientation *o = &sei->display_orientation;
+        double angle = o->anticlockwise_rotation * 360 / (double) (1 << 16);
+        AVFrameSideData *rotation = av_frame_new_side_data(frame,
+                                                           AV_FRAME_DATA_DISPLAYMATRIX,
+                                                           sizeof(int32_t) * 9);
+        if (!rotation)
+            return AVERROR(ENOMEM);
+
+        /* av_display_rotation_set() expects the angle in the clockwise
+         * direction, hence the first minus.
+         * The below code applies the flips after the rotation, yet
+         * the H.2645 specs require flipping to be applied first.
+         * Because of R O(phi) = O(-phi) R (where R is flipping around
+         * an arbitatry axis and O(phi) is the proper rotation by phi)
+         * we can create display matrices as desired by negating
+         * the degree once for every flip applied. */
+        angle = -angle * (1 - 2 * !!o->hflip) * (1 - 2 * !!o->vflip);
+        av_display_rotation_set((int32_t *)rotation->data, angle);
+        av_display_matrix_flip((int32_t *)rotation->data,
+                                o->hflip, o->vflip);
+    }
+
+    if (sei->a53_caption.buf_ref) {
+        H2645SEIA53Caption *a53 = &sei->a53_caption;
+        AVFrameSideData *sd = av_frame_new_side_data_from_buf(frame, AV_FRAME_DATA_A53_CC, a53->buf_ref);
+        if (!sd)
+            av_buffer_unref(&a53->buf_ref);
+        a53->buf_ref = NULL;
+        if (avctx)
+            avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
+    }
+
+    for (unsigned i = 0; i < sei->unregistered.nb_buf_ref; i++) {
+        H2645SEIUnregistered *unreg = &sei->unregistered;
+
+        if (unreg->buf_ref[i]) {
+            AVFrameSideData *sd = av_frame_new_side_data_from_buf(frame,
+                    AV_FRAME_DATA_SEI_UNREGISTERED,
+                    unreg->buf_ref[i]);
+            if (!sd)
+                av_buffer_unref(&unreg->buf_ref[i]);
+            unreg->buf_ref[i] = NULL;
+        }
+    }
+    sei->unregistered.nb_buf_ref = 0;
+
+    return 0;
+}
+
 void ff_h2645_sei_reset(H2645SEI *s)
 {
     av_buffer_unref(&s->a53_caption.buf_ref);
diff --git a/libavcodec/h2645_sei.h b/libavcodec/h2645_sei.h
index 3e088f3307..4600caab29 100644
--- a/libavcodec/h2645_sei.h
+++ b/libavcodec/h2645_sei.h
@@ -134,4 +134,8 @@ int ff_h2645_sei_ctx_replace(H2645SEI *dst, const H2645SEI *src);
 
 void ff_h2645_sei_reset(H2645SEI *s);
 
+int ff_h2645_sei_to_frame(AVFrame *frame, H2645SEI *sei,
+                          enum AVCodecID codec_id,
+                          AVCodecContext *avctx);
+
 #endif /* AVCODEC_H2645_SEI_H */
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index 35e2149b4f..942e7c8a02 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -1145,6 +1145,7 @@ static int h264_export_frame_props(H264Context *h)
     const SPS *sps = h->ps.sps;
     H264Picture *cur = h->cur_pic_ptr;
     AVFrame *out = cur->f;
+    int ret;
 
     out->interlaced_frame = 0;
     out->repeat_pict      = 0;
@@ -1272,31 +1273,6 @@ static int h264_export_frame_props(H264Context *h)
         }
     }
 
-    if (h->sei.common.display_orientation.present &&
-        (h->sei.common.display_orientation.anticlockwise_rotation ||
-         h->sei.common.display_orientation.hflip ||
-         h->sei.common.display_orientation.vflip)) {
-        H2645SEIDisplayOrientation *o = &h->sei.common.display_orientation;
-        double angle = o->anticlockwise_rotation * 360 / (double) (1 << 16);
-        AVFrameSideData *rotation = av_frame_new_side_data(out,
-                                                           AV_FRAME_DATA_DISPLAYMATRIX,
-                                                           sizeof(int32_t) * 9);
-        if (rotation) {
-            /* av_display_rotation_set() expects the angle in the clockwise
-             * direction, hence the first minus.
-             * The below code applies the flips after the rotation, yet
-             * the H.2645 specs require flipping to be applied first.
-             * Because of R O(phi) = O(-phi) R (where R is flipping around
-             * an arbitatry axis and O(phi) is the proper rotation by phi)
-             * we can create display matrices as desired by negating
-             * the degree once for every flip applied. */
-            angle = -angle * (1 - 2 * !!o->hflip) * (1 - 2 * !!o->vflip);
-            av_display_rotation_set((int32_t *)rotation->data, angle);
-            av_display_matrix_flip((int32_t *)rotation->data,
-                                   o->hflip, o->vflip);
-        }
-    }
-
     if (h->sei.common.afd.present) {
         AVFrameSideData *sd = av_frame_new_side_data(out, AV_FRAME_DATA_AFD,
                                                      sizeof(uint8_t));
@@ -1307,30 +1283,9 @@ static int h264_export_frame_props(H264Context *h)
         }
     }
 
-    if (h->sei.common.a53_caption.buf_ref) {
-        H2645SEIA53Caption *a53 = &h->sei.common.a53_caption;
-
-        AVFrameSideData *sd = av_frame_new_side_data_from_buf(out, AV_FRAME_DATA_A53_CC, a53->buf_ref);
-        if (!sd)
-            av_buffer_unref(&a53->buf_ref);
-        a53->buf_ref = NULL;
-
-        h->avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
-    }
-
-    for (int i = 0; i < h->sei.common.unregistered.nb_buf_ref; i++) {
-        H2645SEIUnregistered *unreg = &h->sei.common.unregistered;
-
-        if (unreg->buf_ref[i]) {
-            AVFrameSideData *sd = av_frame_new_side_data_from_buf(out,
-                    AV_FRAME_DATA_SEI_UNREGISTERED,
-                    unreg->buf_ref[i]);
-            if (!sd)
-                av_buffer_unref(&unreg->buf_ref[i]);
-            unreg->buf_ref[i] = NULL;
-        }
-    }
-    h->sei.common.unregistered.nb_buf_ref = 0;
+    ret = ff_h2645_sei_to_frame(out, &h->sei.common, AV_CODEC_ID_H264, h->avctx);
+    if (ret < 0)
+        return ret;
 
     if (h->sei.common.film_grain_characteristics.present) {
         H2645SEIFilmGrainCharacteristics *fgc = &h->sei.common.film_grain_characteristics;
diff --git a/libavcodec/hevcdec.c b/libavcodec/hevcdec.c
index a7e1ebb463..4d633adcca 100644
--- a/libavcodec/hevcdec.c
+++ b/libavcodec/hevcdec.c
@@ -2761,32 +2761,6 @@ static int set_side_data(HEVCContext *s)
         }
     }
 
-    if (s->sei.common.display_orientation.present &&
-        (s->sei.common.display_orientation.anticlockwise_rotation ||
-         s->sei.common.display_orientation.hflip || s->sei.common.display_orientation.vflip)) {
-        double angle = s->sei.common.display_orientation.anticlockwise_rotation * 360 / (double) (1 << 16);
-        AVFrameSideData *rotation = av_frame_new_side_data(out,
-                                                           AV_FRAME_DATA_DISPLAYMATRIX,
-                                                           sizeof(int32_t) * 9);
-        if (!rotation)
-            return AVERROR(ENOMEM);
-
-        /* av_display_rotation_set() expects the angle in the clockwise
-         * direction, hence the first minus.
-         * The below code applies the flips after the rotation, yet
-         * the H.2645 specs require flipping to be applied first.
-         * Because of R O(phi) = O(-phi) R (where R is flipping around
-         * an arbitatry axis and O(phi) is the proper rotation by phi)
-         * we can create display matrices as desired by negating
-         * the degree once for every flip applied. */
-        angle = -angle * (1 - 2 * !!s->sei.common.display_orientation.hflip)
-                       * (1 - 2 * !!s->sei.common.display_orientation.vflip);
-        av_display_rotation_set((int32_t *)rotation->data, angle);
-        av_display_matrix_flip((int32_t *)rotation->data,
-                               s->sei.common.display_orientation.hflip,
-                               s->sei.common.display_orientation.vflip);
-    }
-
     // Decrement the mastering display flag when IRAP frame has no_rasl_output_flag=1
     // so the side data persists for the entire coded video sequence.
     if (s->sei.mastering_display.present > 0 &&
@@ -2856,28 +2830,9 @@ static int set_side_data(HEVCContext *s)
                metadata->MaxCLL, metadata->MaxFALL);
     }
 
-    if (s->sei.common.a53_caption.buf_ref) {
-        H2645SEIA53Caption *a53 = &s->sei.common.a53_caption;
-
-        AVFrameSideData *sd = av_frame_new_side_data_from_buf(out, AV_FRAME_DATA_A53_CC, a53->buf_ref);
-        if (!sd)
-            av_buffer_unref(&a53->buf_ref);
-        a53->buf_ref = NULL;
-    }
-
-    for (int i = 0; i < s->sei.common.unregistered.nb_buf_ref; i++) {
-        H2645SEIUnregistered *unreg = &s->sei.common.unregistered;
-
-        if (unreg->buf_ref[i]) {
-            AVFrameSideData *sd = av_frame_new_side_data_from_buf(out,
-                    AV_FRAME_DATA_SEI_UNREGISTERED,
-                    unreg->buf_ref[i]);
-            if (!sd)
-                av_buffer_unref(&unreg->buf_ref[i]);
-            unreg->buf_ref[i] = NULL;
-        }
-    }
-    s->sei.common.unregistered.nb_buf_ref = 0;
+    ret = ff_h2645_sei_to_frame(out, &s->sei.common, AV_CODEC_ID_HEVC, NULL);
+    if (ret < 0)
+        return ret;
 
     if (s->sei.timecode.present) {
         uint32_t *tc_sd;