libavcodec: factor out SEI generation for A53 captions

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

5 files changed, 101 insertions, 92 deletions

diff --git a/libavcodec/internal.h b/libavcodec/internal.h
index be54471ef6..000fe263cc 100644
--- a/libavcodec/internal.h
+++ b/libavcodec/internal.h
@@ -345,4 +345,19 @@ AVCPBProperties *ff_add_cpb_side_data(AVCodecContext *avctx);
 
 int ff_side_data_set_encoder_stats(AVPacket *pkt, int quality, int64_t *error, int error_count, int pict_type);
 
+/**
+ * Check AVFrame for A53 side data and allocate and fill SEI message with A53 info
+ *
+ * @param frame      Raw frame to get A53 side data from
+ * @param prefix_len Number of bytes to allocate before SEI message
+ * @param data       Pointer to a variable to store allocated memory
+ *                   Upon return the variable will hold NULL on error or if frame has no A53 info.
+ *                   Otherwise it will point to prefix_len uninitialized bytes followed by
+ *                   *sei_size SEI message
+ * @param sei_size   Pointer to a variable to store generated SEI message length
+ * @return           Zero on success, negative error code on failure
+ */
+int ff_alloc_a53_sei(const AVFrame *frame, size_t prefix_len,
+                     void **data, size_t *sei_size);
+
 #endif /* AVCODEC_INTERNAL_H */
diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c
index 4d14665361..85f1996ab6 100644
--- a/libavcodec/libx264.c
+++ b/libavcodec/libx264.c
@@ -310,46 +310,29 @@ static int X264_frame(AVCodecContext *ctx, AVPacket *pkt, const AVFrame *frame,
         reconfig_encoder(ctx, frame);
 
         if (x4->a53_cc) {
-            side_data = av_frame_get_side_data(frame, AV_FRAME_DATA_A53_CC);
-            if (side_data) {
+            void *sei_data;
+            size_t sei_size;
+
+            ret = ff_alloc_a53_sei(frame, 0, &sei_data, &sei_size);
+            if (ret < 0) {
+                av_log(ctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n");
+            } else if (sei_data) {
                 x4->pic.extra_sei.payloads = av_mallocz(sizeof(x4->pic.extra_sei.payloads[0]));
                 if (x4->pic.extra_sei.payloads == NULL) {
                     av_log(ctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n");
-                    goto skip_a53cc;
+                    av_free(sei_data);
+                } else {
+                    x4->pic.extra_sei.sei_free = av_free;
+
+                    x4->pic.extra_sei.payloads[0].payload_size = sei_size;
+                    x4->pic.extra_sei.payloads[0].payload = sei_data;
+                    x4->pic.extra_sei.num_payloads = 1;
+                    x4->pic.extra_sei.payloads[0].payload_type = 4;
                 }
-                x4->pic.extra_sei.sei_free = av_free;
-
-                x4->pic.extra_sei.payloads[0].payload_size = side_data->size + 11;
-                x4->pic.extra_sei.payloads[0].payload = av_mallocz(x4->pic.extra_sei.payloads[0].payload_size);
-                if (x4->pic.extra_sei.payloads[0].payload == NULL) {
-                    av_log(ctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n");
-                    av_freep(&x4->pic.extra_sei.payloads);
-                    goto skip_a53cc;
-                }
-                x4->pic.extra_sei.num_payloads = 1;
-                x4->pic.extra_sei.payloads[0].payload_type = 4;
-                memcpy(x4->pic.extra_sei.payloads[0].payload + 10, side_data->data, side_data->size);
-                x4->pic.extra_sei.payloads[0].payload[0] = 181;
-                x4->pic.extra_sei.payloads[0].payload[1] = 0;
-                x4->pic.extra_sei.payloads[0].payload[2] = 49;
-
-                /**
-                 * 'GA94' is standard in North America for ATSC, but hard coding
-                 * this style may not be the right thing to do -- other formats
-                 * do exist. This information is not available in the side_data
-                 * so we are going with this right now.
-                 */
-                AV_WL32(x4->pic.extra_sei.payloads[0].payload + 3,
-                    MKTAG('G', 'A', '9', '4'));
-                x4->pic.extra_sei.payloads[0].payload[7] = 3;
-                x4->pic.extra_sei.payloads[0].payload[8] =
-                    ((side_data->size/3) & 0x1f) | 0x40;
-                x4->pic.extra_sei.payloads[0].payload[9] = 0;
-                x4->pic.extra_sei.payloads[0].payload[side_data->size+10] = 255;
             }
         }
     }
-skip_a53cc:
+
     do {
         if (x264_encoder_encode(x4->enc, &nal, &nnal, frame? &x4->pic: NULL, &pic_out) < 0)
             return AVERROR_EXTERNAL;
diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c
index 132cf47bb2..f56cb612b6 100644
--- a/libavcodec/qsvenc.c
+++ b/libavcodec/qsvenc.c
@@ -781,9 +781,7 @@ static void free_encoder_ctrl_payloads(mfxEncodeCtrl* enc_ctrl)
     if (enc_ctrl) {
         int i;
         for (i = 0; i < enc_ctrl->NumPayload && i < QSV_MAX_ENC_PAYLOAD; i++) {
-            mfxPayload* pay = enc_ctrl->Payload[i];
-            av_free(enc_ctrl->Payload[i]->Data);
-            av_free(pay);
+            av_free(enc_ctrl->Payload[i]);
         }
         enc_ctrl->NumPayload = 0;
     }
diff --git a/libavcodec/qsvenc_h264.c b/libavcodec/qsvenc_h264.c
index 66e5ee8755..cd08d863e6 100644
--- a/libavcodec/qsvenc_h264.c
+++ b/libavcodec/qsvenc_h264.c
@@ -43,63 +43,33 @@ typedef struct QSVH264EncContext {
 static int qsv_h264_set_encode_ctrl(AVCodecContext *avctx,
                                     const AVFrame *frame, mfxEncodeCtrl* enc_ctrl)
 {
-    AVFrameSideData *side_data = NULL;
-    QSVH264EncContext *qh264 = avctx->priv_data;
-    QSVEncContext *q = &qh264->qsv;
-
     if (q->a53_cc && frame) {
-        side_data = av_frame_get_side_data(frame, AV_FRAME_DATA_A53_CC);
-        if (side_data) {
-
-            int sei_payload_size = 0;
-            mfxU8* sei_data = NULL;
-            mfxPayload* payload = NULL;
-
-            sei_payload_size = side_data->size + 13;
-
-            sei_data = av_mallocz(sei_payload_size);
-            if (!sei_data) {
-                av_log(avctx, AV_LOG_ERROR, "No memory for CC, skipping...\n");
-                return AVERROR(ENOMEM);
-            }
-
-            // SEI header
-            sei_data[0] = 4;
-            sei_data[1] = sei_payload_size - 2; // size of SEI data
-
-            // country code
-            sei_data[2] = 181;
-            sei_data[3] = 0;
-            sei_data[4] = 49;
-
-            // ATSC_identifier - using 'GA94' only
-            AV_WL32(sei_data + 5,
-                MKTAG('G', 'A', '9', '4'));
-            sei_data[9] = 3;
-            sei_data[10] =
-                ((side_data->size/3) & 0x1f) | 0xC0;
-
-            sei_data[11] = 0xFF; // reserved
-
-            memcpy(sei_data + 12, side_data->data, side_data->size);
-
-            sei_data[side_data->size+12] = 255;
-
-            payload = av_mallocz(sizeof(mfxPayload));
-            if (!payload) {
-                av_log(avctx, AV_LOG_ERROR, "No memory, skipping captions\n");
-                av_freep(&sei_data);
-                return AVERROR(ENOMEM);
-            }
-            payload->BufSize = side_data->size + 13;
-            payload->NumBit = payload->BufSize * 8;
-            payload->Type = 4;
-            payload->Data = sei_data;
-
-            enc_ctrl->NumExtParam = 0;
-            enc_ctrl->NumPayload = 1;
-            enc_ctrl->Payload[0] = payload;
-        }
+        AVFrameSideData *side_data = NULL;
+        QSVH264EncContext *qh264 = avctx->priv_data;
+        QSVEncContext *q = &qh264->qsv;
+        mfxPayload* payload;
+        mfxU8* sei_data;
+        size_t sei_size;
+        int res;
+
+        res = ff_alloc_a53_sei(frame, sizeof(mfxPayload) + 2, (void**)&payload, &sei_size);
+        if (res < 0)
+            return res;
+
+        sei_data = (mfxU8*)(payload + 1);
+        // SEI header
+        sei_data[0] = 4;
+        sei_data[1] = (mfxU8)sei_size; // size of SEI data
+        // SEI data filled in by ff_alloc_a53_sei
+
+        payload->BufSize = sei_size + 2;
+        payload->NumBit = payload->BufSize * 8;
+        payload->Type = 4;
+        payload->Data = sei_data;
+
+        enc_ctrl->NumExtParam = 0;
+        enc_ctrl->NumPayload = 1;
+        enc_ctrl->Payload[0] = payload;
     }
     return 0;
 }
diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index 402a9d8f03..54a3e8708d 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -4178,3 +4178,46 @@ int avcodec_parameters_to_context(AVCodecContext *codec,
 
     return 0;
 }
+
+int ff_alloc_a53_sei(const AVFrame *frame, size_t prefix_len,
+                     void **data, size_t *sei_size)
+{
+    AVFrameSideData *side_data = NULL;
+    uint8_t *sei_data;
+
+    if (frame)
+        side_data = av_frame_get_side_data(frame, AV_FRAME_DATA_A53_CC);
+
+    if (!side_data) {
+        *data = NULL;
+        return 0;
+    }
+
+    *sei_size = side_data->size + 11;
+    *data = av_mallocz(*sei_size + prefix_len);
+    if (!*data)
+        return AVERROR(ENOMEM);
+    sei_data = (uint8_t*)*data + prefix_len;
+
+    // country code
+    sei_data[0] = 181;
+    sei_data[1] = 0;
+    sei_data[2] = 49;
+
+    /**
+     * 'GA94' is standard in North America for ATSC, but hard coding
+     * this style may not be the right thing to do -- other formats
+     * do exist. This information is not available in the side_data
+     * so we are going with this right now.
+     */
+    AV_WL32(sei_data + 3, MKTAG('G', 'A', '9', '4'));
+    sei_data[7] = 3;
+    sei_data[8] = ((side_data->size/3) & 0x1f) | 0x40;
+    sei_data[9] = 0;
+
+    memcpy(sei_data + 10, side_data->data, side_data->size);
+
+    sei_data[side_data->size+10] = 255;
+
+    return 0;
+}