vaapi_encode: Write sequence header as extradata

Only works if packed headers are supported, where we can know the
output before generating the first frame.

(cherry picked from commit 0cf86fabfa5820596cca2cfead63c6f8df76c3f2)

2 files changed, 24 insertions, 0 deletions

diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c
index 262968a0ec..a3c99912d1 100644
--- a/libavcodec/vaapi_encode.c
+++ b/libavcodec/vaapi_encode.c
@@ -1405,6 +1405,28 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
     // where it actually overlaps properly, though.)
     ctx->issue_mode = ISSUE_MODE_MAXIMISE_THROUGHPUT;
 
+    if (ctx->va_packed_headers & VA_ENC_PACKED_HEADER_SEQUENCE &&
+        ctx->codec->write_sequence_header) {
+        char data[MAX_PARAM_BUFFER_SIZE];
+        size_t bit_len = 8 * sizeof(data);
+
+        err = ctx->codec->write_sequence_header(avctx, data, &bit_len);
+        if (err < 0) {
+            av_log(avctx, AV_LOG_ERROR, "Failed to write sequence header "
+                   "for extradata: %d.\n", err);
+            goto fail;
+        } else {
+            avctx->extradata_size = (bit_len + 7) / 8;
+            avctx->extradata = av_mallocz(avctx->extradata_size +
+                                          AV_INPUT_BUFFER_PADDING_SIZE);
+            if (!avctx->extradata) {
+                err = AVERROR(ENOMEM);
+                goto fail;
+            }
+            memcpy(avctx->extradata, data, avctx->extradata_size);
+        }
+    }
+
     return 0;
 
 fail:
diff --git a/libavcodec/vaapi_encode.h b/libavcodec/vaapi_encode.h
index abce6b8ed8..cc05ac8298 100644
--- a/libavcodec/vaapi_encode.h
+++ b/libavcodec/vaapi_encode.h
@@ -235,6 +235,8 @@ typedef struct VAAPIEncodeType {
     int slice_header_type;
 
     // Write the packed header data to the provided buffer.
+    // The sequence header is also used to fill the codec extradata
+    // when the encoder is starting.
     int (*write_sequence_header)(AVCodecContext *avctx,
                                  char *data, size_t *data_len);
     int  (*write_picture_header)(AVCodecContext *avctx,