avcodec/dovi_rpuenc: make encapsulation optional

And move the choice of desired container to `flags`. This is needed to
handle differing API requirements (e.g. libx265 requires the NAL RBSP,
but CBS BSF requires the unescaped bytes).

5 files changed, 25 insertions, 21 deletions

diff --git a/libavcodec/dovi_rpu.h b/libavcodec/dovi_rpu.h
index 65a4529106..226a769bff 100644
--- a/libavcodec/dovi_rpu.h
+++ b/libavcodec/dovi_rpu.h
@@ -123,16 +123,20 @@ int ff_dovi_attach_side_data(DOVIContext *s, AVFrame *frame);
  */
 int ff_dovi_configure(DOVIContext *s, AVCodecContext *avctx);
 
+enum {
+    FF_DOVI_WRAP_NAL        = 1 << 0, ///< wrap inside NAL RBSP
+    FF_DOVI_WRAP_T35        = 1 << 1, ///< wrap inside T.35+EMDF
+};
+
 /**
- * Synthesize a Dolby Vision RPU reflecting the current state. Note that this
- * assumes all previous calls to `ff_dovi_rpu_generate` have been appropriately
- * signalled, i.e. it will not re-send already transmitted redundant data.
+ * Synthesize a Dolby Vision RPU reflecting the current state. By default, the
+ * RPU is not encapsulated (see `flags` for more options). Note that this
+ * assumes all previous calls to `ff_dovi_rpu_generate` have been
+ * appropriately signalled, i.e. it will not re-send already transmitted
+ * redundant data.
  *
  * Mutates the internal state of DOVIContext to reflect the change.
  * Returns 0 or a negative error code.
- *
- * This generates a fully formed RPU ready for inclusion in the bitstream,
- * including the EMDF header (profile 10) or NAL encapsulation (otherwise).
  */
 int ff_dovi_rpu_generate(DOVIContext *s, const AVDOVIMetadata *metadata,
                          int flags, uint8_t **out_rpu, int *out_size);
diff --git a/libavcodec/dovi_rpuenc.c b/libavcodec/dovi_rpuenc.c
index e1a70be42d..73db9437a0 100644
--- a/libavcodec/dovi_rpuenc.c
+++ b/libavcodec/dovi_rpuenc.c
@@ -713,9 +713,7 @@ int ff_dovi_rpu_generate(DOVIContext *s, const AVDOVIMetadata *metadata,
     flush_put_bits(pb);
 
     rpu_size = put_bytes_output(pb);
-    switch (s->cfg.dv_profile) {
-    case 10:
-        /* AV1 uses T.35 OBU with EMDF header */
+    if (flags & FF_DOVI_WRAP_T35) {
         *out_rpu = av_malloc(rpu_size + 15);
         if (!*out_rpu)
             return AVERROR(ENOMEM);
@@ -742,10 +740,8 @@ int ff_dovi_rpu_generate(DOVIContext *s, const AVDOVIMetadata *metadata,
         flush_put_bits(pb);
         *out_size = put_bytes_output(pb);
         return 0;
-
-    case 5:
-    case 8:
-        *out_rpu = dst = av_malloc(1 + rpu_size * 3 / 2); /* worst case */
+    } else if (flags & FF_DOVI_WRAP_NAL) {
+        *out_rpu = dst = av_malloc(4 + rpu_size * 3 / 2); /* worst case */
         if (!*out_rpu)
             return AVERROR(ENOMEM);
         *dst++ = 25; /* NAL prefix */
@@ -768,10 +764,12 @@ int ff_dovi_rpu_generate(DOVIContext *s, const AVDOVIMetadata *metadata,
         }
         *out_size = dst - *out_rpu;
         return 0;
-
-    default:
-        /* Should be unreachable */
-        av_assert0(0);
-        return AVERROR_BUG;
+    } else {
+        /* Return intermediate buffer directly */
+        *out_rpu = s->rpu_buf;
+        *out_size = rpu_size;
+        s->rpu_buf = NULL;
+        s->rpu_buf_sz = 0;
+        return 0;
     }
 }
diff --git a/libavcodec/libaomenc.c b/libavcodec/libaomenc.c
index aa51c89e29..fd9bea2505 100644
--- a/libavcodec/libaomenc.c
+++ b/libavcodec/libaomenc.c
@@ -1294,7 +1294,8 @@ FF_ENABLE_DEPRECATION_WARNINGS
             const AVDOVIMetadata *metadata = (const AVDOVIMetadata *)sd->data;
             uint8_t *t35;
             int size;
-            if ((res = ff_dovi_rpu_generate(&ctx->dovi, metadata, 0, &t35, &size)) < 0)
+            if ((res = ff_dovi_rpu_generate(&ctx->dovi, metadata, FF_DOVI_WRAP_T35,
+                                            &t35, &size)) < 0)
                 return res;
             res = aom_img_add_metadata(rawimg, OBU_METADATA_TYPE_ITUT_T35,
                                        t35, size, AOM_MIF_ANY_FRAME);
diff --git a/libavcodec/libsvtav1.c b/libavcodec/libsvtav1.c
index b6db63fd7a..e7b12fb488 100644
--- a/libavcodec/libsvtav1.c
+++ b/libavcodec/libsvtav1.c
@@ -541,7 +541,8 @@ static int eb_send_frame(AVCodecContext *avctx, const AVFrame *frame)
         const AVDOVIMetadata *metadata = (const AVDOVIMetadata *)sd->data;
         uint8_t *t35;
         int size;
-        if ((ret = ff_dovi_rpu_generate(&svt_enc->dovi, metadata, 0, &t35, &size)) < 0)
+        if ((ret = ff_dovi_rpu_generate(&svt_enc->dovi, metadata, FF_DOVI_WRAP_T35,
+                                        &t35, &size)) < 0)
             return ret;
         ret = svt_add_metadata(headerPtr, EB_AV1_METADATA_TYPE_ITUT_T35, t35, size);
         av_free(t35);
diff --git a/libavcodec/libx265.c b/libavcodec/libx265.c
index 12044ae2cc..dbc5b2aab1 100644
--- a/libavcodec/libx265.c
+++ b/libavcodec/libx265.c
@@ -789,7 +789,7 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
         sd = av_frame_get_side_data(pic, AV_FRAME_DATA_DOVI_METADATA);
         if (ctx->dovi.cfg.dv_profile && sd) {
             const AVDOVIMetadata *metadata = (const AVDOVIMetadata *)sd->data;
-            ret = ff_dovi_rpu_generate(&ctx->dovi, metadata, 0,
+            ret = ff_dovi_rpu_generate(&ctx->dovi, metadata, FF_DOVI_WRAP_NAL,
                                        &x265pic.rpu.payload,
                                        &x265pic.rpu.payloadSize);
             if (ret < 0) {