avcodec/libjxlenc: apply displaymatrix side data orientation to output

If the output is tagged with AV_FRAME_DATA_DISPLAYMATRIX we will read
it and apply it to the output file.

Signed-off-by: Leo Izen <leo.izen@gmail.com>

1 files changed, 61 insertions, 4 deletions

diff --git a/libavcodec/libjxlenc.c b/libavcodec/libjxlenc.c
index 40d5f760b1..4f05f015e2 100644
--- a/libavcodec/libjxlenc.c
+++ b/libavcodec/libjxlenc.c
@@ -28,6 +28,7 @@
 
 #include "libavutil/avutil.h"
 #include "libavutil/csp.h"
+#include "libavutil/display.h"
 #include "libavutil/error.h"
 #include "libavutil/frame.h"
 #include "libavutil/libm.h"
@@ -40,6 +41,7 @@
 #include "avcodec.h"
 #include "encode.h"
 #include "codec_internal.h"
+#include "exif_internal.h"
 
 #include <jxl/encode.h>
 #include <jxl/thread_parallel_runner.h>
@@ -322,10 +324,14 @@ static int libjxl_preprocess_stream(AVCodecContext *avctx, const AVFrame *frame,
 {
     LibJxlEncodeContext *ctx = avctx->priv_data;
     AVFrameSideData *sd;
+    int32_t *matrix = (int32_t[9]){ 0 };
+    int ret = 0;
     const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(frame->format);
     JxlBasicInfo info;
     JxlPixelFormat *jxl_fmt = &ctx->jxl_fmt;
     int bits_per_sample;
+    int orientation;
+    AVBufferRef *exif_buffer = NULL;
 #if JPEGXL_NUMERIC_VERSION >= JPEGXL_COMPUTE_NUMERIC_VERSION(0, 8, 0)
     JxlBitDepth jxl_bit_depth;
 #endif
@@ -368,8 +374,51 @@ static int libjxl_preprocess_stream(AVCodecContext *avctx, const AVFrame *frame,
     /* bitexact lossless requires there to be no XYB transform */
     info.uses_original_profile = ctx->distance == 0.0 || !ctx->xyb;
 
-    /* libjxl doesn't support negative linesizes so we use orientation to work around this */
-    info.orientation = frame->linesize[0] >= 0 ? JXL_ORIENT_IDENTITY : JXL_ORIENT_FLIP_VERTICAL;
+    sd = av_frame_get_side_data(frame, AV_FRAME_DATA_EXIF);
+    if (sd) {
+        AVExifMetadata ifd = { 0 };
+        AVExifEntry *orient = NULL;
+        uint16_t tag = av_exif_get_tag_id("Orientation");
+        ret = av_exif_parse_buffer(avctx, sd->data, sd->size, &ifd, AV_EXIF_TIFF_HEADER);
+        if (ret >= 0)
+            ret = ff_exif_sanitize_ifd(avctx, frame, &ifd);
+        if (ret >= 0)
+            ret = av_exif_get_entry(avctx, &ifd, tag, 0, &orient);
+        if (ret >= 0 && orient && orient->value.uint[0] >= 1 && orient->value.uint[0] <= 8) {
+            av_exif_orientation_to_matrix(matrix, orient->value.uint[0]);
+            ret = av_exif_remove_entry(avctx, &ifd, tag, 0);
+        } else {
+            av_exif_orientation_to_matrix(matrix, 1);
+        }
+        if (ret >= 0)
+            ret = av_exif_write(avctx, &ifd, &exif_buffer, AV_EXIF_TIFF_HEADER);
+        if (ret < 0)
+            av_log(avctx, AV_LOG_WARNING, "unable to process EXIF frame data\n");
+    } else {
+        sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DISPLAYMATRIX);
+        if (sd)
+            matrix = (int32_t *) sd->data;
+        else
+            av_exif_orientation_to_matrix(matrix, 1);
+    }
+
+    /* av_display_matrix_flip is a right-multipilcation */
+    /* i.e. flip is applied before the previous matrix */
+    if (frame->linesize < 0)
+        av_display_matrix_flip(matrix, 0, 1);
+
+    orientation = av_exif_matrix_to_orientation(matrix);
+    /* JPEG XL orientation flag agrees with EXIF for values 1-8 */
+    if (orientation) {
+        info.orientation = orientation;
+    } else {
+        av_log(avctx, AV_LOG_WARNING, "singular displaymatrix data\n");
+        info.orientation = frame->linesize[0] >= 0 ? JXL_ORIENT_IDENTITY : JXL_ORIENT_FLIP_VERTICAL;
+    }
+
+    /* restore the previous value */
+    if (frame->linesize < 0)
+        av_display_matrix_flip(matrix, 0, 1);
 
     if (animated) {
         info.have_animation = 1;
@@ -382,7 +431,8 @@ static int libjxl_preprocess_stream(AVCodecContext *avctx, const AVFrame *frame,
 
     if (JxlEncoderSetBasicInfo(ctx->encoder, &info) != JXL_ENC_SUCCESS) {
         av_log(avctx, AV_LOG_ERROR, "Failed to set JxlBasicInfo\n");
-        return AVERROR_EXTERNAL;
+        ret = AVERROR_EXTERNAL;
+        goto end;
     }
 
     sd = av_frame_get_side_data(frame, AV_FRAME_DATA_ICC_PROFILE);
@@ -399,6 +449,11 @@ static int libjxl_preprocess_stream(AVCodecContext *avctx, const AVFrame *frame,
         av_log(avctx, AV_LOG_WARNING, "Failed to set JxlBitDepth\n");
 #endif
 
+    if (exif_buffer) {
+        if (JxlEncoderUseBoxes(ctx->encoder) != JXL_ENC_SUCCESS)
+            av_log(avctx, AV_LOG_WARNING, "Couldn't enable UseBoxes\n");
+    }
+
     /* depending on basic info, level 10 might
      * be required instead of level 5 */
     if (JxlEncoderGetRequiredCodestreamLevel(ctx->encoder) > 5) {
@@ -406,7 +461,9 @@ static int libjxl_preprocess_stream(AVCodecContext *avctx, const AVFrame *frame,
             av_log(avctx, AV_LOG_WARNING, "Could not increase codestream level\n");
     }
 
-    return 0;
+end:
+    av_buffer_unref(&exif_buffer);
+    return ret;
 }
 
 /**