avcodec/exif: add orientation ID and matrix conversion routines

Takes existing code that makes display matricies work with EXIF
orientation tags and expose the conversion as a public API.

Signed-off-by: Leo Izen <leo.izen@gmail.com>

2 files changed, 70 insertions, 54 deletions

diff --git a/libavcodec/exif.c b/libavcodec/exif.c
index fa46202874..d174d16ddc 100644
--- a/libavcodec/exif.c
+++ b/libavcodec/exif.c
@@ -811,7 +811,7 @@ static int attach_displaymatrix(void *logctx, AVFrame *frame, int orientation)
     int32_t *matrix;
     /* invalid orientation */
     if (orientation < 2 || orientation > 8)
-        return 0;
+        return AVERROR_INVALIDDATA;
     sd = av_frame_new_side_data(frame, AV_FRAME_DATA_DISPLAYMATRIX, sizeof(int32_t) * 9);
     if (!sd) {
         av_log(logctx, AV_LOG_ERROR, "Could not allocate frame side data\n");
@@ -819,37 +819,7 @@ static int attach_displaymatrix(void *logctx, AVFrame *frame, int orientation)
     }
     matrix = (int32_t *) sd->data;
 
-    switch (orientation) {
-    case 2:
-        av_display_rotation_set(matrix, 0.0);
-        av_display_matrix_flip(matrix, 1, 0);
-        break;
-    case 3:
-        av_display_rotation_set(matrix, 180.0);
-        break;
-    case 4:
-        av_display_rotation_set(matrix, 180.0);
-        av_display_matrix_flip(matrix, 1, 0);
-        break;
-    case 5:
-        av_display_rotation_set(matrix, 90.0);
-        av_display_matrix_flip(matrix, 1, 0);
-        break;
-    case 6:
-        av_display_rotation_set(matrix, 90.0);
-        break;
-    case 7:
-        av_display_rotation_set(matrix, -90.0);
-        av_display_matrix_flip(matrix, 1, 0);
-        break;
-    case 8:
-        av_display_rotation_set(matrix, -90.0);
-        break;
-    default:
-        av_assert0(0);
-    }
-
-    return 0;
+    return av_exif_orientation_to_matrix(matrix, orientation);
 }
 
 #define COLUMN_SEP(i, c) ((i) ? ((i) % (c) ? ", " : "\n") : "")
@@ -1215,31 +1185,59 @@ end:
     return ret;
 }
 
-static int exif_get_orientation(const int32_t *display_matrix)
+static const int rotation_lut[2][4] = {
+    {1, 8, 3, 6}, {4, 7, 2, 5},
+};
+
+int av_exif_matrix_to_orientation(const int32_t *matrix)
 {
-    double rotation = av_display_rotation_get(display_matrix);
+    double rotation = av_display_rotation_get(matrix);
     // determinant
-    int vflip = ((int64_t)display_matrix[0] * (int64_t)display_matrix[4]
-               - (int64_t)display_matrix[1] * (int64_t)display_matrix[3]) < 0;
+    int vflip = ((int64_t)matrix[0] * (int64_t)matrix[4]
+               - (int64_t)matrix[1] * (int64_t)matrix[3]) < 0;
     if (!isfinite(rotation))
-        return 1;
-    if (vflip) {
-        if (rotation > 181.0 || rotation > -179.0 && rotation < -89.0)
-            return 5;
-        if (rotation > 91.0 || rotation < -179.0 && rotation > -269.0)
-            return 2;
-        if (rotation > 1.0 || rotation < -269.0)
-            return 7;
-        return 4;
-    } else {
-        if (rotation > 181.0 || rotation > -179.0 && rotation < -89.0)
-            return 6;
-        if (rotation > 91.0 || rotation < -179.0 && rotation > -269.0)
-            return 3;
-        if (rotation > 1.0 || rotation < -269.0)
-            return 8;
-        return 1;
+        return 0;
+    int rot = (int)(rotation + 0.5);
+    rot = (((rot % 360) + 360) % 360) / 90;
+    return rotation_lut[vflip][rot];
+}
+
+int av_exif_orientation_to_matrix(int32_t *matrix, int orientation)
+{
+    switch (orientation) {
+        case 1:
+            av_display_rotation_set(matrix, 0.0);
+            break;
+        case 2:
+            av_display_rotation_set(matrix, 0.0);
+            av_display_matrix_flip(matrix, 1, 0);
+            break;
+        case 3:
+            av_display_rotation_set(matrix, 180.0);
+            break;
+        case 4:
+            av_display_rotation_set(matrix, 180.0);
+            av_display_matrix_flip(matrix, 1, 0);
+            break;
+        case 5:
+            av_display_rotation_set(matrix, 90.0);
+            av_display_matrix_flip(matrix, 1, 0);
+            break;
+        case 6:
+            av_display_rotation_set(matrix, 90.0);
+            break;
+        case 7:
+            av_display_rotation_set(matrix, -90.0);
+            av_display_matrix_flip(matrix, 1, 0);
+            break;
+        case 8:
+            av_display_rotation_set(matrix, -90.0);
+            break;
+        default:
+            return AVERROR(EINVAL);
     }
+
+    return 0;
 }
 
 int ff_exif_get_buffer(void *logctx, const AVFrame *frame, AVBufferRef **buffer_ptr, enum AVExifHeaderMode header_mode)
@@ -1270,7 +1268,7 @@ int ff_exif_get_buffer(void *logctx, const AVFrame *frame, AVBufferRef **buffer_
         return 0;
 
     if (sd_orient)
-        orientation = exif_get_orientation((int32_t *) sd_orient->data);
+        orientation = av_exif_matrix_to_orientation((int32_t *) sd_orient->data);
     if (orientation != 1)
         av_log(logctx, AV_LOG_DEBUG, "matrix contains nontrivial EXIF orientation: %" PRIu64 "\n", orientation);
 
diff --git a/libavcodec/exif.h b/libavcodec/exif.h
index 2661f86582..23c1b37c5d 100644
--- a/libavcodec/exif.h
+++ b/libavcodec/exif.h
@@ -184,6 +184,24 @@ int av_exif_ifd_to_dict(void *logctx, const AVExifMetadata *ifd, AVDictionary **
  */
 AVExifMetadata *av_exif_clone_ifd(const AVExifMetadata *ifd);
 
+/**
+ * Convert a display matrix used by AV_FRAME_DATA_DISPLAYMATRIX
+ * into an orientation constant used by EXIF's orientation tag.
+ *
+ * Returns an EXIF orientation between 1 and 8 (inclusive) depending
+ * on the rotation and flip factors. Returns 0 if the matrix is singular.
+ */
+int av_exif_matrix_to_orientation(const int32_t *matrix);
+
+/**
+ * Convert an orientation constant used by EXIF's orientation tag
+ * into a display matrix used by AV_FRAME_DATA_DISPLAYMATRIX.
+ *
+ * Returns 0 on success and negative if the orientation is invalid,
+ * i.e. not between 1 and 8 (inclusive).
+ */
+int av_exif_orientation_to_matrix(int32_t *matrix, int orientation);
+
 #if FF_API_OLD_EXIF
 /* Used by the AVI demuxer */
 int avpriv_exif_decode_ifd(void *logctx, const uint8_t *buf, int size,