avcodec/utvideodec: decode to GBR(A)P

This is actually internal utvideo format.
Allows to make use of SIMD for median prediction for rgb(a) formats,
thus speeding up decoding.
Simplifies code, eases further developement and maintenance.

Update FATE because of pixel format switch.

Signed-off-by: Paul B Mahol <onemda@gmail.com>

7 files changed, 49 insertions, 277 deletions

diff --git a/libavcodec/utvideo.h b/libavcodec/utvideo.h
index 1d708658f5..9559c831fe 100644
--- a/libavcodec/utvideo.h
+++ b/libavcodec/utvideo.h
@@ -63,9 +63,6 @@ enum {
 /* Mapping of libavcodec prediction modes to Ut Video's */
 extern const int ff_ut_pred_order[5];
 
-/* Order of RGB(A) planes in Ut Video */
-extern const int ff_ut_rgb_order[4];
-
 typedef struct UtvideoContext {
     const AVClass *class;
     AVCodecContext *avctx;
diff --git a/libavcodec/utvideodec.c b/libavcodec/utvideodec.c
index 7979618f42..0c6f89e83a 100644
--- a/libavcodec/utvideodec.c
+++ b/libavcodec/utvideodec.c
@@ -333,21 +333,25 @@ fail:
     return AVERROR_INVALIDDATA;
 }
 
-static void restore_rgb_planes(uint8_t *src, int step, ptrdiff_t stride,
-                               int width, int height)
+static void restore_rgb_planes(AVFrame *frame, int width, int height)
 {
-    int i, j;
+    uint8_t *src_r = (uint8_t *)frame->data[2];
+    uint8_t *src_g = (uint8_t *)frame->data[0];
+    uint8_t *src_b = (uint8_t *)frame->data[1];
     uint8_t r, g, b;
+    int i, j;
 
     for (j = 0; j < height; j++) {
-        for (i = 0; i < width * step; i += step) {
-            r = src[i];
-            g = src[i + 1];
-            b = src[i + 2];
-            src[i]     = r + g - 0x80;
-            src[i + 2] = b + g - 0x80;
+        for (i = 0; i < width; i++) {
+            r = src_r[i];
+            g = src_g[i];
+            b = src_b[i];
+            src_r[i] = r + g - 0x80;
+            src_b[i] = b + g - 0x80;
         }
-        src += stride;
+        src_r += frame->linesize[2];
+        src_g += frame->linesize[0];
+        src_b += frame->linesize[1];
     }
 }
 
@@ -476,132 +480,6 @@ static void restore_median_planar_il(UtvideoContext *c, uint8_t *src, ptrdiff_t
     }
 }
 
-static void restore_median_packed(uint8_t *src, int step, ptrdiff_t stride,
-                                  int width, int height, int slices, int rmode)
-{
-    int i, j, slice;
-    int A, B, C;
-    uint8_t *bsrc;
-    int slice_start, slice_height;
-    const int cmask = ~rmode;
-
-    for (slice = 0; slice < slices; slice++) {
-        slice_start  = ((slice * height) / slices) & cmask;
-        slice_height = ((((slice + 1) * height) / slices) & cmask) -
-                       slice_start;
-
-        if (!slice_height)
-            continue;
-        bsrc = src + slice_start * stride;
-
-        // first line - left neighbour prediction
-        bsrc[0] += 0x80;
-        A = bsrc[0];
-        for (i = step; i < width * step; i += step) {
-            bsrc[i] += A;
-            A        = bsrc[i];
-        }
-        bsrc += stride;
-        if (slice_height <= 1)
-            continue;
-        // second line - first element has top prediction, the rest uses median
-        C        = bsrc[-stride];
-        bsrc[0] += C;
-        A        = bsrc[0];
-        for (i = step; i < width * step; i += step) {
-            B        = bsrc[i - stride];
-            bsrc[i] += mid_pred(A, B, (uint8_t)(A + B - C));
-            C        = B;
-            A        = bsrc[i];
-        }
-        bsrc += stride;
-        // the rest of lines use continuous median prediction
-        for (j = 2; j < slice_height; j++) {
-            for (i = 0; i < width * step; i += step) {
-                B        = bsrc[i - stride];
-                bsrc[i] += mid_pred(A, B, (uint8_t)(A + B - C));
-                C        = B;
-                A        = bsrc[i];
-            }
-            bsrc += stride;
-        }
-    }
-}
-
-/* UtVideo interlaced mode treats every two lines as a single one,
- * so restoring function should take care of possible padding between
- * two parts of the same "line".
- */
-static void restore_median_packed_il(uint8_t *src, int step, ptrdiff_t stride,
-                                     int width, int height, int slices, int rmode)
-{
-    int i, j, slice;
-    int A, B, C;
-    uint8_t *bsrc;
-    int slice_start, slice_height;
-    const int cmask   = ~(rmode ? 3 : 1);
-    const ptrdiff_t stride2 = stride << 1;
-
-    for (slice = 0; slice < slices; slice++) {
-        slice_start    = ((slice * height) / slices) & cmask;
-        slice_height   = ((((slice + 1) * height) / slices) & cmask) -
-                         slice_start;
-        slice_height >>= 1;
-        if (!slice_height)
-            continue;
-
-        bsrc = src + slice_start * stride;
-
-        // first line - left neighbour prediction
-        bsrc[0] += 0x80;
-        A        = bsrc[0];
-        for (i = step; i < width * step; i += step) {
-            bsrc[i] += A;
-            A        = bsrc[i];
-        }
-        for (i = 0; i < width * step; i += step) {
-            bsrc[stride + i] += A;
-            A                 = bsrc[stride + i];
-        }
-        bsrc += stride2;
-        if (slice_height <= 1)
-            continue;
-        // second line - first element has top prediction, the rest uses median
-        C        = bsrc[-stride2];
-        bsrc[0] += C;
-        A        = bsrc[0];
-        for (i = step; i < width * step; i += step) {
-            B        = bsrc[i - stride2];
-            bsrc[i] += mid_pred(A, B, (uint8_t)(A + B - C));
-            C        = B;
-            A        = bsrc[i];
-        }
-        for (i = 0; i < width * step; i += step) {
-            B                 = bsrc[i - stride];
-            bsrc[stride + i] += mid_pred(A, B, (uint8_t)(A + B - C));
-            C                 = B;
-            A                 = bsrc[stride + i];
-        }
-        bsrc += stride2;
-        // the rest of lines use continuous median prediction
-        for (j = 2; j < slice_height; j++) {
-            for (i = 0; i < width * step; i += step) {
-                B        = bsrc[i - stride2];
-                bsrc[i] += mid_pred(A, B, (uint8_t)(A + B - C));
-                C        = B;
-                A        = bsrc[i];
-            }
-            for (i = 0; i < width * step; i += step) {
-                B                 = bsrc[i - stride];
-                bsrc[i + stride] += mid_pred(A, B, (uint8_t)(A + B - C));
-                C                 = B;
-                A                 = bsrc[i + stride];
-            }
-            bsrc += stride2;
-        }
-    }
-}
-
 static void restore_gradient_planar(UtvideoContext *c, uint8_t *src, ptrdiff_t stride,
                                     int width, int height, int slices, int rmode)
 {
@@ -691,108 +569,6 @@ static void restore_gradient_planar_il(UtvideoContext *c, uint8_t *src, ptrdiff_
     }
 }
 
-static void restore_gradient_packed(uint8_t *src, int step, ptrdiff_t stride,
-                                    int width, int height, int slices, int rmode)
-{
-    int i, j, slice;
-    int A, B, C;
-    uint8_t *bsrc;
-    int slice_start, slice_height;
-    const int cmask = ~rmode;
-
-    for (slice = 0; slice < slices; slice++) {
-        slice_start  = ((slice * height) / slices) & cmask;
-        slice_height = ((((slice + 1) * height) / slices) & cmask) -
-                       slice_start;
-
-        if (!slice_height)
-            continue;
-        bsrc = src + slice_start * stride;
-
-        // first line - left neighbour prediction
-        bsrc[0] += 0x80;
-        A = bsrc[0];
-        for (i = step; i < width * step; i += step) {
-            bsrc[i] += A;
-            A        = bsrc[i];
-        }
-        bsrc += stride;
-        if (slice_height <= 1)
-            continue;
-        for (j = 1; j < slice_height; j++) {
-            // second line - first element has top prediction, the rest uses gradient
-            C        = bsrc[-stride];
-            bsrc[0] += C;
-            for (i = step; i < width * step; i += step) {
-                A = bsrc[i - stride];
-                B = bsrc[i - (stride + step)];
-                C = bsrc[i - step];
-                bsrc[i] = (A - B + C + bsrc[i]) & 0xFF;
-            }
-            bsrc += stride;
-        }
-    }
-}
-
-static void restore_gradient_packed_il(uint8_t *src, int step, ptrdiff_t stride,
-                                       int width, int height, int slices, int rmode)
-{
-    int i, j, slice;
-    int A, B, C;
-    uint8_t *bsrc;
-    int slice_start, slice_height;
-    const int cmask   = ~(rmode ? 3 : 1);
-    const ptrdiff_t stride2 = stride << 1;
-
-    for (slice = 0; slice < slices; slice++) {
-        slice_start    = ((slice * height) / slices) & cmask;
-        slice_height   = ((((slice + 1) * height) / slices) & cmask) -
-                         slice_start;
-        slice_height >>= 1;
-        if (!slice_height)
-            continue;
-
-        bsrc = src + slice_start * stride;
-
-        // first line - left neighbour prediction
-        bsrc[0] += 0x80;
-        A        = bsrc[0];
-        for (i = step; i < width * step; i += step) {
-            bsrc[i] += A;
-            A        = bsrc[i];
-        }
-        for (i = 0; i < width * step; i += step) {
-            bsrc[stride + i] += A;
-            A                 = bsrc[stride + i];
-        }
-        bsrc += stride2;
-        if (slice_height <= 1)
-            continue;
-        for (j = 1; j < slice_height; j++) {
-            // second line - first element has top prediction, the rest uses gradient
-            C        = bsrc[-stride2];
-            bsrc[0] += C;
-            for (i = step; i < width * step; i += step) {
-                A = bsrc[i - stride2];
-                B = bsrc[i - (stride2 + step)];
-                C = bsrc[i - step];
-                bsrc[i] = (A - B + C + bsrc[i]) & 0xFF;
-            }
-            A = bsrc[-stride];
-            B = bsrc[-(step + stride + stride - width * step)];
-            C = bsrc[width * step - step];
-            bsrc[stride] = (A - B + C + bsrc[stride]) & 0xFF;
-            for (i = step; i < width * step; i += step) {
-                A = bsrc[i - stride];
-                B = bsrc[i - (step + stride)];
-                C = bsrc[i - step + stride];
-                bsrc[i + stride] = (A - B + C + bsrc[i + stride]) & 0xFF;
-            }
-            bsrc += stride2;
-        }
-    }
-}
-
 static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
                         AVPacket *avpkt)
 {
@@ -887,41 +663,40 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
     }
 
     switch (c->avctx->pix_fmt) {
-    case AV_PIX_FMT_RGB24:
-    case AV_PIX_FMT_RGBA:
+    case AV_PIX_FMT_GBRP:
+    case AV_PIX_FMT_GBRAP:
         for (i = 0; i < c->planes; i++) {
-            ret = decode_plane(c, i, frame.f->data[0] + ff_ut_rgb_order[i],
-                               c->planes, frame.f->linesize[0], avctx->width,
+            ret = decode_plane(c, i, frame.f->data[i], 1,
+                               frame.f->linesize[i], avctx->width,
                                avctx->height, plane_start[i],
                                c->frame_pred == PRED_LEFT);
             if (ret)
                 return ret;
             if (c->frame_pred == PRED_MEDIAN) {
                 if (!c->interlaced) {
-                    restore_median_packed(frame.f->data[0] + ff_ut_rgb_order[i],
-                                          c->planes, frame.f->linesize[0], avctx->width,
+                    restore_median_planar(c, frame.f->data[i],
+                                          frame.f->linesize[i], avctx->width,
                                           avctx->height, c->slices, 0);
                 } else {
-                    restore_median_packed_il(frame.f->data[0] + ff_ut_rgb_order[i],
-                                             c->planes, frame.f->linesize[0],
+                    restore_median_planar_il(c, frame.f->data[i],
+                                             frame.f->linesize[i],
                                              avctx->width, avctx->height, c->slices,
                                              0);
                 }
             } else if (c->frame_pred == PRED_GRADIENT) {
                 if (!c->interlaced) {
-                    restore_gradient_packed(frame.f->data[0] + ff_ut_rgb_order[i],
-                                            c->planes, frame.f->linesize[0], avctx->width,
+                    restore_gradient_planar(c, frame.f->data[i],
+                                            frame.f->linesize[i], avctx->width,
                                             avctx->height, c->slices, 0);
                 } else {
-                    restore_gradient_packed_il(frame.f->data[0] + ff_ut_rgb_order[i],
-                                               c->planes, frame.f->linesize[0],
+                    restore_gradient_planar_il(c, frame.f->data[i],
+                                               frame.f->linesize[i],
                                                avctx->width, avctx->height, c->slices,
                                                0);
                 }
             }
         }
-        restore_rgb_planes(frame.f->data[0], c->planes, frame.f->linesize[0],
-                           avctx->width, avctx->height);
+        restore_rgb_planes(frame.f, avctx->width, avctx->height);
         break;
     case AV_PIX_FMT_GBRAP10:
     case AV_PIX_FMT_GBRP10:
@@ -1094,11 +869,11 @@ static av_cold int decode_init(AVCodecContext *avctx)
     switch (avctx->codec_tag) {
     case MKTAG('U', 'L', 'R', 'G'):
         c->planes      = 3;
-        avctx->pix_fmt = AV_PIX_FMT_RGB24;
+        avctx->pix_fmt = AV_PIX_FMT_GBRP;
         break;
     case MKTAG('U', 'L', 'R', 'A'):
         c->planes      = 4;
-        avctx->pix_fmt = AV_PIX_FMT_RGBA;
+        avctx->pix_fmt = AV_PIX_FMT_GBRAP;
         break;
     case MKTAG('U', 'L', 'Y', '0'):
         c->planes      = 3;
diff --git a/tests/ref/fate/utvideo_rgb_left b/tests/ref/fate/utvideo_rgb_left
index d2ccbcea03..23e48c426d 100644
--- a/tests/ref/fate/utvideo_rgb_left
+++ b/tests/ref/fate/utvideo_rgb_left
@@ -3,7 +3,7 @@
 #codec_id 0: rawvideo
 #dimensions 0: 640x480
 #sar 0: 0/1
-0,          0,          0,        1,   921600, 0x27e6001e
-0,          1,          1,        1,   921600, 0x7c0a92bc
-0,          2,          2,        1,   921600, 0x4d2be42c
-0,          3,          3,        1,   921600, 0x58ddd0be
+0,          0,          0,        1,   921600, 0xb457001e
+0,          1,          1,        1,   921600, 0xceff92bc
+0,          2,          2,        1,   921600, 0x762de42c
+0,          3,          3,        1,   921600, 0xef14d0be
diff --git a/tests/ref/fate/utvideo_rgb_median b/tests/ref/fate/utvideo_rgb_median
index 913b697be2..e7c623a8da 100644
--- a/tests/ref/fate/utvideo_rgb_median
+++ b/tests/ref/fate/utvideo_rgb_median
@@ -3,8 +3,8 @@
 #codec_id 0: rawvideo
 #dimensions 0: 640x480
 #sar 0: 0/1
-0,          0,          0,        1,   921600, 0x9776611f
-0,          1,          1,        1,   921600, 0xdbfa64f4
-0,          2,          2,        1,   921600, 0xed2a0580
-0,          3,          3,        1,   921600, 0x6ecc80bc
-0,          4,          4,        1,   921600, 0x58ddd0be
+0,          0,          0,        1,   921600, 0x85af611f
+0,          1,          1,        1,   921600, 0xc97a64f4
+0,          2,          2,        1,   921600, 0xb1db0580
+0,          3,          3,        1,   921600, 0xa18d80bc
+0,          4,          4,        1,   921600, 0xef14d0be
diff --git a/tests/ref/fate/utvideo_rgba_left b/tests/ref/fate/utvideo_rgba_left
index cb7876f1a7..b24b7f4bfa 100644
--- a/tests/ref/fate/utvideo_rgba_left
+++ b/tests/ref/fate/utvideo_rgba_left
@@ -3,8 +3,8 @@
 #codec_id 0: rawvideo
 #dimensions 0: 640x480
 #sar 0: 0/1
-0,          0,          0,        1,  1228800, 0xf1bc9432
-0,          1,          1,        1,  1228800, 0x8480d1e5
-0,          2,          2,        1,  1228800, 0xb01d5fb2
-0,          3,          3,        1,  1228800, 0x53cb42c4
-0,          4,          4,        1,  1228800, 0x2b2ea176
+0,          0,          0,        1,  1228800, 0xf9f49432
+0,          1,          1,        1,  1228800, 0xf089d1e5
+0,          2,          2,        1,  1228800, 0xbd025fb2
+0,          3,          3,        1,  1228800, 0x17bf42c4
+0,          4,          4,        1,  1228800, 0xe31ea176
diff --git a/tests/ref/fate/utvideo_rgba_median b/tests/ref/fate/utvideo_rgba_median
index cb7876f1a7..b24b7f4bfa 100644
--- a/tests/ref/fate/utvideo_rgba_median
+++ b/tests/ref/fate/utvideo_rgba_median
@@ -3,8 +3,8 @@
 #codec_id 0: rawvideo
 #dimensions 0: 640x480
 #sar 0: 0/1
-0,          0,          0,        1,  1228800, 0xf1bc9432
-0,          1,          1,        1,  1228800, 0x8480d1e5
-0,          2,          2,        1,  1228800, 0xb01d5fb2
-0,          3,          3,        1,  1228800, 0x53cb42c4
-0,          4,          4,        1,  1228800, 0x2b2ea176
+0,          0,          0,        1,  1228800, 0xf9f49432
+0,          1,          1,        1,  1228800, 0xf089d1e5
+0,          2,          2,        1,  1228800, 0xbd025fb2
+0,          3,          3,        1,  1228800, 0x17bf42c4
+0,          4,          4,        1,  1228800, 0xe31ea176
diff --git a/tests/ref/fate/utvideo_rgba_single_symbol b/tests/ref/fate/utvideo_rgba_single_symbol
index 553c83552e..a20dc3dda4 100644
--- a/tests/ref/fate/utvideo_rgba_single_symbol
+++ b/tests/ref/fate/utvideo_rgba_single_symbol
@@ -3,4 +3,4 @@
 #codec_id 0: rawvideo
 #dimensions 0: 1024x768
 #sar 0: 0/1
-0,          0,          0,        1,  3145728, 0xac95c593
+0,          0,          0,        1,  3145728, 0xa07dc593