lavfi/stereo3d: subsampled yuv support for non-anaglyph outputs

Signed-off-by: Paul B Mahol <onemda@gmail.com>

9 files changed, 80 insertions, 47 deletions

diff --git a/libavfilter/vf_stereo3d.c b/libavfilter/vf_stereo3d.c
index ec3ab478b8..56b523ba5e 100644
--- a/libavfilter/vf_stereo3d.c
+++ b/libavfilter/vf_stereo3d.c
@@ -136,6 +136,8 @@ typedef struct Stereo3DContext {
     int ana_matrix[3][6];
     int nb_planes;
     int linesize[4];
+    int pheight[4];
+    int hsub, vsub;
     int pixstep[4];
     AVFrame *prev;
     double ts_unit;
@@ -207,14 +209,39 @@ static const enum AVPixelFormat other_pix_fmts[] = {
     AV_PIX_FMT_GBRP12BE, AV_PIX_FMT_GBRP12LE,
     AV_PIX_FMT_GBRP14BE, AV_PIX_FMT_GBRP14LE,
     AV_PIX_FMT_GBRP16BE, AV_PIX_FMT_GBRP16LE,
+    AV_PIX_FMT_YUV410P,
+    AV_PIX_FMT_YUV411P,
+    AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUVA420P,
+    AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUVA422P,
+    AV_PIX_FMT_YUV440P,
     AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUVA444P,
+    AV_PIX_FMT_YUVJ411P,
+    AV_PIX_FMT_YUVJ420P,
+    AV_PIX_FMT_YUVJ422P,
+    AV_PIX_FMT_YUVJ440P,
     AV_PIX_FMT_YUVJ444P,
+    AV_PIX_FMT_YUV420P9LE,  AV_PIX_FMT_YUVA420P9LE,
+    AV_PIX_FMT_YUV420P9BE,  AV_PIX_FMT_YUVA420P9BE,
+    AV_PIX_FMT_YUV422P9LE,  AV_PIX_FMT_YUVA422P9LE,
+    AV_PIX_FMT_YUV422P9BE,  AV_PIX_FMT_YUVA422P9BE,
     AV_PIX_FMT_YUV444P9LE,  AV_PIX_FMT_YUVA444P9LE,
     AV_PIX_FMT_YUV444P9BE,  AV_PIX_FMT_YUVA444P9BE,
+    AV_PIX_FMT_YUV420P10LE, AV_PIX_FMT_YUVA420P10LE,
+    AV_PIX_FMT_YUV420P10BE, AV_PIX_FMT_YUVA420P10BE,
+    AV_PIX_FMT_YUV422P10LE, AV_PIX_FMT_YUVA422P10LE,
+    AV_PIX_FMT_YUV422P10BE, AV_PIX_FMT_YUVA422P10BE,
     AV_PIX_FMT_YUV444P10LE, AV_PIX_FMT_YUVA444P10LE,
     AV_PIX_FMT_YUV444P10BE, AV_PIX_FMT_YUVA444P10BE,
+    AV_PIX_FMT_YUV420P12BE,  AV_PIX_FMT_YUV420P12LE,
+    AV_PIX_FMT_YUV422P12BE,  AV_PIX_FMT_YUV422P12LE,
     AV_PIX_FMT_YUV444P12BE,  AV_PIX_FMT_YUV444P12LE,
+    AV_PIX_FMT_YUV420P14BE,  AV_PIX_FMT_YUV420P14LE,
+    AV_PIX_FMT_YUV422P14BE,  AV_PIX_FMT_YUV422P14LE,
     AV_PIX_FMT_YUV444P14BE,  AV_PIX_FMT_YUV444P14LE,
+    AV_PIX_FMT_YUV420P16LE, AV_PIX_FMT_YUVA420P16LE,
+    AV_PIX_FMT_YUV420P16BE, AV_PIX_FMT_YUVA420P16BE,
+    AV_PIX_FMT_YUV422P16LE, AV_PIX_FMT_YUVA422P16LE,
+    AV_PIX_FMT_YUV422P16BE, AV_PIX_FMT_YUVA422P16BE,
     AV_PIX_FMT_YUV444P16LE, AV_PIX_FMT_YUVA444P16LE,
     AV_PIX_FMT_YUV444P16BE, AV_PIX_FMT_YUVA444P16BE,
     AV_PIX_FMT_NONE
@@ -426,6 +453,10 @@ static int config_output(AVFilterLink *outlink)
     s->nb_planes = av_pix_fmt_count_planes(outlink->format);
     av_image_fill_max_pixsteps(s->pixstep, NULL, desc);
     s->ts_unit = av_q2d(av_inv_q(av_mul_q(outlink->frame_rate, outlink->time_base)));
+    s->pheight[1] = s->pheight[2] = FF_CEIL_RSHIFT(s->height, desc->log2_chroma_h);
+    s->pheight[0] = s->pheight[3] = s->height;
+    s->hsub = desc->log2_chroma_w;
+    s->vsub = desc->log2_chroma_h;
 
     return 0;
 }
@@ -488,10 +519,12 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *inpicref)
     }
 
     for (i = 0; i < 4; i++) {
-        in_off_left[i]   = (s->in.row_left   + s->in.off_lstep)  * ileft->linesize[i]  + s->in.off_left   * s->pixstep[i];
-        in_off_right[i]  = (s->in.row_right  + s->in.off_rstep)  * iright->linesize[i] + s->in.off_right  * s->pixstep[i];
-        out_off_left[i]  = (s->out.row_left  + s->out.off_lstep) * oleft->linesize[i]  + s->out.off_left  * s->pixstep[i];
-        out_off_right[i] = (s->out.row_right + s->out.off_rstep) * oright->linesize[i] + s->out.off_right * s->pixstep[i];
+        int hsub = i == 1 || i == 2 ? s->hsub : 0;
+        int vsub = i == 1 || i == 2 ? s->vsub : 0;
+        in_off_left[i]   = (FF_CEIL_RSHIFT(s->in.row_left,   vsub) + s->in.off_lstep)  * ileft->linesize[i]  + FF_CEIL_RSHIFT(s->in.off_left   * s->pixstep[i], hsub);
+        in_off_right[i]  = (FF_CEIL_RSHIFT(s->in.row_right,  vsub) + s->in.off_rstep)  * iright->linesize[i] + FF_CEIL_RSHIFT(s->in.off_right  * s->pixstep[i], hsub);
+        out_off_left[i]  = (FF_CEIL_RSHIFT(s->out.row_left,  vsub) + s->out.off_lstep) * oleft->linesize[i]  + FF_CEIL_RSHIFT(s->out.off_left  * s->pixstep[i], hsub);
+        out_off_right[i] = (FF_CEIL_RSHIFT(s->out.row_right, vsub) + s->out.off_rstep) * oright->linesize[i] + FF_CEIL_RSHIFT(s->out.off_right * s->pixstep[i], hsub);
     }
 
     switch (s->out.format) {
@@ -512,12 +545,12 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *inpicref)
                                 oleft->linesize[i] * s->row_step,
                                 ileft->data[i] + in_off_left[i],
                                 ileft->linesize[i] * s->row_step,
-                                s->linesize[i], s->height);
+                                s->linesize[i], s->pheight[i]);
             av_image_copy_plane(oright->data[i] + out_off_right[i],
                                 oright->linesize[i] * s->row_step,
                                 iright->data[i] + in_off_right[i],
                                 iright->linesize[i] * s->row_step,
-                                s->linesize[i], s->height);
+                                s->linesize[i], s->pheight[i]);
         }
         break;
     case MONO_L:
@@ -527,7 +560,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *inpicref)
             av_image_copy_plane(out->data[i], out->linesize[i],
                                 iright->data[i] + in_off_left[i],
                                 iright->linesize[i],
-                                s->linesize[i], s->height);
+                                s->linesize[i], s->pheight[i]);
         }
         break;
     case ANAGLYPH_RB_GRAY:
diff --git a/tests/ref/fate/filter-stereo3d-abr-ml b/tests/ref/fate/filter-stereo3d-abr-ml
index 41c913db50..d0597a7fb6 100644
--- a/tests/ref/fate/filter-stereo3d-abr-ml
+++ b/tests/ref/fate/filter-stereo3d-abr-ml
@@ -1,6 +1,6 @@
 #tb 0: 1/25
-0,          0,          0,        1,   152064, 0xb787afe4
-0,          1,          1,        1,   152064, 0x21f0e774
-0,          2,          2,        1,   152064, 0x5704de73
-0,          3,          3,        1,   152064, 0xe80907da
-0,          4,          4,        1,   152064, 0xbaed9048
+0,          0,          0,        1,    76032, 0xad4c8a22
+0,          1,          1,        1,    76032, 0x0dfdc6ad
+0,          2,          2,        1,    76032, 0x6341da83
+0,          3,          3,        1,    76032, 0x8dbcb05f
+0,          4,          4,        1,    76032, 0xad87bbee
diff --git a/tests/ref/fate/filter-stereo3d-abr-mr b/tests/ref/fate/filter-stereo3d-abr-mr
index 03217aa9c1..44a8091bb9 100644
--- a/tests/ref/fate/filter-stereo3d-abr-mr
+++ b/tests/ref/fate/filter-stereo3d-abr-mr
@@ -1,6 +1,6 @@
 #tb 0: 1/25
-0,          0,          0,        1,   152064, 0x3064732b
-0,          1,          1,        1,   152064, 0xc6cde2ab
-0,          2,          2,        1,   152064, 0x3c039f65
-0,          3,          3,        1,   152064, 0xbd3b53e0
-0,          4,          4,        1,   152064, 0x3ab585b6
+0,          0,          0,        1,    76032, 0x5d0effbe
+0,          1,          1,        1,    76032, 0x5bb49e95
+0,          2,          2,        1,    76032, 0xb6301bc7
+0,          3,          3,        1,    76032, 0x7cf1d042
+0,          4,          4,        1,    76032, 0x24fafa55
diff --git a/tests/ref/fate/filter-stereo3d-al-sbsl b/tests/ref/fate/filter-stereo3d-al-sbsl
index 8c92a44d37..b19cbb82c9 100644
--- a/tests/ref/fate/filter-stereo3d-al-sbsl
+++ b/tests/ref/fate/filter-stereo3d-al-sbsl
@@ -1,6 +1,6 @@
 #tb 0: 2/25
-0,          0,          0,        1,   608256, 0x3b69ed4c
-0,          1,          1,        1,   608256, 0x0e09d9a1
-0,          2,          2,        1,   608256, 0x2edb143c
-0,          3,          3,        1,   608256, 0xb5084f1e
-0,          4,          4,        1,   608256, 0x8be7da2d
+0,          0,          0,        1,   304128, 0xb9c0ef40
+0,          1,          1,        1,   304128, 0xfa0b7709
+0,          2,          2,        1,   304128, 0x14255f47
+0,          3,          3,        1,   304128, 0xa9da07de
+0,          4,          4,        1,   304128, 0x9d64b93b
diff --git a/tests/ref/fate/filter-stereo3d-ar-abl b/tests/ref/fate/filter-stereo3d-ar-abl
index 0ef3a8ae2f..0358199f9e 100644
--- a/tests/ref/fate/filter-stereo3d-ar-abl
+++ b/tests/ref/fate/filter-stereo3d-ar-abl
@@ -1,6 +1,6 @@
 #tb 0: 2/25
-0,          0,          0,        1,   608256, 0x3f2fed4c
-0,          1,          1,        1,   608256, 0xc3c3d9a1
-0,          2,          2,        1,   608256, 0x80f5143c
-0,          3,          3,        1,   608256, 0x23004f1e
-0,          4,          4,        1,   608256, 0xc596da2d
+0,          0,          0,        1,   304128, 0x7babef40
+0,          1,          1,        1,   304128, 0x287d7709
+0,          2,          2,        1,   304128, 0x385f5f47
+0,          3,          3,        1,   304128, 0x503507de
+0,          4,          4,        1,   304128, 0xb97db93b
diff --git a/tests/ref/fate/filter-stereo3d-sbsl-abl b/tests/ref/fate/filter-stereo3d-sbsl-abl
index e925b3906a..273e520ba7 100644
--- a/tests/ref/fate/filter-stereo3d-sbsl-abl
+++ b/tests/ref/fate/filter-stereo3d-sbsl-abl
@@ -1,6 +1,6 @@
 #tb 0: 1/25
-0,          0,          0,        1,   304128, 0x4213231e
-0,          1,          1,        1,   304128, 0xfd78ca2e
-0,          2,          2,        1,   304128, 0xbd197de7
-0,          3,          3,        1,   304128, 0x1e935bba
-0,          4,          4,        1,   304128, 0x45b1160d
+0,          0,          0,        1,   152064, 0x2f9389ef
+0,          1,          1,        1,   152064, 0x16c66551
+0,          2,          2,        1,   152064, 0xd779f64a
+0,          3,          3,        1,   152064, 0x7a7c80b0
+0,          4,          4,        1,   152064, 0x0702b652
diff --git a/tests/ref/fate/filter-stereo3d-sbsl-abr b/tests/ref/fate/filter-stereo3d-sbsl-abr
index 025b451f78..c827e3c456 100644
--- a/tests/ref/fate/filter-stereo3d-sbsl-abr
+++ b/tests/ref/fate/filter-stereo3d-sbsl-abr
@@ -1,6 +1,6 @@
 #tb 0: 1/25
-0,          0,          0,        1,   304128, 0x5ac6231e
-0,          1,          1,        1,   304128, 0x4ea3ca2e
-0,          2,          2,        1,   304128, 0x1b717de7
-0,          3,          3,        1,   304128, 0xf9e75bba
-0,          4,          4,        1,   304128, 0xb3b3160d
+0,          0,          0,        1,   152064, 0xd1f489ef
+0,          1,          1,        1,   152064, 0xe4fe6551
+0,          2,          2,        1,   152064, 0xd0d9f64a
+0,          3,          3,        1,   152064, 0xfd3a80b0
+0,          4,          4,        1,   152064, 0x488eb652
diff --git a/tests/ref/fate/filter-stereo3d-sbsl-al b/tests/ref/fate/filter-stereo3d-sbsl-al
index b6af912aeb..5c8a429903 100644
--- a/tests/ref/fate/filter-stereo3d-sbsl-al
+++ b/tests/ref/fate/filter-stereo3d-sbsl-al
@@ -1,6 +1,6 @@
 #tb 0: 1/50
-0,          0,          0,        1,   152064, 0x269fdfa9
-0,          1,          1,        1,   152064, 0xfbea4366
-0,          2,          2,        1,   152064, 0xc957dddc
-0,          3,          3,        1,   152064, 0xf3dcec43
-0,          4,          4,        1,   152064, 0x0b5d30a2
+0,          0,          0,        1,    76032, 0x3ff06515
+0,          1,          1,        1,    76032, 0x00d824da
+0,          2,          2,        1,    76032, 0x64a33c64
+0,          3,          3,        1,    76032, 0xeef328ed
+0,          4,          4,        1,    76032, 0x4bb209cc
diff --git a/tests/ref/fate/filter-stereo3d-sbsl-sbsr b/tests/ref/fate/filter-stereo3d-sbsl-sbsr
index f6263029ea..b417ae4f4e 100644
--- a/tests/ref/fate/filter-stereo3d-sbsl-sbsr
+++ b/tests/ref/fate/filter-stereo3d-sbsl-sbsr
@@ -1,6 +1,6 @@
 #tb 0: 1/25
-0,          0,          0,        1,   304128, 0x341f231e
-0,          1,          1,        1,   304128, 0x6230ca2e
-0,          2,          2,        1,   304128, 0x364e7de7
-0,          3,          3,        1,   304128, 0xd8305bba
-0,          4,          4,        1,   304128, 0xdb0a160d
+0,          0,          0,        1,   152064, 0xbe2f89ef
+0,          1,          1,        1,   152064, 0x4c806551
+0,          2,          2,        1,   152064, 0x8165f64a
+0,          3,          3,        1,   152064, 0xadf280b0
+0,          4,          4,        1,   152064, 0x6677b652