avcodec/snow: gray support

Fixes Ticket839

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

4 files changed, 77 insertions, 46 deletions

diff --git a/libavcodec/snow.c b/libavcodec/snow.c
index 1c27ae2cf9..4ea82b35e9 100644
--- a/libavcodec/snow.c
+++ b/libavcodec/snow.c
@@ -491,7 +491,7 @@ int ff_snow_common_init_after_header(AVCodecContext *avctx) {
         return AVERROR_INVALIDDATA;
     }
 
-    for(plane_index=0; plane_index<3; plane_index++){
+    for(plane_index=0; plane_index < s->nb_planes; plane_index++){
         int w= s->avctx->width;
         int h= s->avctx->height;
 
@@ -549,7 +549,7 @@ fail:
 static int halfpel_interpol(SnowContext *s, uint8_t *halfpel[4][4], AVFrame *frame){
     int p,x,y;
 
-    for(p=0; p<3; p++){
+    for(p=0; p < s->nb_planes; p++){
         int is_chroma= !!p;
         int w= is_chroma ? s->avctx->width >>s->chroma_h_shift : s->avctx->width;
         int h= is_chroma ? s->avctx->height>>s->chroma_v_shift : s->avctx->height;
@@ -614,12 +614,14 @@ int ff_snow_frame_start(SnowContext *s){
         s->dsp.draw_edges(s->current_picture->data[0],
                           s->current_picture->linesize[0], w   , h   ,
                           EDGE_WIDTH  , EDGE_WIDTH  , EDGE_TOP | EDGE_BOTTOM);
-        s->dsp.draw_edges(s->current_picture->data[1],
-                          s->current_picture->linesize[1], w>>s->chroma_h_shift, h>>s->chroma_v_shift,
-                          EDGE_WIDTH>>s->chroma_h_shift, EDGE_WIDTH>>s->chroma_v_shift, EDGE_TOP | EDGE_BOTTOM);
-        s->dsp.draw_edges(s->current_picture->data[2],
-                          s->current_picture->linesize[2], w>>s->chroma_h_shift, h>>s->chroma_v_shift,
-                          EDGE_WIDTH>>s->chroma_h_shift, EDGE_WIDTH>>s->chroma_v_shift, EDGE_TOP | EDGE_BOTTOM);
+        if (s->current_picture->data[2]) {
+            s->dsp.draw_edges(s->current_picture->data[1],
+                            s->current_picture->linesize[1], w>>s->chroma_h_shift, h>>s->chroma_v_shift,
+                            EDGE_WIDTH>>s->chroma_h_shift, EDGE_WIDTH>>s->chroma_v_shift, EDGE_TOP | EDGE_BOTTOM);
+            s->dsp.draw_edges(s->current_picture->data[2],
+                            s->current_picture->linesize[2], w>>s->chroma_h_shift, h>>s->chroma_v_shift,
+                            EDGE_WIDTH>>s->chroma_h_shift, EDGE_WIDTH>>s->chroma_v_shift, EDGE_TOP | EDGE_BOTTOM);
+        }
     }
 
     ff_snow_release_buffer(s->avctx);
@@ -686,7 +688,7 @@ av_cold void ff_snow_common_end(SnowContext *s)
         av_frame_free(&s->last_picture[i]);
     }
 
-    for(plane_index=0; plane_index<3; plane_index++){
+    for(plane_index=0; plane_index < s->nb_planes; plane_index++){
         for(level=s->spatial_decomposition_count-1; level>=0; level--){
             for(orientation=level ? 1 : 0; orientation<4; orientation++){
                 SubBand *b= &s->plane[plane_index].band[level][orientation];
diff --git a/libavcodec/snow.h b/libavcodec/snow.h
index 06f3731f9e..c3e50adf5b 100644
--- a/libavcodec/snow.h
+++ b/libavcodec/snow.h
@@ -159,6 +159,7 @@ typedef struct SnowContext{
     int b_height;
     int block_max_depth;
     int last_block_max_depth;
+    int nb_planes;
     Plane plane[MAX_PLANES];
     BlockNode *block;
 #define ME_CACHE_SIZE 1024
diff --git a/libavcodec/snowdec.c b/libavcodec/snowdec.c
index 8da2f17ffb..8bd4dc19c9 100644
--- a/libavcodec/snowdec.c
+++ b/libavcodec/snowdec.c
@@ -163,8 +163,10 @@ static int decode_q_branch(SnowContext *s, int level, int x, int y){
         if(type){
             pred_mv(s, &mx, &my, 0, left, top, tr);
             l += get_symbol(&s->c, &s->block_state[32], 1);
-            cb+= get_symbol(&s->c, &s->block_state[64], 1);
-            cr+= get_symbol(&s->c, &s->block_state[96], 1);
+            if (s->nb_planes > 2) {
+                cb+= get_symbol(&s->c, &s->block_state[64], 1);
+                cr+= get_symbol(&s->c, &s->block_state[96], 1);
+            }
         }else{
             if(s->ref_frames > 1)
                 ref= get_symbol(&s->c, &s->block_state[128 + 1024 + 32*ref_context], 0);
@@ -243,7 +245,7 @@ static void correlate_slice_buffered(SnowContext *s, slice_buffer * sb, SubBand
 static void decode_qlogs(SnowContext *s){
     int plane_index, level, orientation;
 
-    for(plane_index=0; plane_index<3; plane_index++){
+    for(plane_index=0; plane_index < s->nb_planes; plane_index++){
         for(level=0; level<s->spatial_decomposition_count; level++){
             for(orientation=level ? 1:0; orientation<4; orientation++){
                 int q;
@@ -286,22 +288,34 @@ static int decode_header(SnowContext *s){
         s->temporal_decomposition_count= get_symbol(&s->c, s->header_state, 0);
         GET_S(s->spatial_decomposition_count, 0 < tmp && tmp <= MAX_DECOMPOSITIONS)
         s->colorspace_type= get_symbol(&s->c, s->header_state, 0);
-        s->chroma_h_shift= get_symbol(&s->c, s->header_state, 0);
-        s->chroma_v_shift= get_symbol(&s->c, s->header_state, 0);
-
-        if(s->chroma_h_shift == 1 && s->chroma_v_shift==1){
-            s->avctx->pix_fmt= AV_PIX_FMT_YUV420P;
-        }else if(s->chroma_h_shift == 0 && s->chroma_v_shift==0){
-            s->avctx->pix_fmt= AV_PIX_FMT_YUV444P;
-        }else if(s->chroma_h_shift == 2 && s->chroma_v_shift==2){
-            s->avctx->pix_fmt= AV_PIX_FMT_YUV410P;
+        if (s->colorspace_type == 1) {
+            s->avctx->pix_fmt= AV_PIX_FMT_GRAY8;
+            s->nb_planes = 1;
+        } else if(s->colorspace_type == 0) {
+            s->chroma_h_shift= get_symbol(&s->c, s->header_state, 0);
+            s->chroma_v_shift= get_symbol(&s->c, s->header_state, 0);
+
+            if(s->chroma_h_shift == 1 && s->chroma_v_shift==1){
+                s->avctx->pix_fmt= AV_PIX_FMT_YUV420P;
+            }else if(s->chroma_h_shift == 0 && s->chroma_v_shift==0){
+                s->avctx->pix_fmt= AV_PIX_FMT_YUV444P;
+            }else if(s->chroma_h_shift == 2 && s->chroma_v_shift==2){
+                s->avctx->pix_fmt= AV_PIX_FMT_YUV410P;
+            } else {
+                av_log(s, AV_LOG_ERROR, "unsupported color subsample mode %d %d\n", s->chroma_h_shift, s->chroma_v_shift);
+                s->chroma_h_shift = s->chroma_v_shift = 1;
+                s->avctx->pix_fmt= AV_PIX_FMT_YUV420P;
+                return AVERROR_INVALIDDATA;
+            }
+            s->nb_planes = 3;
         } else {
-            av_log(s, AV_LOG_ERROR, "unsupported color subsample mode %d %d\n", s->chroma_h_shift, s->chroma_v_shift);
+            av_log(s, AV_LOG_ERROR, "unsupported color space\n");
             s->chroma_h_shift = s->chroma_v_shift = 1;
             s->avctx->pix_fmt= AV_PIX_FMT_YUV420P;
             return AVERROR_INVALIDDATA;
         }
 
+
         s->spatial_scalability= get_rac(&s->c, s->header_state);
 //        s->rate_scalability= get_rac(&s->c, s->header_state);
         GET_S(s->max_ref_frames, tmp < (unsigned)MAX_REF_FRAMES)
@@ -312,7 +326,7 @@ static int decode_header(SnowContext *s){
 
     if(!s->keyframe){
         if(get_rac(&s->c, s->header_state)){
-            for(plane_index=0; plane_index<2; plane_index++){
+            for(plane_index=0; plane_index<FFMIN(s->nb_planes, 2); plane_index++){
                 int htaps, i, sum=0;
                 Plane *p= &s->plane[plane_index];
                 p->diag_mc= get_rac(&s->c, s->header_state);
@@ -418,7 +432,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
                                     s->spatial_idwt_buffer)) < 0)
         return res;
 
-    for(plane_index=0; plane_index<3; plane_index++){
+    for(plane_index=0; plane_index < s->nb_planes; plane_index++){
         Plane *p= &s->plane[plane_index];
         p->fast_mc= p->diag_mc && p->htaps==6 && p->hcoeff[0]==40
                                               && p->hcoeff[1]==-10
@@ -436,7 +450,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
     if ((res = decode_blocks(s)) < 0)
         return res;
 
-    for(plane_index=0; plane_index<3; plane_index++){
+    for(plane_index=0; plane_index < s->nb_planes; plane_index++){
         Plane *p= &s->plane[plane_index];
         int w= p->width;
         int h= p->height;
diff --git a/libavcodec/snowenc.c b/libavcodec/snowenc.c
index 10864e7c96..befa4d1373 100644
--- a/libavcodec/snowenc.c
+++ b/libavcodec/snowenc.c
@@ -107,11 +107,15 @@ static av_cold int encode_init(AVCodecContext *avctx)
     case AV_PIX_FMT_YUV444P:
 //    case AV_PIX_FMT_YUV422P:
     case AV_PIX_FMT_YUV420P:
-//     case AV_PIX_FMT_GRAY8:
 //    case AV_PIX_FMT_YUV411P:
     case AV_PIX_FMT_YUV410P:
+        s->nb_planes = 3;
         s->colorspace_type= 0;
         break;
+    case AV_PIX_FMT_GRAY8:
+        s->nb_planes = 1;
+        s->colorspace_type = 1;
+        break;
 /*    case AV_PIX_FMT_RGB32:
         s->colorspace= 1;
         break;*/
@@ -363,13 +367,16 @@ static int encode_q_branch(SnowContext *s, int level, int x, int y){
     l= (sum + block_s/2)/block_s;
     iscore = pix_norm1(current_data[0], stride, block_w) - 2*l*sum + l*l*block_s;
 
-    block_s= block_w*block_w>>(s->chroma_h_shift + s->chroma_v_shift);
-    sum = pix_sum(current_data[1], uvstride, block_w>>s->chroma_h_shift, block_w>>s->chroma_v_shift);
-    cb= (sum + block_s/2)/block_s;
-//    iscore += pix_norm1(&current_mb[1][0], uvstride, block_w>>1) - 2*cb*sum + cb*cb*block_s;
-    sum = pix_sum(current_data[2], uvstride, block_w>>s->chroma_h_shift, block_w>>s->chroma_v_shift);
-    cr= (sum + block_s/2)/block_s;
-//    iscore += pix_norm1(&current_mb[2][0], uvstride, block_w>>1) - 2*cr*sum + cr*cr*block_s;
+    if (s->nb_planes > 2) {
+        block_s= block_w*block_w>>(s->chroma_h_shift + s->chroma_v_shift);
+        sum = pix_sum(current_data[1], uvstride, block_w>>s->chroma_h_shift, block_w>>s->chroma_v_shift);
+        cb= (sum + block_s/2)/block_s;
+    //    iscore += pix_norm1(&current_mb[1][0], uvstride, block_w>>1) - 2*cb*sum + cb*cb*block_s;
+        sum = pix_sum(current_data[2], uvstride, block_w>>s->chroma_h_shift, block_w>>s->chroma_v_shift);
+        cr= (sum + block_s/2)/block_s;
+    //    iscore += pix_norm1(&current_mb[2][0], uvstride, block_w>>1) - 2*cr*sum + cr*cr*block_s;
+    }else
+        cb = cr = 0;
 
     ic= s->c;
     ic.bytestream_start=
@@ -379,8 +386,10 @@ static int encode_q_branch(SnowContext *s, int level, int x, int y){
         put_rac(&ic, &i_state[4 + s_context], 1);
     put_rac(&ic, &i_state[1 + left->type + top->type], 1);
     put_symbol(&ic, &i_state[32],  l-pl , 1);
-    put_symbol(&ic, &i_state[64], cb-pcb, 1);
-    put_symbol(&ic, &i_state[96], cr-pcr, 1);
+    if (s->nb_planes > 2) {
+        put_symbol(&ic, &i_state[64], cb-pcb, 1);
+        put_symbol(&ic, &i_state[96], cr-pcr, 1);
+    }
     i_len= ic.bytestream - ic.bytestream_start;
     iscore += (s->lambda2*(get_rac_count(&ic)-base_bits))>>FF_LAMBDA_SHIFT;
 
@@ -471,8 +480,10 @@ static void encode_q_branch2(SnowContext *s, int level, int x, int y){
         pred_mv(s, &pmx, &pmy, 0, left, top, tr);
         put_rac(&s->c, &s->block_state[1 + (left->type&1) + (top->type&1)], 1);
         put_symbol(&s->c, &s->block_state[32], b->color[0]-pl , 1);
-        put_symbol(&s->c, &s->block_state[64], b->color[1]-pcb, 1);
-        put_symbol(&s->c, &s->block_state[96], b->color[2]-pcr, 1);
+        if (s->nb_planes > 2) {
+            put_symbol(&s->c, &s->block_state[64], b->color[1]-pcb, 1);
+            put_symbol(&s->c, &s->block_state[96], b->color[2]-pcr, 1);
+        }
         set_blocks(s, level, x, y, b->color[0], b->color[1], b->color[2], pmx, pmy, 0, BLOCK_INTRA);
     }else{
         pred_mv(s, &pmx, &pmy, b->ref, left, top, tr);
@@ -1072,7 +1083,7 @@ static void iterative_me(SnowContext *s){
                 }
 
                 // intra(black) = neighbors' contribution to the current block
-                for(i=0; i<3; i++)
+                for(i=0; i < s->nb_planes; i++)
                     color[i]= get_dc(s, mb_x, mb_y, i);
 
                 // get previous score (cannot be cached due to OBMC)
@@ -1350,7 +1361,7 @@ static void correlate(SnowContext *s, SubBand *b, IDWTELEM *src, int stride, int
 static void encode_qlogs(SnowContext *s){
     int plane_index, level, orientation;
 
-    for(plane_index=0; plane_index<2; plane_index++){
+    for(plane_index=0; plane_index<FFMIN(s->nb_planes, 2); plane_index++){
         for(level=0; level<s->spatial_decomposition_count; level++){
             for(orientation=level ? 1:0; orientation<4; orientation++){
                 if(orientation==2) continue;
@@ -1388,8 +1399,10 @@ static void encode_header(SnowContext *s){
         put_symbol(&s->c, s->header_state, s->temporal_decomposition_count, 0);
         put_symbol(&s->c, s->header_state, s->spatial_decomposition_count, 0);
         put_symbol(&s->c, s->header_state, s->colorspace_type, 0);
-        put_symbol(&s->c, s->header_state, s->chroma_h_shift, 0);
-        put_symbol(&s->c, s->header_state, s->chroma_v_shift, 0);
+        if (s->nb_planes > 2) {
+            put_symbol(&s->c, s->header_state, s->chroma_h_shift, 0);
+            put_symbol(&s->c, s->header_state, s->chroma_v_shift, 0);
+        }
         put_rac(&s->c, s->header_state, s->spatial_scalability);
 //        put_rac(&s->c, s->header_state, s->rate_scalability);
         put_symbol(&s->c, s->header_state, s->max_ref_frames-1, 0);
@@ -1399,7 +1412,7 @@ static void encode_header(SnowContext *s){
 
     if(!s->keyframe){
         int update_mc=0;
-        for(plane_index=0; plane_index<2; plane_index++){
+        for(plane_index=0; plane_index<FFMIN(s->nb_planes, 2); plane_index++){
             Plane *p= &s->plane[plane_index];
             update_mc |= p->last_htaps   != p->htaps;
             update_mc |= p->last_diag_mc != p->diag_mc;
@@ -1407,7 +1420,7 @@ static void encode_header(SnowContext *s){
         }
         put_rac(&s->c, s->header_state, update_mc);
         if(update_mc){
-            for(plane_index=0; plane_index<2; plane_index++){
+            for(plane_index=0; plane_index<FFMIN(s->nb_planes, 2); plane_index++){
                 Plane *p= &s->plane[plane_index];
                 put_rac(&s->c, s->header_state, p->diag_mc);
                 put_symbol(&s->c, s->header_state, p->htaps/2-1, 0);
@@ -1552,7 +1565,7 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     ff_init_range_encoder(c, pkt->data, pkt->size);
     ff_build_rac_states(c, 0.05*(1LL<<32), 256-8);
 
-    for(i=0; i<3; i++){
+    for(i=0; i < s->nb_planes; i++){
         int hshift= i ? s->chroma_h_shift : 0;
         int vshift= i ? s->chroma_v_shift : 0;
         for(y=0; y<(height>>vshift); y++)
@@ -1670,7 +1683,7 @@ redo_frame:
     ff_snow_common_init_after_header(avctx);
 
     if(s->last_spatial_decomposition_count != s->spatial_decomposition_count){
-        for(plane_index=0; plane_index<3; plane_index++){
+        for(plane_index=0; plane_index < s->nb_planes; plane_index++){
             calculate_visual_weight(s, &s->plane[plane_index]);
         }
     }
@@ -1680,7 +1693,7 @@ redo_frame:
     encode_blocks(s, 1);
     s->m.mv_bits = 8*(s->c.bytestream - s->c.bytestream_start) - s->m.misc_bits;
 
-    for(plane_index=0; plane_index<3; plane_index++){
+    for(plane_index=0; plane_index < s->nb_planes; plane_index++){
         Plane *p= &s->plane[plane_index];
         int w= p->width;
         int h= p->height;
@@ -1872,6 +1885,7 @@ AVCodec ff_snow_encoder = {
     .close          = encode_end,
     .pix_fmts       = (const enum AVPixelFormat[]){
         AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV444P,
+        AV_PIX_FMT_GRAY8,
         AV_PIX_FMT_NONE
     },
     .long_name      = NULL_IF_CONFIG_SMALL("Snow"),