lavc/hevc/ps: implement SPS parsing for nuh_layer_id>0

Cf. F.7.3.2.2 "Sequence parameter set RBSP syntax", which extends normal
SPS parsing with special clauses depending on MultiLayerExtSpsFlag.

6 files changed, 68 insertions, 13 deletions

diff --git a/libavcodec/hevc/hevcdec.c b/libavcodec/hevc/hevcdec.c
index d915d74d22..ebe2f5ad11 100644
--- a/libavcodec/hevc/hevcdec.c
+++ b/libavcodec/hevc/hevcdec.c
@@ -3260,7 +3260,7 @@ static int decode_nal_unit(HEVCContext *s, const H2645NAL *nal)
         break;
     case HEVC_NAL_SPS:
         ret = ff_hevc_decode_nal_sps(&gb, s->avctx, &s->ps,
-                                     s->apply_defdispwin);
+                                     nal->nuh_layer_id, s->apply_defdispwin);
         if (ret < 0)
             goto fail;
         break;
diff --git a/libavcodec/hevc/parse.c b/libavcodec/hevc/parse.c
index ec8d1aeacf..ad84b7b152 100644
--- a/libavcodec/hevc/parse.c
+++ b/libavcodec/hevc/parse.c
@@ -49,7 +49,8 @@ static int hevc_decode_nal_units(const uint8_t *buf, int buf_size, HEVCParamSets
                 goto done;
             break;
         case HEVC_NAL_SPS:
-            ret = ff_hevc_decode_nal_sps(&nal->gb, logctx, ps, apply_defdispwin);
+            ret = ff_hevc_decode_nal_sps(&nal->gb, logctx, ps,
+                                         nal->nuh_layer_id, apply_defdispwin);
             if (ret < 0)
                 goto done;
             break;
diff --git a/libavcodec/hevc/parser.c b/libavcodec/hevc/parser.c
index 2d14b4fae2..8c7444a162 100644
--- a/libavcodec/hevc/parser.c
+++ b/libavcodec/hevc/parser.c
@@ -209,7 +209,7 @@ static int parse_nal_units(AVCodecParserContext *s, const uint8_t *buf,
             ff_hevc_decode_nal_vps(gb, avctx, ps);
             break;
         case HEVC_NAL_SPS:
-            ff_hevc_decode_nal_sps(gb, avctx, ps, 1);
+            ff_hevc_decode_nal_sps(gb, avctx, ps, nal->nuh_layer_id, 1);
             break;
         case HEVC_NAL_PPS:
             ff_hevc_decode_nal_pps(gb, avctx, ps);
diff --git a/libavcodec/hevc/ps.c b/libavcodec/hevc/ps.c
index 81b3f9cd09..2d09500278 100644
--- a/libavcodec/hevc/ps.c
+++ b/libavcodec/hevc/ps.c
@@ -1148,12 +1148,12 @@ static int map_pixel_format(AVCodecContext *avctx, HEVCSPS *sps)
 }
 
 int ff_hevc_parse_sps(HEVCSPS *sps, GetBitContext *gb, unsigned int *sps_id,
-                      int apply_defdispwin, const HEVCVPS * const *vps_list,
-                      AVCodecContext *avctx)
+                      unsigned nuh_layer_id, int apply_defdispwin,
+                      const HEVCVPS * const *vps_list, AVCodecContext *avctx)
 {
     HEVCWindow *ow;
     int ret = 0;
-    int bit_depth_chroma, start, num_comps;
+    int bit_depth_chroma, num_comps, multi_layer_ext;
     int i;
 
     // Coded parameters
@@ -1170,16 +1170,29 @@ int ff_hevc_parse_sps(HEVCSPS *sps, GetBitContext *gb, unsigned int *sps_id,
     }
 
     sps->max_sub_layers = get_bits(gb, 3) + 1;
+    multi_layer_ext = nuh_layer_id > 0 &&
+                      sps->max_sub_layers == HEVC_MAX_SUB_LAYERS + 1;
+    if (multi_layer_ext) {
+        if (!sps->vps)
+            return AVERROR(EINVAL);
+
+        sps->max_sub_layers = sps->vps->vps_max_sub_layers;
+    }
     if (sps->max_sub_layers > HEVC_MAX_SUB_LAYERS) {
         av_log(avctx, AV_LOG_ERROR, "sps_max_sub_layers out of range: %d\n",
                sps->max_sub_layers);
         return AVERROR_INVALIDDATA;
     }
 
+    if (!multi_layer_ext) {
     sps->temporal_id_nesting = get_bits(gb, 1);
 
     if ((ret = parse_ptl(gb, avctx, 1, &sps->ptl, sps->max_sub_layers)) < 0)
         return ret;
+    } else {
+        sps->temporal_id_nesting = sps->max_sub_layers > 1 ?
+                                   sps->vps->vps_max_sub_layers : 1;
+    }
 
     *sps_id = get_ue_golomb_long(gb);
     if (*sps_id >= HEVC_MAX_SPS_COUNT) {
@@ -1187,6 +1200,28 @@ int ff_hevc_parse_sps(HEVCSPS *sps, GetBitContext *gb, unsigned int *sps_id,
         return AVERROR_INVALIDDATA;
     }
 
+    if (multi_layer_ext) {
+        const RepFormat *rf = &sps->vps->rep_format;
+
+        if (get_bits1(gb) &&    // update_rep_format_flag
+            get_bits(gb, 8)) {  // sps_rep_format_idx
+            av_log(avctx, AV_LOG_ERROR, "sps_rep_format_idx!=0\n");
+            return AVERROR_PATCHWELCOME;
+        }
+
+        sps->separate_colour_plane = rf->separate_colour_plane_flag;
+        sps->chroma_format_idc     = sps->separate_colour_plane ? 0 :
+                                     rf->chroma_format_idc;
+        sps->bit_depth             = rf->bit_depth_luma;
+        sps->width                 = rf->pic_width_in_luma_samples;
+        sps->height                = rf->pic_height_in_luma_samples;
+
+        sps->pic_conf_win.left_offset   = rf->conf_win_left_offset;
+        sps->pic_conf_win.right_offset  = rf->conf_win_right_offset;
+        sps->pic_conf_win.top_offset    = rf->conf_win_top_offset;
+        sps->pic_conf_win.bottom_offset = rf->conf_win_bottom_offset;
+
+    } else {
     sps->chroma_format_idc = get_ue_golomb_long(gb);
     if (sps->chroma_format_idc > 3U) {
         av_log(avctx, AV_LOG_ERROR, "chroma_format_idc %d is invalid\n", sps->chroma_format_idc);
@@ -1228,7 +1263,6 @@ int ff_hevc_parse_sps(HEVCSPS *sps, GetBitContext *gb, unsigned int *sps_id,
             sps->pic_conf_win.top_offset    =
             sps->pic_conf_win.bottom_offset = 0;
         }
-        sps->output_window = sps->pic_conf_win;
     }
 
     sps->bit_depth = get_ue_golomb_31(gb) + 8;
@@ -1251,6 +1285,9 @@ int ff_hevc_parse_sps(HEVCSPS *sps, GetBitContext *gb, unsigned int *sps_id,
         return AVERROR_INVALIDDATA;
     }
     sps->bit_depth_chroma = bit_depth_chroma;
+    }
+
+    sps->output_window = sps->pic_conf_win;
 
     ret = map_pixel_format(avctx, sps);
     if (ret < 0)
@@ -1263,6 +1300,9 @@ int ff_hevc_parse_sps(HEVCSPS *sps, GetBitContext *gb, unsigned int *sps_id,
         return AVERROR_INVALIDDATA;
     }
 
+    if (!multi_layer_ext) {
+    int start;
+
     sps->sublayer_ordering_info = get_bits1(gb);
     start = sps->sublayer_ordering_info ? 0 : sps->max_sub_layers - 1;
     for (i = start; i < sps->max_sub_layers; i++) {
@@ -1292,6 +1332,13 @@ int ff_hevc_parse_sps(HEVCSPS *sps, GetBitContext *gb, unsigned int *sps_id,
             sps->temporal_layer[i].max_latency_increase  = sps->temporal_layer[start].max_latency_increase;
         }
     }
+    } else {
+        for (int i = 0; i < sps->max_sub_layers; i++) {
+            sps->temporal_layer[i].max_dec_pic_buffering = sps->vps->dpb_size.max_dec_pic_buffering;
+            sps->temporal_layer[i].num_reorder_pics      = sps->vps->dpb_size.max_num_reorder_pics;
+            sps->temporal_layer[i].max_latency_increase  = sps->vps->dpb_size.max_latency_increase;
+        }
+    }
 
     sps->log2_min_cb_size                       = get_ue_golomb_long(gb) + 3;
     sps->log2_diff_max_min_coding_block_size    = get_ue_golomb_long(gb);
@@ -1328,6 +1375,11 @@ int ff_hevc_parse_sps(HEVCSPS *sps, GetBitContext *gb, unsigned int *sps_id,
     if (sps->scaling_list_enabled) {
         set_default_scaling_list_data(&sps->scaling_list);
 
+        if (multi_layer_ext && get_bits1(gb)) { // sps_infer_scaling_list_flag
+            av_log(avctx, AV_LOG_ERROR, "sps_infer_scaling_list_flag=1 not supported\n");
+            return AVERROR_PATCHWELCOME;
+        }
+
         if (get_bits1(gb)) {
             ret = scaling_list_data(gb, avctx, &sps->scaling_list, sps);
             if (ret < 0)
@@ -1582,7 +1634,8 @@ static int compare_sps(const HEVCSPS *sps1, const HEVCSPS *sps2)
 }
 
 int ff_hevc_decode_nal_sps(GetBitContext *gb, AVCodecContext *avctx,
-                           HEVCParamSets *ps, int apply_defdispwin)
+                           HEVCParamSets *ps, unsigned nuh_layer_id,
+                           int apply_defdispwin)
 {
     HEVCSPS *sps = ff_refstruct_alloc_ext(sizeof(*sps), 0, NULL, hevc_sps_free);
     unsigned int sps_id;
@@ -1601,7 +1654,7 @@ int ff_hevc_decode_nal_sps(GetBitContext *gb, AVCodecContext *avctx,
     }
 
     ret = ff_hevc_parse_sps(sps, gb, &sps_id,
-                            apply_defdispwin,
+                            nuh_layer_id, apply_defdispwin,
                             ps->vps_list, avctx);
     if (ret < 0)
         goto err;
diff --git a/libavcodec/hevc/ps.h b/libavcodec/hevc/ps.h
index bd1acf12e6..6f5b1f8755 100644
--- a/libavcodec/hevc/ps.h
+++ b/libavcodec/hevc/ps.h
@@ -521,13 +521,14 @@ typedef struct HEVCParamSets {
  *                 to an existing VPS
  */
 int ff_hevc_parse_sps(HEVCSPS *sps, GetBitContext *gb, unsigned int *sps_id,
-                      int apply_defdispwin, const HEVCVPS * const *vps_list,
-                      AVCodecContext *avctx);
+                      unsigned nuh_layer_id, int apply_defdispwin,
+                      const HEVCVPS * const *vps_list, AVCodecContext *avctx);
 
 int ff_hevc_decode_nal_vps(GetBitContext *gb, AVCodecContext *avctx,
                            HEVCParamSets *ps);
 int ff_hevc_decode_nal_sps(GetBitContext *gb, AVCodecContext *avctx,
-                           HEVCParamSets *ps, int apply_defdispwin);
+                           HEVCParamSets *ps, unsigned nuh_layer_id,
+                           int apply_defdispwin);
 int ff_hevc_decode_nal_pps(GetBitContext *gb, AVCodecContext *avctx,
                            HEVCParamSets *ps);
 
diff --git a/libavcodec/qsvenc_hevc.c b/libavcodec/qsvenc_hevc.c
index 2a397a2919..80fdf782a7 100644
--- a/libavcodec/qsvenc_hevc.c
+++ b/libavcodec/qsvenc_hevc.c
@@ -99,7 +99,7 @@ static int generate_fake_vps(QSVEncContext *q, AVCodecContext *avctx)
     }
     get_bits(&gb, 9);
 
-    ret = ff_hevc_parse_sps(&sps, &gb, &sps_id, 0, NULL, avctx);
+    ret = ff_hevc_parse_sps(&sps, &gb, &sps_id, 0, 0, NULL, avctx);
     av_freep(&sps_rbsp.rbsp_buffer);
     if (ret < 0) {
         av_log(avctx, AV_LOG_ERROR, "Error parsing the SPS\n");