h264: set parameters from SPS whenever it changes

Fixes a crash in the fuzzed sample sample_varPAR.avi_s26638 with
alternating bit depths.

3 files changed, 68 insertions, 49 deletions

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 95a8f4fe0d..e4887ffa67 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -2340,6 +2340,52 @@ int ff_h264_get_profile(SPS *sps)
     return profile;
 }
 
+static int h264_set_parameter_from_sps(H264Context *h)
+{
+    MpegEncContext *s = &h->s;
+
+    if (s->flags & CODEC_FLAG_LOW_DELAY ||
+        (h->sps.bitstream_restriction_flag &&
+         !h->sps.num_reorder_frames)) {
+        if (s->avctx->has_b_frames > 1 || h->delayed_pic[0])
+            av_log(h->s.avctx, AV_LOG_WARNING, "Delayed frames seen. "
+                   "Reenabling low delay requires a codec flush.\n");
+        else
+            s->low_delay = 1;
+    }
+
+    if (s->avctx->has_b_frames < 2)
+        s->avctx->has_b_frames = !s->low_delay;
+
+    if (s->avctx->bits_per_raw_sample != h->sps.bit_depth_luma ||
+        h->cur_chroma_format_idc      != h->sps.chroma_format_idc) {
+        if (s->avctx->codec &&
+            s->avctx->codec->capabilities & CODEC_CAP_HWACCEL_VDPAU &&
+            (h->sps.bit_depth_luma != 8 || h->sps.chroma_format_idc > 1)) {
+            av_log(s->avctx, AV_LOG_ERROR,
+                   "VDPAU decoding does not support video colorspace.\n");
+            return AVERROR_INVALIDDATA;
+        }
+        if (h->sps.bit_depth_luma >= 8 && h->sps.bit_depth_luma <= 10) {
+            s->avctx->bits_per_raw_sample = h->sps.bit_depth_luma;
+            h->cur_chroma_format_idc      = h->sps.chroma_format_idc;
+            h->pixel_shift                = h->sps.bit_depth_luma > 8;
+
+            ff_h264dsp_init(&h->h264dsp, h->sps.bit_depth_luma,
+                            h->sps.chroma_format_idc);
+            ff_h264_pred_init(&h->hpc, s->codec_id, h->sps.bit_depth_luma,
+                              h->sps.chroma_format_idc);
+            s->dsp.dct_bits = h->sps.bit_depth_luma > 8 ? 32 : 16;
+            ff_dsputil_init(&s->dsp, s->avctx);
+        } else {
+            av_log(s->avctx, AV_LOG_ERROR, "Unsupported bit depth: %d\n",
+                   h->sps.bit_depth_luma);
+            return AVERROR_INVALIDDATA;
+        }
+    }
+    return 0;
+}
+
 /**
  * Decode a slice header.
  * This will also call ff_MPV_common_init() and frame_start() as needed.
@@ -2356,7 +2402,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
     MpegEncContext *const s0 = &h0->s;
     unsigned int first_mb_in_slice;
     unsigned int pps_id;
-    int num_ref_idx_active_override_flag, max_refs;
+    int num_ref_idx_active_override_flag, max_refs, ret;
     unsigned int slice_type, tmp, i, j;
     int default_ref_list_done = 0;
     int last_pic_structure, last_pic_droppable;
@@ -2432,7 +2478,17 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
                h->pps.sps_id);
         return -1;
     }
-    h->sps = *h0->sps_buffers[h->pps.sps_id];
+
+    if (h->pps.sps_id != h->current_sps_id ||
+        h0->sps_buffers[h->pps.sps_id]->new) {
+        h0->sps_buffers[h->pps.sps_id]->new = 0;
+
+        h->current_sps_id = h->pps.sps_id;
+        h->sps            = *h0->sps_buffers[h->pps.sps_id];
+
+        if ((ret = h264_set_parameter_from_sps(h)) < 0)
+            return ret;
+    }
 
     s->avctx->profile = ff_h264_get_profile(&h->sps);
     s->avctx->level   = h->sps.level_idc;
@@ -3878,51 +3934,9 @@ again:
                     ff_h264_decode_seq_parameter_set(h);
                 }
 
-                if (s->flags & CODEC_FLAG_LOW_DELAY ||
-                    (h->sps.bitstream_restriction_flag &&
-                     !h->sps.num_reorder_frames)) {
-                    if (s->avctx->has_b_frames > 1 || h->delayed_pic[0])
-                        av_log(avctx, AV_LOG_WARNING, "Delayed frames seen "
-                               "reenabling low delay requires a codec "
-                               "flush.\n");
-                        else
-                            s->low_delay = 1;
-                }
-
-                if (avctx->has_b_frames < 2)
-                    avctx->has_b_frames = !s->low_delay;
-
-                if (avctx->bits_per_raw_sample != h->sps.bit_depth_luma ||
-                    h->cur_chroma_format_idc   != h->sps.chroma_format_idc) {
-                    if (s->avctx->codec &&
-                        s->avctx->codec->capabilities & CODEC_CAP_HWACCEL_VDPAU
-                        && (h->sps.bit_depth_luma != 8 ||
-                            h->sps.chroma_format_idc > 1)) {
-                        av_log(avctx, AV_LOG_ERROR,
-                               "VDPAU decoding does not support video "
-                               "colorspace\n");
-                        buf_index = -1;
-                        goto end;
-                    }
-                    if (h->sps.bit_depth_luma >= 8 && h->sps.bit_depth_luma <= 10) {
-                        avctx->bits_per_raw_sample = h->sps.bit_depth_luma;
-                        h->cur_chroma_format_idc   = h->sps.chroma_format_idc;
-                        h->pixel_shift             = h->sps.bit_depth_luma > 8;
-
-                        ff_h264dsp_init(&h->h264dsp, h->sps.bit_depth_luma,
-                                        h->sps.chroma_format_idc);
-                        ff_h264_pred_init(&h->hpc, s->codec_id,
-                                          h->sps.bit_depth_luma,
-                                          h->sps.chroma_format_idc);
-                        s->dsp.dct_bits = h->sps.bit_depth_luma > 8 ? 32 : 16;
-                        ff_dsputil_init(&s->dsp, s->avctx);
-                    } else {
-                        av_log(avctx, AV_LOG_ERROR,
-                               "Unsupported bit depth: %d\n",
-                               h->sps.bit_depth_luma);
-                        buf_index = -1;
-                        goto end;
-                    }
+                if (h264_set_parameter_from_sps(h) < 0) {
+                    buf_index = -1;
+                    goto end;
                 }
                 break;
             case NAL_PPS:
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index b8ded57fdc..17f63c4b5b 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -203,6 +203,7 @@ typedef struct SPS {
     int bit_depth_chroma;                 ///< bit_depth_chroma_minus8 + 8
     int residual_color_transform_flag;    ///< residual_colour_transform_flag
     int constraint_set_flags;             ///< constraint_set[0-3]_flag
+    int new;                              ///< flag to keep track if the decoder context needs re-init due to changed SPS
 } SPS;
 
 /**
@@ -329,6 +330,7 @@ typedef struct H264Context {
     int emu_edge_width;
     int emu_edge_height;
 
+    unsigned current_sps_id; ///< id of the current SPS
     SPS sps; ///< current sps
 
     /**
diff --git a/libavcodec/h264_ps.c b/libavcodec/h264_ps.c
index 7555daacb6..0166ad2e94 100644
--- a/libavcodec/h264_ps.c
+++ b/libavcodec/h264_ps.c
@@ -459,10 +459,13 @@ int ff_h264_decode_seq_parameter_set(H264Context *h){
                sps->timing_info_present_flag ? sps->time_scale : 0
                );
     }
+    sps->new = 1;
 
     av_free(h->sps_buffers[sps_id]);
-    h->sps_buffers[sps_id]= sps;
-    h->sps = *sps;
+    h->sps_buffers[sps_id] = sps;
+    h->sps                 = *sps;
+    h->current_sps_id      = sps_id;
+
     return 0;
 fail:
     av_free(sps);