Merge commit '98c97994c5b90bdae02accb155eeceeb5224b8ef'

* commit '98c97994c5b90bdae02accb155eeceeb5224b8ef':
  h264: decouple extradata parsing from the decoder

Main changes:

- move get_avc_nalsize() inside h264_parser.c and make it use
  H264ParseContext instead of H264Context. This helps fixing
  fate-flv-demux.

- Also use is_avc/nal_length_size from the H264ParseContext in various
  places instead of the H264Context one as that's the fields now filled
  by ff_h264_decode_extradata()

- h264_parse: dont fail decode_extradata_ps() due to nal split failure.
  Change by Michael to fix decoding of h264/ref_10.avi.

Merged-by: Clément Bœsch <u@pkh.me>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

1 files changed, 12 insertions, 141 deletions

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 831d91880e..b21835064b 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -300,120 +300,6 @@ fail:
     return AVERROR(ENOMEM); // ff_h264_free_tables will clean up for us
 }
 
-static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size,
-                            int parse_extradata);
-
-/* There are (invalid) samples in the wild with mp4-style extradata, where the
- * parameter sets are stored unescaped (i.e. as RBSP).
- * This function catches the parameter set decoding failure and tries again
- * after escaping it */
-static int decode_extradata_ps_mp4(H264Context *h, const uint8_t *buf, int buf_size)
-{
-    int ret;
-
-    ret = decode_nal_units(h, buf, buf_size, 1);
-    if (ret < 0 && !(h->avctx->err_recognition & AV_EF_EXPLODE)) {
-        GetByteContext gbc;
-        PutByteContext pbc;
-        uint8_t *escaped_buf;
-        int escaped_buf_size;
-
-        av_log(h->avctx, AV_LOG_WARNING,
-               "SPS decoding failure, trying again after escaping the NAL\n");
-
-        if (buf_size / 2 >= (INT16_MAX - AV_INPUT_BUFFER_PADDING_SIZE) / 3)
-            return AVERROR(ERANGE);
-        escaped_buf_size = buf_size * 3 / 2 + AV_INPUT_BUFFER_PADDING_SIZE;
-        escaped_buf = av_mallocz(escaped_buf_size);
-        if (!escaped_buf)
-            return AVERROR(ENOMEM);
-
-        bytestream2_init(&gbc, buf, buf_size);
-        bytestream2_init_writer(&pbc, escaped_buf, escaped_buf_size);
-
-        while (bytestream2_get_bytes_left(&gbc)) {
-            if (bytestream2_get_bytes_left(&gbc) >= 3 &&
-                bytestream2_peek_be24(&gbc) <= 3) {
-                bytestream2_put_be24(&pbc, 3);
-                bytestream2_skip(&gbc, 2);
-            } else
-                bytestream2_put_byte(&pbc, bytestream2_get_byte(&gbc));
-        }
-
-        escaped_buf_size = bytestream2_tell_p(&pbc);
-        AV_WB16(escaped_buf, escaped_buf_size - 2);
-
-        ret = decode_nal_units(h, escaped_buf, escaped_buf_size, 1);
-        av_freep(&escaped_buf);
-        if (ret < 0)
-            return ret;
-    }
-
-    return 0;
-}
-
-int ff_h264_decode_extradata(H264Context *h, const uint8_t *buf, int size)
-{
-    AVCodecContext *avctx = h->avctx;
-    int ret;
-
-    if (!buf || size <= 0)
-        return -1;
-
-    if (buf[0] == 1) {
-        int i, cnt, nalsize;
-        const unsigned char *p = buf;
-
-        h->is_avc = 1;
-
-        if (size < 7) {
-            av_log(avctx, AV_LOG_ERROR,
-                   "avcC %d too short\n", size);
-            return AVERROR_INVALIDDATA;
-        }
-        /* sps and pps in the avcC always have length coded with 2 bytes,
-         * so put a fake nal_length_size = 2 while parsing them */
-        h->nal_length_size = 2;
-        // Decode sps from avcC
-        cnt = *(p + 5) & 0x1f; // Number of sps
-        p  += 6;
-        for (i = 0; i < cnt; i++) {
-            nalsize = AV_RB16(p) + 2;
-            if(nalsize > size - (p-buf))
-                return AVERROR_INVALIDDATA;
-            ret = decode_extradata_ps_mp4(h, p, nalsize);
-            if (ret < 0) {
-                av_log(avctx, AV_LOG_ERROR,
-                       "Decoding sps %d from avcC failed\n", i);
-                return ret;
-            }
-            p += nalsize;
-        }
-        // Decode pps from avcC
-        cnt = *(p++); // Number of pps
-        for (i = 0; i < cnt; i++) {
-            nalsize = AV_RB16(p) + 2;
-            if(nalsize > size - (p-buf))
-                return AVERROR_INVALIDDATA;
-            ret = decode_extradata_ps_mp4(h, p, nalsize);
-            if (ret < 0) {
-                av_log(avctx, AV_LOG_ERROR,
-                       "Decoding pps %d from avcC failed\n", i);
-                return ret;
-            }
-            p += nalsize;
-        }
-        // Store right nal length size that will be used to parse all other nals
-        h->nal_length_size = (buf[4] & 0x03) + 1;
-    } else {
-        h->is_avc = 0;
-        ret = decode_nal_units(h, buf, size, 1);
-        if (ret < 0)
-            return ret;
-    }
-    return size;
-}
-
 static int h264_init_context(AVCodecContext *avctx, H264Context *h)
 {
     int i;
@@ -503,7 +389,9 @@ av_cold int ff_h264_decode_init(AVCodecContext *avctx)
     }
 
     if (avctx->extradata_size > 0 && avctx->extradata) {
-        ret = ff_h264_decode_extradata(h, avctx->extradata, avctx->extradata_size);
+        ret = ff_h264_decode_extradata(avctx->extradata, avctx->extradata_size,
+                                       &h->ps, &h->is_avc, &h->nal_length_size,
+                                       avctx->err_recognition, avctx);
         if (ret < 0) {
             h264_decode_end(avctx);
             return ret;
@@ -992,8 +880,7 @@ static void debug_green_metadata(const H264SEIGreenMetaData *gm, void *logctx)
     }
 }
 
-static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size,
-                            int parse_extradata)
+static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size)
 {
     AVCodecContext *const avctx = h->avctx;
     unsigned context_count = 0;
@@ -1025,8 +912,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size,
     if (ret < 0) {
         av_log(avctx, AV_LOG_ERROR,
                "Error splitting the input into NAL units.\n");
-        /* don't consider NAL parsing failure a fatal error when parsing extradata, as the stream may work without it */
-        return parse_extradata ? buf_size : ret;
+        return ret;
     }
 
     if (avctx->active_thread_type & FF_THREAD_FRAME)
@@ -1044,25 +930,6 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size,
             continue;
 
 again:
-        /* Ignore per frame NAL unit type during extradata
-         * parsing. Decoding slices is not possible in codec init
-         * with frame-mt */
-        if (parse_extradata) {
-            switch (nal->type) {
-            case NAL_IDR_SLICE:
-            case NAL_SLICE:
-            case NAL_DPA:
-            case NAL_DPB:
-            case NAL_DPC:
-                av_log(h->avctx, AV_LOG_WARNING,
-                       "Ignoring NAL %d in global header/extradata\n",
-                       nal->type);
-                // fall through to next case
-            case NAL_AUXILIARY_SLICE:
-                nal->type = NAL_FF_IGNORE;
-            }
-        }
-
         // FIXME these should stop being context-global variables
         h->nal_ref_idc   = nal->ref_idc;
         h->nal_unit_type = nal->type;
@@ -1440,14 +1307,18 @@ static int h264_decode_frame(AVCodecContext *avctx, void *data,
         int side_size;
         uint8_t *side = av_packet_get_side_data(avpkt, AV_PKT_DATA_NEW_EXTRADATA, &side_size);
         if (is_extra(side, side_size))
-            ff_h264_decode_extradata(h, side, side_size);
+            ff_h264_decode_extradata(side, side_size,
+                                     &h->ps, &h->is_avc, &h->nal_length_size,
+                                     avctx->err_recognition, avctx);
     }
     if(h->is_avc && buf_size >= 9 && buf[0]==1 && buf[2]==0 && (buf[4]&0xFC)==0xFC && (buf[5]&0x1F) && buf[8]==0x67){
         if (is_extra(buf, buf_size))
-            return ff_h264_decode_extradata(h, buf, buf_size);
+            return ff_h264_decode_extradata(buf, buf_size,
+                                            &h->ps, &h->is_avc, &h->nal_length_size,
+                                            avctx->err_recognition, avctx);
     }
 
-    buf_index = decode_nal_units(h, buf, buf_size, 0);
+    buf_index = decode_nal_units(h, buf, buf_size);
     if (buf_index < 0)
         return AVERROR_INVALIDDATA;