h264: return meaningful values

Signed-off-by: Anton Khirnov <anton@khirnov.net>

1 files changed, 94 insertions, 77 deletions

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 89f3e2cf9f..f68b189294 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -414,7 +414,7 @@ int ff_h264_check_intra4x4_pred_mode(H264Context *h)
                 av_log(h->avctx, AV_LOG_ERROR,
                        "top block unavailable for requested intra4x4 mode %d at %d %d\n",
                        status, h->mb_x, h->mb_y);
-                return -1;
+                return AVERROR_INVALIDDATA;
             } else if (status) {
                 h->intra4x4_pred_mode_cache[scan8[0] + i] = status;
             }
@@ -430,7 +430,7 @@ int ff_h264_check_intra4x4_pred_mode(H264Context *h)
                     av_log(h->avctx, AV_LOG_ERROR,
                            "left block unavailable for requested intra4x4 mode %d at %d %d\n",
                            status, h->mb_x, h->mb_y);
-                    return -1;
+                    return AVERROR_INVALIDDATA;
                 } else if (status) {
                     h->intra4x4_pred_mode_cache[scan8[0] + 8 * i] = status;
                 }
@@ -453,7 +453,7 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma)
         av_log(h->avctx, AV_LOG_ERROR,
                "out of range intra chroma pred mode at %d %d\n",
                h->mb_x, h->mb_y);
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 
     if (!(h->top_samples_available & 0x8000)) {
@@ -462,7 +462,7 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma)
             av_log(h->avctx, AV_LOG_ERROR,
                    "top block unavailable for requested intra mode at %d %d\n",
                    h->mb_x, h->mb_y);
-            return -1;
+            return AVERROR_INVALIDDATA;
         }
     }
 
@@ -478,7 +478,7 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma)
             av_log(h->avctx, AV_LOG_ERROR,
                    "left block unavailable for requested intra mode at %d %d\n",
                    h->mb_x, h->mb_y);
-            return -1;
+            return AVERROR_INVALIDDATA;
         }
     }
 
@@ -1259,7 +1259,7 @@ int ff_h264_alloc_tables(H264Context *h)
 
 fail:
     free_tables(h, 1);
-    return -1;
+    return AVERROR(ENOMEM);
 }
 
 /**
@@ -1358,7 +1358,7 @@ static int context_init(H264Context *h)
     return 0;
 
 fail:
-    return -1; // free_tables will clean up for us
+    return AVERROR(ENOMEM); // free_tables will clean up for us
 }
 
 static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size,
@@ -1367,6 +1367,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size,
 int ff_h264_decode_extradata(H264Context *h)
 {
     AVCodecContext *avctx = h->avctx;
+    int ret;
 
     if (avctx->extradata[0] == 1) {
         int i, cnt, nalsize;
@@ -1376,7 +1377,7 @@ int ff_h264_decode_extradata(H264Context *h)
 
         if (avctx->extradata_size < 7) {
             av_log(avctx, AV_LOG_ERROR, "avcC too short\n");
-            return -1;
+            return AVERROR_INVALIDDATA;
         }
         /* sps and pps in the avcC always have length coded with 2 bytes,
          * so put a fake nal_length_size = 2 while parsing them */
@@ -1387,11 +1388,12 @@ int ff_h264_decode_extradata(H264Context *h)
         for (i = 0; i < cnt; i++) {
             nalsize = AV_RB16(p) + 2;
             if (p - avctx->extradata + nalsize > avctx->extradata_size)
-                return -1;
-            if (decode_nal_units(h, p, nalsize, 1) < 0) {
+                return AVERROR_INVALIDDATA;
+            ret = decode_nal_units(h, p, nalsize, 1);
+            if (ret < 0) {
                 av_log(avctx, AV_LOG_ERROR,
                        "Decoding sps %d from avcC failed\n", i);
-                return -1;
+                return ret;
             }
             p += nalsize;
         }
@@ -1400,11 +1402,12 @@ int ff_h264_decode_extradata(H264Context *h)
         for (i = 0; i < cnt; i++) {
             nalsize = AV_RB16(p) + 2;
             if (p - avctx->extradata + nalsize > avctx->extradata_size)
-                return -1;
-            if (decode_nal_units(h, p, nalsize, 1) < 0) {
+                return AVERROR_INVALIDDATA;
+            ret = decode_nal_units(h, p, nalsize, 1);
+            if (ret < 0) {
                 av_log(avctx, AV_LOG_ERROR,
                        "Decoding pps %d from avcC failed\n", i);
-                return -1;
+                return ret;
             }
             p += nalsize;
         }
@@ -1412,8 +1415,9 @@ int ff_h264_decode_extradata(H264Context *h)
         h->nal_length_size = (avctx->extradata[4] & 0x03) + 1;
     } else {
         h->is_avc = 0;
-        if (decode_nal_units(h, avctx->extradata, avctx->extradata_size, 1) < 0)
-            return -1;
+        ret = decode_nal_units(h, avctx->extradata, avctx->extradata_size, 1);
+        if (ret < 0)
+            return ret;
     }
     return 0;
 }
@@ -1422,6 +1426,7 @@ av_cold int ff_h264_decode_init(AVCodecContext *avctx)
 {
     H264Context *h = avctx->priv_data;
     int i;
+    int ret;
 
     h->avctx = avctx;
 
@@ -1473,9 +1478,11 @@ av_cold int ff_h264_decode_init(AVCodecContext *avctx)
         avctx->ticks_per_frame = 2;
     }
 
-    if (avctx->extradata_size > 0 && avctx->extradata &&
-        ff_h264_decode_extradata(h))
-        return -1;
+    if (avctx->extradata_size > 0 && avctx->extradata) {
+       ret = ff_h264_decode_extradata(h);
+       if (ret < 0)
+           return ret;
+    }
 
     if (h->sps.bitstream_restriction_flag &&
         h->avctx->has_b_frames < h->sps.num_reorder_frames) {
@@ -1625,9 +1632,10 @@ static int decode_update_thread_context(AVCodecContext *dst,
         h->ref_index_pool    = NULL;
         h->motion_val_pool   = NULL;
 
-        if (ff_h264_alloc_tables(h) < 0) {
+        ret = ff_h264_alloc_tables(h);
+        if (ret < 0) {
             av_log(dst, AV_LOG_ERROR, "Could not allocate memory for h264\n");
-            return AVERROR(ENOMEM);
+            return ret;
         }
         context_init(h);
 
@@ -3002,7 +3010,7 @@ static int h264_slice_header_init(H264Context *h, int reinit)
     int nb_slices = (HAVE_THREADS &&
                      h->avctx->active_thread_type & FF_THREAD_SLICE) ?
                     h->avctx->thread_count : 1;
-    int i;
+    int i, ret;
 
     h->avctx->sample_aspect_ratio = h->sps.sar;
     av_assert0(h->avctx->sample_aspect_ratio.den);
@@ -3025,10 +3033,11 @@ static int h264_slice_header_init(H264Context *h, int reinit)
     h->prev_interlaced_frame = 1;
 
     init_scan_tables(h);
-    if (ff_h264_alloc_tables(h) < 0) {
+    ret = ff_h264_alloc_tables(h);
+    if (ret < 0) {
         av_log(h->avctx, AV_LOG_ERROR,
                "Could not allocate memory for h264\n");
-        return AVERROR(ENOMEM);
+        return ret;
     }
 
     if (nb_slices > MAX_THREADS || (nb_slices > h->mb_height && h->mb_height)) {
@@ -3044,9 +3053,10 @@ static int h264_slice_header_init(H264Context *h, int reinit)
     h->slice_context_count = nb_slices;
 
     if (!HAVE_THREADS || !(h->avctx->active_thread_type & FF_THREAD_SLICE)) {
-        if (context_init(h) < 0) {
+        ret = context_init(h);
+        if (ret < 0) {
             av_log(h->avctx, AV_LOG_ERROR, "context_init() failed.\n");
-            return -1;
+            return ret;
         }
     } else {
         for (i = 1; i < h->slice_context_count; i++) {
@@ -3085,9 +3095,9 @@ static int h264_slice_header_init(H264Context *h, int reinit)
         }
 
         for (i = 0; i < h->slice_context_count; i++)
-            if (context_init(h->thread_context[i]) < 0) {
+            if ((ret = context_init(h->thread_context[i])) < 0) {
                 av_log(h->avctx, AV_LOG_ERROR, "context_init() failed.\n");
-                return -1;
+                return ret;
             }
     }
 
@@ -3141,7 +3151,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
         av_log(h->avctx, AV_LOG_ERROR,
                "slice type too large (%d) at %d %d\n",
                h->slice_type, h->mb_x, h->mb_y);
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
     if (slice_type > 4) {
         slice_type -= 5;
@@ -3163,13 +3173,13 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
     pps_id = get_ue_golomb(&h->gb);
     if (pps_id >= MAX_PPS_COUNT) {
         av_log(h->avctx, AV_LOG_ERROR, "pps_id out of range\n");
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
     if (!h0->pps_buffers[pps_id]) {
         av_log(h->avctx, AV_LOG_ERROR,
                "non-existing PPS %u referenced\n",
                pps_id);
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
     h->pps = *h0->pps_buffers[pps_id];
 
@@ -3177,7 +3187,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
         av_log(h->avctx, AV_LOG_ERROR,
                "non-existing SPS %u referenced\n",
                h->pps.sps_id);
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 
     if (h->pps.sps_id != h->current_sps_id ||
@@ -3262,7 +3272,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
         if (h != h0) {
             av_log(h->avctx, AV_LOG_ERROR,
                    "Cannot (re-)initialize context during parallel decoding.\n");
-            return -1;
+            return AVERROR_PATCHWELCOME;
         }
 
         if ((ret = get_pixel_format(h)) < 0)
@@ -3390,19 +3400,20 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
             Picture *prev = h->short_ref_count ? h->short_ref[0] : NULL;
             av_log(h->avctx, AV_LOG_DEBUG, "Frame num gap %d %d\n",
                    h->frame_num, h->prev_frame_num);
-            if (h264_frame_start(h) < 0)
-                return -1;
+            ret = h264_frame_start(h);
+            if (ret < 0)
+                return ret;
             h->prev_frame_num++;
             h->prev_frame_num        %= 1 << h->sps.log2_max_frame_num;
             h->cur_pic_ptr->frame_num = h->prev_frame_num;
             ff_thread_report_progress(&h->cur_pic_ptr->tf, INT_MAX, 0);
             ff_thread_report_progress(&h->cur_pic_ptr->tf, INT_MAX, 1);
-            if ((ret = ff_generate_sliding_window_mmcos(h, 1)) < 0 &&
-                h->avctx->err_recognition & AV_EF_EXPLODE)
+            ret = ff_generate_sliding_window_mmcos(h, 1);
+            if (ret < 0 && (h->avctx->err_recognition & AV_EF_EXPLODE))
+                return ret;
+            ret = ff_h264_execute_ref_pic_marking(h, h->mmco, h->mmco_index);
+            if (ret < 0 && (h->avctx->err_recognition & AV_EF_EXPLODE))
                 return ret;
-            if (ff_h264_execute_ref_pic_marking(h, h->mmco, h->mmco_index) < 0 &&
-                (h->avctx->err_recognition & AV_EF_EXPLODE))
-                return AVERROR_INVALIDDATA;
             /* Error concealment: If a ref is missing, copy the previous ref
              * in its place.
              * FIXME: Avoiding a memcpy would be nice, but ref handling makes
@@ -3459,7 +3470,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
         if (!FIELD_PICTURE(h) || h0->first_field) {
             if (h264_frame_start(h) < 0) {
                 h0->first_field = 0;
-                return -1;
+                return AVERROR_INVALIDDATA;
             }
         } else {
             release_unused_pictures(h, 0);
@@ -3474,7 +3485,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
     if (first_mb_in_slice << FIELD_OR_MBAFF_PICTURE(h) >= h->mb_num ||
         first_mb_in_slice >= h->mb_num) {
         av_log(h->avctx, AV_LOG_ERROR, "first_mb_in_slice overflow\n");
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
     h->resync_mb_x = h->mb_x =  first_mb_in_slice % h->mb_width;
     h->resync_mb_y = h->mb_y = (first_mb_in_slice / h->mb_width) <<
@@ -3553,10 +3564,12 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
     if (!default_ref_list_done)
         ff_h264_fill_default_ref_list(h);
 
-    if (h->slice_type_nos != AV_PICTURE_TYPE_I &&
-        ff_h264_decode_ref_pic_list_reordering(h) < 0) {
-        h->ref_count[1] = h->ref_count[0] = 0;
-        return -1;
+    if (h->slice_type_nos != AV_PICTURE_TYPE_I) {
+       ret = ff_h264_decode_ref_pic_list_reordering(h);
+       if (ret < 0) {
+           h->ref_count[1] = h->ref_count[0] = 0;
+           return ret;
+       }
     }
 
     if ((h->pps.weighted_pred && h->slice_type_nos == AV_PICTURE_TYPE_P) ||
@@ -3579,12 +3592,13 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
     // or h->mmco, which will cause ref list mix-ups and decoding errors
     // further down the line. This may break decoding if the first slice is
     // corrupt, thus we only do this if frame-mt is enabled.
-    if (h->nal_ref_idc &&
-        ff_h264_decode_ref_pic_marking(h0, &h->gb,
-                                       !(h->avctx->active_thread_type & FF_THREAD_FRAME) ||
-                                       h0->current_slice == 0) < 0 &&
-        (h->avctx->err_recognition & AV_EF_EXPLODE))
-        return AVERROR_INVALIDDATA;
+    if (h->nal_ref_idc) {
+        ret = ff_h264_decode_ref_pic_marking(h0, &h->gb,
+                                             !(h->avctx->active_thread_type & FF_THREAD_FRAME) ||
+                                             h0->current_slice == 0);
+        if (ret < 0 && (h->avctx->err_recognition & AV_EF_EXPLODE))
+            return AVERROR_INVALIDDATA;
+    }
 
     if (FRAME_MBAFF(h)) {
         ff_h264_fill_mbaff_ref_list(h);
@@ -3603,7 +3617,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
         tmp = get_ue_golomb_31(&h->gb);
         if (tmp > 2) {
             av_log(h->avctx, AV_LOG_ERROR, "cabac_init_idc overflow\n");
-            return -1;
+            return AVERROR_INVALIDDATA;
         }
         h->cabac_init_idc = tmp;
     }
@@ -3612,7 +3626,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
     tmp = h->pps.init_qp + get_se_golomb(&h->gb);
     if (tmp > 51 + 6 * (h->sps.bit_depth_luma - 8)) {
         av_log(h->avctx, AV_LOG_ERROR, "QP %u out of range\n", tmp);
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
     h->qscale       = tmp;
     h->chroma_qp[0] = get_chroma_qp(h, 0, h->qscale);
@@ -3632,7 +3646,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
         if (tmp > 2) {
             av_log(h->avctx, AV_LOG_ERROR,
                    "deblocking_filter_idc %u out of range\n", tmp);
-            return -1;
+            return AVERROR_INVALIDDATA;
         }
         h->deblocking_filter = tmp;
         if (h->deblocking_filter < 2)
@@ -3646,7 +3660,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
                 av_log(h->avctx, AV_LOG_ERROR,
                        "deblocking filter parameters %d %d out of range\n",
                        h->slice_alpha_c0_offset, h->slice_beta_offset);
-                return -1;
+                return AVERROR_INVALIDDATA;
             }
         }
     }
@@ -3764,7 +3778,7 @@ int ff_h264_get_slice_type(const H264Context *h)
     case AV_PICTURE_TYPE_SI:
         return 4;
     default:
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 }
 
@@ -4180,7 +4194,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg)
                        h->cabac.bytestream_end - h->cabac.bytestream);
                 er_add_slice(h, h->resync_mb_x, h->resync_mb_y, h->mb_x,
                              h->mb_y, ER_MB_ERROR);
-                return -1;
+                return AVERROR_INVALIDDATA;
             }
 
             if (++h->mb_x >= h->mb_width) {
@@ -4227,7 +4241,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg)
                        "error while decoding MB %d %d\n", h->mb_x, h->mb_y);
                 er_add_slice(h, h->resync_mb_x, h->resync_mb_y, h->mb_x,
                              h->mb_y, ER_MB_ERROR);
-                return -1;
+                return ret;
             }
 
             if (++h->mb_x >= h->mb_width) {
@@ -4255,7 +4269,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg)
                                      h->mb_x - 1, h->mb_y,
                                      ER_MB_END);
 
-                        return -1;
+                        return AVERROR_INVALIDDATA;
                     }
                 }
             }
@@ -4263,6 +4277,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg)
             if (get_bits_left(&h->gb) <= 0 && h->mb_skip_run <= 0) {
                 tprintf(h->avctx, "slice end %d %d\n",
                         get_bits_count(&h->gb), h->gb.size_in_bits);
+
                 if (get_bits_left(&h->gb) == 0) {
                     er_add_slice(h, h->resync_mb_x, h->resync_mb_y,
                                  h->mb_x - 1, h->mb_y,
@@ -4275,7 +4290,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg)
                     er_add_slice(h, h->resync_mb_x, h->resync_mb_y, h->mb_x,
                                  h->mb_y, ER_MB_ERROR);
 
-                    return -1;
+                    return AVERROR_INVALIDDATA;
                 }
             }
         }
@@ -4333,6 +4348,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size,
     int pass = !(avctx->active_thread_type & FF_THREAD_FRAME);
     int nals_needed = 0; ///< number of NALs that need decoding before the next frame thread starts
     int nal_index;
+    int ret = 0;
 
     h->max_contexts = h->slice_context_count;
     if (!(avctx->flags2 & CODEC_FLAG2_CHUNKS)) {
@@ -4391,7 +4407,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size,
             ptr = ff_h264_decode_nal(hx, buf + buf_index, &dst_length,
                                      &consumed, next_avc - buf_index);
             if (ptr == NULL || dst_length < 0) {
-                buf_index = -1;
+                ret = -1;
                 goto end;
             }
             i = buf_index + consumed;
@@ -4462,7 +4478,7 @@ again:
                 if (h->nal_unit_type != NAL_IDR_SLICE) {
                     av_log(h->avctx, AV_LOG_ERROR,
                            "Invalid mix of idr and non-idr slices\n");
-                    buf_index = -1;
+                    ret = -1;
                     goto end;
                 }
                 idr(h); // FIXME ensure we don't lose some frames if there is reordering
@@ -4484,8 +4500,8 @@ again:
                         decode_postinit(h, nal_index >= nals_needed);
 
                     if (h->avctx->hwaccel &&
-                        h->avctx->hwaccel->start_frame(h->avctx, NULL, 0) < 0)
-                        return -1;
+                        (ret = h->avctx->hwaccel->start_frame(h->avctx, NULL, 0)) < 0)
+                        return ret;
                 }
 
                 if (hx->redundant_pic_count == 0 &&
@@ -4497,10 +4513,11 @@ again:
                      hx->slice_type_nos == AV_PICTURE_TYPE_I) &&
                     avctx->skip_frame < AVDISCARD_ALL) {
                     if (avctx->hwaccel) {
-                        if (avctx->hwaccel->decode_slice(avctx,
-                                                         &buf[buf_index - consumed],
-                                                         consumed) < 0)
-                            return -1;
+                        ret = avctx->hwaccel->decode_slice(avctx,
+                                                           &buf[buf_index - consumed],
+                                                           consumed);
+                        if (ret < 0)
+                            return ret;
                     } else
                         context_count++;
                 }
@@ -4541,8 +4558,8 @@ again:
                 break;
             case NAL_SPS:
                 init_get_bits(&h->gb, ptr, bit_length);
-                if (ff_h264_decode_seq_parameter_set(h) < 0 &&
-                    h->is_avc && (nalsize != consumed) && nalsize) {
+                ret = ff_h264_decode_seq_parameter_set(h);
+                if (ret < 0 && h->is_avc && (nalsize != consumed) && nalsize) {
                     av_log(h->avctx, AV_LOG_DEBUG,
                            "SPS decoding failure, trying again with the complete NAL\n");
                     init_get_bits(&h->gb, buf + buf_index + 1 - consumed,
@@ -4550,10 +4567,10 @@ again:
                     ff_h264_decode_seq_parameter_set(h);
                 }
 
-                if (h264_set_parameter_from_sps(h) < 0) {
-                    buf_index = -1;
+                ret = h264_set_parameter_from_sps(h);
+                if (ret < 0)
                     goto end;
-                }
+
                 break;
             case NAL_PPS:
                 init_get_bits(&h->gb, ptr, bit_length);
@@ -4602,7 +4619,7 @@ end:
                                   h->picture_structure == PICT_BOTTOM_FIELD);
     }
 
-    return buf_index;
+    return (ret < 0) ? ret : buf_index;
 }
 
 /**
@@ -4686,7 +4703,7 @@ out:
 
     buf_index = decode_nal_units(h, buf, buf_size, 0);
     if (buf_index < 0)
-        return -1;
+        return AVERROR_INVALIDDATA;
 
     if (!h->cur_pic_ptr && h->nal_unit_type == NAL_END_SEQUENCE) {
         buf_size = 0;
@@ -4697,7 +4714,7 @@ out:
         if (avctx->skip_frame >= AVDISCARD_NONREF)
             return 0;
         av_log(avctx, AV_LOG_ERROR, "no frame!\n");
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 
     if (!(avctx->flags2 & CODEC_FLAG2_CHUNKS) ||