Merge remote-tracking branch 'qatar/master'

* qatar/master:
  h264: drop ff_h264_ prefix from static function ff_h264_decode_rbsp_trailing()
  h264: Make ff_h264_decode_end() static, it is not used externally.
  output-example: K&R formatting cosmetics, comment spelling fixes
  avf: make the example output the proper message
  avf: fix audio writing in the output-example
  mov: don't overwrite existing indexes.
  lzw: fix potential integer overflow.
  truemotion: forbid invalid VLC bitsizes and token values.
  truemotion2: handle out-of-frame motion vectors through edge extension.
  configure: Check for a different SDL function

Conflicts:
	configure
	doc/examples/muxing.c
	libavcodec/truemotion2.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

4 files changed, 112 insertions, 38 deletions

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index b504dcf1cb..c6dc24391c 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -271,7 +271,7 @@ nsc:
  * Identify the exact end of the bitstream
  * @return the length of the trailing, or 0 if damaged
  */
-static int ff_h264_decode_rbsp_trailing(H264Context *h, const uint8_t *src)
+static int decode_rbsp_trailing(H264Context *h, const uint8_t *src)
 {
     int v = *src;
     int r;
@@ -4260,7 +4260,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size)
                     dst_length--;
             bit_length = !dst_length ? 0
                                      : (8 * dst_length -
-                                        ff_h264_decode_rbsp_trailing(h, ptr + dst_length - 1));
+                                        decode_rbsp_trailing(h, ptr + dst_length - 1));
 
             if (s->avctx->debug & FF_DEBUG_STARTCODE)
                 av_log(h->s.avctx, AV_LOG_DEBUG, "NAL %d/%d at %d/%d length %d pass %d\n", hx->nal_unit_type, hx->nal_ref_idc, buf_index, buf_size, dst_length, pass);
@@ -4606,7 +4606,7 @@ av_cold void ff_h264_free_context(H264Context *h)
         av_freep(h->pps_buffers + i);
 }
 
-av_cold int ff_h264_decode_end(AVCodecContext *avctx)
+static av_cold int h264_decode_end(AVCodecContext *avctx)
 {
     H264Context *h    = avctx->priv_data;
     MpegEncContext *s = &h->s;
@@ -4664,7 +4664,7 @@ AVCodec ff_h264_decoder = {
     .id                    = CODEC_ID_H264,
     .priv_data_size        = sizeof(H264Context),
     .init                  = ff_h264_decode_init,
-    .close                 = ff_h264_decode_end,
+    .close                 = h264_decode_end,
     .decode                = decode_frame,
     .capabilities          = /*CODEC_CAP_DRAW_HORIZ_BAND |*/ CODEC_CAP_DR1 |
                              CODEC_CAP_DELAY | CODEC_CAP_SLICE_THREADS |
@@ -4684,7 +4684,7 @@ AVCodec ff_h264_vdpau_decoder = {
     .id             = CODEC_ID_H264,
     .priv_data_size = sizeof(H264Context),
     .init           = ff_h264_decode_init,
-    .close          = ff_h264_decode_end,
+    .close          = h264_decode_end,
     .decode         = decode_frame,
     .capabilities   = CODEC_CAP_DR1 | CODEC_CAP_DELAY | CODEC_CAP_HWACCEL_VDPAU,
     .flush          = flush_dpb,
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index 7739754948..153d83c695 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -678,7 +678,6 @@ void ff_h264_hl_decode_mb(H264Context *h);
 int ff_h264_frame_start(H264Context *h);
 int ff_h264_decode_extradata(H264Context *h, const uint8_t *buf, int size);
 av_cold int ff_h264_decode_init(AVCodecContext *avctx);
-av_cold int ff_h264_decode_end(AVCodecContext *avctx);
 av_cold void ff_h264_decode_init_vlc(void);
 
 /**
diff --git a/libavcodec/lzw.c b/libavcodec/lzw.c
index aed1a43fac..348d2de06b 100644
--- a/libavcodec/lzw.c
+++ b/libavcodec/lzw.c
@@ -102,7 +102,7 @@ void ff_lzw_decode_tail(LZWState *p)
 
     if(s->mode == FF_LZW_GIF) {
         while (s->bs > 0) {
-            if (s->pbuf + s->bs >= s->ebuf) {
+            if (s->bs >= s->ebuf - s->pbuf) {
                 s->pbuf = s->ebuf;
                 break;
             } else {
diff --git a/libavcodec/truemotion2.c b/libavcodec/truemotion2.c
index 38c01c100d..c4f66e8f6d 100644
--- a/libavcodec/truemotion2.c
+++ b/libavcodec/truemotion2.c
@@ -60,7 +60,9 @@ typedef struct TM2Context{
     int *clast;
 
     /* data for current and previous frame */
+    int *Y1_base, *U1_base, *V1_base, *Y2_base, *U2_base, *V2_base;
     int *Y1, *U1, *V1, *Y2, *U2, *V2;
+    int y_stride, uv_stride;
     int cur;
 } TM2Context;
 
@@ -131,7 +133,7 @@ static int tm2_build_huff_table(TM2Context *ctx, TM2Codes *code)
 
     /* check for correct codes parameters */
     if((huff.val_bits < 1) || (huff.val_bits > 32) ||
-       (huff.max_bits < 0) || (huff.max_bits > 32)) {
+       (huff.max_bits < 0) || (huff.max_bits > 25)) {
         av_log(ctx->avctx, AV_LOG_ERROR, "Incorrect tree parameters - literal length: %i, max code length: %i\n",
                huff.val_bits, huff.max_bits);
         return -1;
@@ -328,10 +330,21 @@ static int tm2_read_stream(TM2Context *ctx, const uint8_t *buf, int stream_id, i
                 return -1;
             }
             ctx->tokens[stream_id][i] = tm2_get_token(&ctx->gb, &codes);
+            if (stream_id <= TM2_MOT && ctx->tokens[stream_id][i] >= TM2_DELTAS) {
+                av_log(ctx->avctx, AV_LOG_ERROR, "Invalid delta token index %d for type %d, n=%d\n",
+                       ctx->tokens[stream_id][i], stream_id, i);
+                return AVERROR_INVALIDDATA;
+            }
         }
     } else {
-        for(i = 0; i < toks; i++)
+        for(i = 0; i < toks; i++) {
             ctx->tokens[stream_id][i] = codes.recode[0];
+            if (stream_id <= TM2_MOT && ctx->tokens[stream_id][i] >= TM2_DELTAS) {
+                av_log(ctx->avctx, AV_LOG_ERROR, "Invalid delta token index %d for type %d, n=%d\n",
+                       ctx->tokens[stream_id][i], stream_id, i);
+                return AVERROR_INVALIDDATA;
+            }
+        }
     }
     tm2_free_codes(&codes);
 
@@ -361,9 +374,9 @@ static inline int GET_TOK(TM2Context *ctx,int type) {
     int *Y, *U, *V;\
     int Ystride, Ustride, Vstride;\
 \
-    Ystride = ctx->avctx->width;\
-    Vstride = (ctx->avctx->width + 1) >> 1;\
-    Ustride = (ctx->avctx->width + 1) >> 1;\
+    Ystride = ctx->y_stride;\
+    Vstride = ctx->uv_stride;\
+    Ustride = ctx->uv_stride;\
     Y = (ctx->cur?ctx->Y2:ctx->Y1) + by * 4 * Ystride + bx * 4;\
     V = (ctx->cur?ctx->V2:ctx->V1) + by * 2 * Vstride + bx * 2;\
     U = (ctx->cur?ctx->U2:ctx->U1) + by * 2 * Ustride + bx * 2;\
@@ -651,6 +664,8 @@ static inline void tm2_motion_block(TM2Context *ctx, AVFrame *pic, int bx, int b
 
     mx = GET_TOK(ctx, TM2_MOT);
     my = GET_TOK(ctx, TM2_MOT);
+    mx = av_clip(mx, -(bx * 4 + 4), ctx->avctx->width  - bx * 4);
+    my = av_clip(my, -(by * 4 + 4), ctx->avctx->height - by * 4);
 
     if (4*bx+mx<0 || 4*by+my<0 || 4*bx+mx+4 > ctx->avctx->width || 4*by+my+4 > ctx->avctx->height) {
         av_log(0,0, "MV out of picture\n");
@@ -696,15 +711,12 @@ static inline void tm2_motion_block(TM2Context *ctx, AVFrame *pic, int bx, int b
 static int tm2_decode_blocks(TM2Context *ctx, AVFrame *p)
 {
     int i, j;
-    int bw, bh;
+    int w = ctx->avctx->width, h = ctx->avctx->height, bw = w >> 2, bh = h >> 2, cw = w >> 1;
     int type;
     int keyframe = 1;
     int *Y, *U, *V;
     uint8_t *dst;
 
-    bw = ctx->avctx->width >> 2;
-    bh = ctx->avctx->height >> 2;
-
     for(i = 0; i < TM2_NUM_STREAMS; i++)
         ctx->tok_ptrs[i] = 0;
 
@@ -757,17 +769,54 @@ static int tm2_decode_blocks(TM2Context *ctx, AVFrame *p)
     U = (ctx->cur?ctx->U2:ctx->U1);
     V = (ctx->cur?ctx->V2:ctx->V1);
     dst = p->data[0];
-    for(j = 0; j < ctx->avctx->height; j++){
-        for(i = 0; i < ctx->avctx->width; i++){
+    for(j = 0; j < h; j++){
+        for(i = 0; i < w; i++){
             int y = Y[i], u = U[i >> 1], v = V[i >> 1];
             dst[3*i+0] = av_clip_uint8(y + v);
             dst[3*i+1] = av_clip_uint8(y);
             dst[3*i+2] = av_clip_uint8(y + u);
         }
-        Y += ctx->avctx->width;
+
+        /* horizontal edge extension */
+        Y[-4]    = Y[-3]    = Y[-2]    = Y[-1] = Y[0];
+        Y[w + 3] = Y[w + 2] = Y[w + 1] = Y[w]  = Y[w - 1];
+
+        /* vertical edge extension */
+        if (j == 0) {
+            memcpy(Y - 4 - 1 * ctx->y_stride, Y - 4, ctx->y_stride);
+            memcpy(Y - 4 - 2 * ctx->y_stride, Y - 4, ctx->y_stride);
+            memcpy(Y - 4 - 3 * ctx->y_stride, Y - 4, ctx->y_stride);
+            memcpy(Y - 4 - 4 * ctx->y_stride, Y - 4, ctx->y_stride);
+        } else if (j == h - 1) {
+            memcpy(Y - 4 + 1 * ctx->y_stride, Y - 4, ctx->y_stride);
+            memcpy(Y - 4 + 2 * ctx->y_stride, Y - 4, ctx->y_stride);
+            memcpy(Y - 4 + 3 * ctx->y_stride, Y - 4, ctx->y_stride);
+            memcpy(Y - 4 + 4 * ctx->y_stride, Y - 4, ctx->y_stride);
+        }
+
+        Y += ctx->y_stride;
         if (j & 1) {
-            U += ctx->avctx->width >> 1;
-            V += ctx->avctx->width >> 1;
+            /* horizontal edge extension */
+            U[-2]     = U[-1] = U[0];
+            V[-2]     = V[-1] = V[0];
+            U[cw + 1] = U[cw] = U[cw - 1];
+            V[cw + 1] = V[cw] = V[cw - 1];
+
+            /* vertical edge extension */
+            if (j == 1) {
+                memcpy(U - 2 - 1 * ctx->uv_stride, U - 2, ctx->uv_stride);
+                memcpy(V - 2 - 1 * ctx->uv_stride, V - 2, ctx->uv_stride);
+                memcpy(U - 2 - 2 * ctx->uv_stride, U - 2, ctx->uv_stride);
+                memcpy(V - 2 - 2 * ctx->uv_stride, V - 2, ctx->uv_stride);
+            } else if (j == h - 1) {
+                memcpy(U - 2 + 1 * ctx->uv_stride, U - 2, ctx->uv_stride);
+                memcpy(V - 2 + 1 * ctx->uv_stride, V - 2, ctx->uv_stride);
+                memcpy(U - 2 + 2 * ctx->uv_stride, U - 2, ctx->uv_stride);
+                memcpy(V - 2 + 2 * ctx->uv_stride, V - 2, ctx->uv_stride);
+            }
+
+            U += ctx->uv_stride;
+            V += ctx->uv_stride;
         }
         dst += p->linesize[0];
     }
@@ -813,9 +862,10 @@ static int decode_frame(AVCodecContext *avctx,
             av_log(avctx, AV_LOG_ERROR, "no space for tm2_read_stream\n");
             return AVERROR_INVALIDDATA;
         }
+
         t = tm2_read_stream(l, l->buffer + skip, tm2_stream_order[i], buf_size - skip);
-        if(t == -1){
-            return -1;
+        if(t < 0){
+            return t;
         }
         skip += t;
     }
@@ -834,7 +884,7 @@ static int decode_frame(AVCodecContext *avctx,
 
 static av_cold int decode_init(AVCodecContext *avctx){
     TM2Context * const l = avctx->priv_data;
-    int i;
+    int i, w = avctx->width, h = avctx->height;
 
     if((avctx->width & 3) || (avctx->height & 3)){
         av_log(avctx, AV_LOG_ERROR, "Width and height must be multiple of 4\n");
@@ -848,21 +898,46 @@ static av_cold int decode_init(AVCodecContext *avctx){
 
     ff_dsputil_init(&l->dsp, avctx);
 
-    l->last = av_malloc(4 * sizeof(int) * (avctx->width >> 2));
-    l->clast = av_malloc(4 * sizeof(int) * (avctx->width >> 2));
+    l->last  = av_malloc(4 * sizeof(*l->last)  * (w >> 2));
+    l->clast = av_malloc(4 * sizeof(*l->clast) * (w >> 2));
 
     for(i = 0; i < TM2_NUM_STREAMS; i++) {
         l->tokens[i] = NULL;
         l->tok_lens[i] = 0;
     }
 
-    l->Y1 = av_malloc(sizeof(int) * avctx->width * avctx->height);
-    l->U1 = av_malloc(sizeof(int) * ((avctx->width + 1) >> 1) * ((avctx->height + 1) >> 1));
-    l->V1 = av_malloc(sizeof(int) * ((avctx->width + 1) >> 1) * ((avctx->height + 1) >> 1));
-    l->Y2 = av_malloc(sizeof(int) * avctx->width * avctx->height);
-    l->U2 = av_malloc(sizeof(int) * ((avctx->width + 1) >> 1) * ((avctx->height + 1) >> 1));
-    l->V2 = av_malloc(sizeof(int) * ((avctx->width + 1) >> 1) * ((avctx->height + 1) >> 1));
+    w += 8;
+    h += 8;
+    l->Y1_base = av_malloc(sizeof(*l->Y1_base) * w * h);
+    l->Y2_base = av_malloc(sizeof(*l->Y2_base) * w * h);
+    l->y_stride = w;
+    w = (w + 1) >> 1;
+    h = (h + 1) >> 1;
+    l->U1_base = av_malloc(sizeof(*l->U1_base) * w * h);
+    l->V1_base = av_malloc(sizeof(*l->V1_base) * w * h);
+    l->U2_base = av_malloc(sizeof(*l->U2_base) * w * h);
+    l->V2_base = av_malloc(sizeof(*l->V1_base) * w * h);
+    l->uv_stride = w;
     l->cur = 0;
+    if (!l->Y1_base || !l->Y2_base || !l->U1_base ||
+        !l->V1_base || !l->U2_base || !l->V2_base ||
+        !l->last    || !l->clast) {
+        av_freep(l->Y1_base);
+        av_freep(l->Y2_base);
+        av_freep(l->U1_base);
+        av_freep(l->U2_base);
+        av_freep(l->V1_base);
+        av_freep(l->V2_base);
+        av_freep(l->last);
+        av_freep(l->clast);
+        return AVERROR(ENOMEM);
+    }
+    l->Y1 = l->Y1_base + l->y_stride  * 4 + 4;
+    l->Y2 = l->Y2_base + l->y_stride  * 4 + 4;
+    l->U1 = l->U1_base + l->uv_stride * 2 + 2;
+    l->U2 = l->U2_base + l->uv_stride * 2 + 2;
+    l->V1 = l->V1_base + l->uv_stride * 2 + 2;
+    l->V2 = l->V2_base + l->uv_stride * 2 + 2;
 
     return 0;
 }
@@ -877,12 +952,12 @@ static av_cold int decode_end(AVCodecContext *avctx){
     for(i = 0; i < TM2_NUM_STREAMS; i++)
         av_free(l->tokens[i]);
     if(l->Y1){
-        av_free(l->Y1);
-        av_free(l->U1);
-        av_free(l->V1);
-        av_free(l->Y2);
-        av_free(l->U2);
-        av_free(l->V2);
+        av_free(l->Y1_base);
+        av_free(l->U1_base);
+        av_free(l->V1_base);
+        av_free(l->Y2_base);
+        av_free(l->U2_base);
+        av_free(l->V2_base);
     }
     av_freep(&l->buffer);
     l->buffer_size = 0;