Convert multiplier for MV from int to ptrdiff_t.

This prevents emulated_edge_mc from not undoing mvy*stride-related
integer overflows.

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

17 files changed, 32 insertions, 31 deletions

diff --git a/libavcodec/cavs.h b/libavcodec/cavs.h
index b0cdb8f200..f3c05dc9f0 100644
--- a/libavcodec/cavs.h
+++ b/libavcodec/cavs.h
@@ -210,7 +210,7 @@ typedef struct AVSContext {
        6:    A3  X2  X3   */
     int pred_mode_Y[3*3];
     int *top_pred_Y;
-    int l_stride, c_stride;
+    ptrdiff_t l_stride, c_stride;
     int luma_scan[4];
     int qp;
     int qp_fixed;
diff --git a/libavcodec/diracdec.c b/libavcodec/diracdec.c
index 7468280637..e5239a77f8 100644
--- a/libavcodec/diracdec.c
+++ b/libavcodec/diracdec.c
@@ -112,7 +112,7 @@ typedef struct SubBand {
 typedef struct Plane {
     int width;
     int height;
-    int stride;
+    ptrdiff_t stride;
 
     int idwt_width;
     int idwt_height;
diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 51484edce1..acaab39f08 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -911,7 +911,7 @@ static av_always_inline void mc_dir_part(H264Context *h, Picture *pic,
     const int mx      = h->mv_cache[list][scan8[n]][0] + src_x_offset * 8;
     int my            = h->mv_cache[list][scan8[n]][1] + src_y_offset * 8;
     const int luma_xy = (mx & 3) + ((my & 3) << 2);
-    int offset        = ((mx >> 2) << pixel_shift) + (my >> 2) * h->mb_linesize;
+    ptrdiff_t offset  = ((mx >> 2) << pixel_shift) + (my >> 2) * h->mb_linesize;
     uint8_t *src_y    = pic->f.data[0] + offset;
     uint8_t *src_cb, *src_cr;
     int extra_width  = 0;
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index b2365bce83..f1db519e47 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -306,7 +306,7 @@ typedef struct H264Context {
 
     /* coded dimensions -- 16 * mb w/h */
     int width, height;
-    int linesize, uvlinesize;
+    ptrdiff_t linesize, uvlinesize;
     int chroma_x_shift, chroma_y_shift;
 
     int qscale;
@@ -379,8 +379,8 @@ typedef struct H264Context {
     uint32_t *mb2br_xy;
     int b_stride;       // FIXME use s->b4_stride
 
-    int mb_linesize;    ///< may be equal to s->linesize or s->linesize * 2, for mbaff
-    int mb_uvlinesize;
+    ptrdiff_t mb_linesize;    ///< may be equal to s->linesize or s->linesize * 2, for mbaff
+    ptrdiff_t mb_uvlinesize;
 
     unsigned current_sps_id; ///< id of the current SPS
     SPS sps; ///< current sps
diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index 6cfd2a402f..96afca5d7c 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -2179,7 +2179,7 @@ static inline int hpel_motion_lowres(MpegEncContext *s,
                                      uint8_t *dest, uint8_t *src,
                                      int field_based, int field_select,
                                      int src_x, int src_y,
-                                     int width, int height, int stride,
+                                     int width, int height, ptrdiff_t stride,
                                      int h_edge_pos, int v_edge_pos,
                                      int w, int h, h264_chroma_mc_func *pix_op,
                                      int motion_x, int motion_y)
@@ -2235,8 +2235,8 @@ static av_always_inline void mpeg_motion_lowres(MpegEncContext *s,
                                                 int h, int mb_y)
 {
     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
-    int mx, my, src_x, src_y, uvsrc_x, uvsrc_y, uvlinesize, linesize, sx, sy,
-        uvsx, uvsy;
+    int mx, my, src_x, src_y, uvsrc_x, uvsrc_y, sx, sy, uvsx, uvsy;
+    ptrdiff_t uvlinesize, linesize;
     const int lowres     = s->avctx->lowres;
     const int op_index   = FFMIN(lowres-1+s->chroma_x_shift, 3);
     const int block_s    = 8>>lowres;
@@ -2367,7 +2367,8 @@ static inline void chroma_4mv_motion_lowres(MpegEncContext *s,
     const int s_mask     = (2 << lowres) - 1;
     const int h_edge_pos = s->h_edge_pos >> lowres + 1;
     const int v_edge_pos = s->v_edge_pos >> lowres + 1;
-    int emu = 0, src_x, src_y, offset, sx, sy;
+    int emu = 0, src_x, src_y, sx, sy;
+    ptrdiff_t offset;
     uint8_t *ptr;
 
     if (s->quarter_sample) {
diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h
index d17426cd4b..2152af2e10 100644
--- a/libavcodec/mpegvideo.h
+++ b/libavcodec/mpegvideo.h
@@ -284,8 +284,8 @@ typedef struct MpegEncContext {
     int b4_stride;             ///< 4*mb_width+1 used for some 4x4 block arrays to allow simple addressing
     int h_edge_pos, v_edge_pos;///< horizontal / vertical position of the right/bottom edge (pixel replication)
     int mb_num;                ///< number of MBs of a picture
-    int linesize;              ///< line size, in bytes, may be different from width
-    int uvlinesize;            ///< line size, for chroma in bytes, may be different from width
+    ptrdiff_t linesize;        ///< line size, in bytes, may be different from width
+    ptrdiff_t uvlinesize;      ///< line size, for chroma in bytes, may be different from width
     Picture *picture;          ///< main picture buffer
     Picture **input_picture;   ///< next pictures on display order for encoding
     Picture **reordered_input_picture; ///< pointer to the next pictures in codedorder for encoding
diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c
index a85cf563de..6f0d96168a 100644
--- a/libavcodec/mpegvideo_enc.c
+++ b/libavcodec/mpegvideo_enc.c
@@ -985,7 +985,7 @@ static int load_input_picture(MpegEncContext *s, const AVFrame *pic_arg)
             direct = 0;
 
         av_dlog(s->avctx, "%d %d %d %d\n", pic_arg->linesize[0],
-                pic_arg->linesize[1], s->linesize, s->uvlinesize);
+                pic_arg->linesize[1], (int) s->linesize, (int) s->uvlinesize);
 
         if (direct) {
             i = ff_find_unused_picture(s, 1);
@@ -1771,7 +1771,7 @@ static av_always_inline void encode_mb_internal(MpegEncContext *s,
     int dct_offset = s->linesize * 8; // default for progressive frames
     int uv_dct_offset = s->uvlinesize * 8;
     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
-    int wrap_y, wrap_c;
+    ptrdiff_t wrap_y, wrap_c;
 
     for (i = 0; i < mb_block_count; i++)
         skip_dct[i] = s->skipdct;
diff --git a/libavcodec/mpegvideo_motion.c b/libavcodec/mpegvideo_motion.c
index 3851739512..e5b7e4648a 100644
--- a/libavcodec/mpegvideo_motion.c
+++ b/libavcodec/mpegvideo_motion.c
@@ -38,7 +38,8 @@ static void gmc1_motion(MpegEncContext *s,
                         uint8_t **ref_picture)
 {
     uint8_t *ptr;
-    int offset, src_x, src_y, linesize, uvlinesize;
+    int src_x, src_y;
+    ptrdiff_t offset, linesize, uvlinesize;
     int motion_x, motion_y;
     int emu=0;
 
@@ -215,7 +216,7 @@ void mpeg_motion_internal(MpegEncContext *s,
     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
     int dxy, uvdxy, mx, my, src_x, src_y,
         uvsrc_x, uvsrc_y, v_edge_pos;
-    emuedge_linesize_type uvlinesize, linesize;
+    ptrdiff_t uvlinesize, linesize;
 
 #if 0
 if(s->quarter_sample)
@@ -459,7 +460,8 @@ static inline void qpel_motion(MpegEncContext *s,
                                int motion_x, int motion_y, int h)
 {
     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
-    int dxy, uvdxy, mx, my, src_x, src_y, uvsrc_x, uvsrc_y, v_edge_pos, linesize, uvlinesize;
+    int dxy, uvdxy, mx, my, src_x, src_y, uvsrc_x, uvsrc_y, v_edge_pos;
+    ptrdiff_t linesize, uvlinesize;
 
     dxy = ((motion_y & 3) << 2) | (motion_x & 3);
     src_x = s->mb_x *  16                 + (motion_x >> 2);
@@ -552,7 +554,8 @@ static void chroma_4mv_motion(MpegEncContext *s,
                               op_pixels_func *pix_op,
                               int mx, int my)
 {
-    int dxy, emu=0, src_x, src_y, offset;
+    int dxy, emu=0, src_x, src_y;
+    ptrdiff_t offset;
     uint8_t *ptr;
 
     /* In case of 8X8, we construct a single chroma motion vector
diff --git a/libavcodec/snow.c b/libavcodec/snow.c
index 1125b25367..dc5c78e84a 100644
--- a/libavcodec/snow.c
+++ b/libavcodec/snow.c
@@ -297,7 +297,7 @@ static void mc_block(Plane *p, uint8_t *dst, const uint8_t *src, int stride, int
     }
 }
 
-void ff_snow_pred_block(SnowContext *s, uint8_t *dst, uint8_t *tmp, int stride, int sx, int sy, int b_w, int b_h, BlockNode *block, int plane_index, int w, int h){
+void ff_snow_pred_block(SnowContext *s, uint8_t *dst, uint8_t *tmp, ptrdiff_t stride, int sx, int sy, int b_w, int b_h, BlockNode *block, int plane_index, int w, int h){
     if(block->type & BLOCK_INTRA){
         int x, y;
         const unsigned color  = block->color[plane_index];
diff --git a/libavcodec/snow.h b/libavcodec/snow.h
index c3e50adf5b..1222a776de 100644
--- a/libavcodec/snow.h
+++ b/libavcodec/snow.h
@@ -227,7 +227,7 @@ void ff_snow_release_buffer(AVCodecContext *avctx);
 void ff_snow_reset_contexts(SnowContext *s);
 int ff_snow_alloc_blocks(SnowContext *s);
 int ff_snow_frame_start(SnowContext *s);
-void ff_snow_pred_block(SnowContext *s, uint8_t *dst, uint8_t *tmp, int stride,
+void ff_snow_pred_block(SnowContext *s, uint8_t *dst, uint8_t *tmp, ptrdiff_t stride,
                      int sx, int sy, int b_w, int b_h, BlockNode *block,
                      int plane_index, int w, int h);
 /* common inline functions */
diff --git a/libavcodec/videodsp.h b/libavcodec/videodsp.h
index 07519d71ba..e397720773 100644
--- a/libavcodec/videodsp.h
+++ b/libavcodec/videodsp.h
@@ -29,8 +29,6 @@
 #include <stddef.h>
 #include <stdint.h>
 
-typedef int emuedge_linesize_type;
-
 #define EMULATED_EDGE(depth) \
 void ff_emulated_edge_mc_ ## depth (uint8_t *buf, const uint8_t *src, ptrdiff_t linesize,\
                          int block_w, int block_h,\
diff --git a/libavcodec/videodsp_template.c b/libavcodec/videodsp_template.c
index 1f709c499d..39aad4deec 100644
--- a/libavcodec/videodsp_template.c
+++ b/libavcodec/videodsp_template.c
@@ -21,13 +21,12 @@
 
 #include "bit_depth_template.c"
 void FUNC(ff_emulated_edge_mc)(uint8_t *buf, const uint8_t *src,
-                                      ptrdiff_t linesize_arg,
+                                      ptrdiff_t linesize,
                                       int block_w, int block_h,
                                       int src_x, int src_y, int w, int h)
 {
     int x, y;
     int start_y, start_x, end_y, end_x;
-    emuedge_linesize_type linesize = linesize_arg;
 
     if (!w || !h)
         return;
diff --git a/libavcodec/vp3.c b/libavcodec/vp3.c
index a6e2d0eea7..78c094c038 100644
--- a/libavcodec/vp3.c
+++ b/libavcodec/vp3.c
@@ -1476,7 +1476,7 @@ static void render_slice(Vp3DecodeContext *s, int slice)
         uint8_t *output_plane = s->current_frame.f->data    [plane] + s->data_offset[plane];
         uint8_t *  last_plane = s->   last_frame.f->data    [plane] + s->data_offset[plane];
         uint8_t *golden_plane = s-> golden_frame.f->data    [plane] + s->data_offset[plane];
-        int stride            = s->current_frame.f->linesize[plane];
+        ptrdiff_t stride      = s->current_frame.f->linesize[plane];
         int plane_width       = s->width  >> (plane && s->chroma_x_shift);
         int plane_height      = s->height >> (plane && s->chroma_y_shift);
         int8_t (*motion_val)[2] = s->motion_val[!!plane];
diff --git a/libavcodec/vp56.c b/libavcodec/vp56.c
index d67eaa6b0d..b5d1884aa5 100644
--- a/libavcodec/vp56.c
+++ b/libavcodec/vp56.c
@@ -311,7 +311,7 @@ static void vp56_deblock_filter(VP56Context *s, uint8_t *yuv,
 }
 
 static void vp56_mc(VP56Context *s, int b, int plane, uint8_t *src,
-                    int stride, int x, int y)
+                    ptrdiff_t stride, int x, int y)
 {
     uint8_t *dst = s->frames[VP56_FRAME_CURRENT]->data[plane] + s->block_offset[b];
     uint8_t *src_block;
diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c
index d7fac64411..48432c17c2 100644
--- a/libavcodec/vp8.c
+++ b/libavcodec/vp8.c
@@ -1180,7 +1180,7 @@ static av_always_inline
 void vp8_mc_luma(VP8Context *s, VP8ThreadData *td, uint8_t *dst,
                  ThreadFrame *ref, const VP56mv *mv,
                  int x_off, int y_off, int block_w, int block_h,
-                 int width, int height, int linesize,
+                 int width, int height, ptrdiff_t linesize,
                  vp8_mc_func mc_func[3][3])
 {
     uint8_t *src = ref->f->data[0];
@@ -1230,7 +1230,7 @@ void vp8_mc_luma(VP8Context *s, VP8ThreadData *td, uint8_t *dst,
 static av_always_inline
 void vp8_mc_chroma(VP8Context *s, VP8ThreadData *td, uint8_t *dst1, uint8_t *dst2,
                    ThreadFrame *ref, const VP56mv *mv, int x_off, int y_off,
-                   int block_w, int block_h, int width, int height, int linesize,
+                   int block_w, int block_h, int width, int height, ptrdiff_t linesize,
                    vp8_mc_func mc_func[3][3])
 {
     uint8_t *src1 = ref->f->data[1], *src2 = ref->f->data[2];
diff --git a/libavcodec/wmv2.c b/libavcodec/wmv2.c
index 918d438ec2..eea819bfdb 100644
--- a/libavcodec/wmv2.c
+++ b/libavcodec/wmv2.c
@@ -94,7 +94,8 @@ void ff_mspel_motion(MpegEncContext *s,
 {
     Wmv2Context * const w= (Wmv2Context*)s;
     uint8_t *ptr;
-    int dxy, offset, mx, my, src_x, src_y, v_edge_pos, linesize, uvlinesize;
+    int dxy, mx, my, src_x, src_y, v_edge_pos;
+    ptrdiff_t offset, linesize, uvlinesize;
     int emu=0;
 
     dxy = ((motion_y & 1) << 1) | (motion_x & 1);
diff --git a/libavcodec/x86/videodsp_init.c b/libavcodec/x86/videodsp_init.c
index 95c71a280f..17a90115cf 100644
--- a/libavcodec/x86/videodsp_init.c
+++ b/libavcodec/x86/videodsp_init.c
@@ -39,14 +39,13 @@ extern emu_edge_core_func ff_emu_edge_core_mmx;
 extern emu_edge_core_func ff_emu_edge_core_sse;
 
 static av_always_inline void emulated_edge_mc(uint8_t *buf, const uint8_t *src,
-                                              ptrdiff_t linesize_arg,
+                                              ptrdiff_t linesize,
                                               int block_w, int block_h,
                                               int src_x, int src_y,
                                               int w, int h,
                                               emu_edge_core_func *core_fn)
 {
     int start_y, start_x, end_y, end_x, src_y_add = 0;
-    emuedge_linesize_type linesize = linesize_arg;
 
     if(!w || !h)
         return;