vp8: change mv_{min,max}.{x,y} type to int

If one of the dimensions is larger than 8176, s->mb_width or
s->mb_height is larger than 511, leading to an int16_t overflow of
s->mv_max.{x,y}. This then causes av_clip to be called with amin > amax.

Changing the type to int avoids the overflow and has no negative
effect, because s->mv_max is only used in clamp_mv for clipping.
Since mv_max.{x,y} is positive and mv_min.{x,y} negative, av_clip can't
increase the absolute value. The input to av_clip is an int16_t, and
thus the output fits into int16_t as well.

For additional safety, s->mv_{min,max}.{x,y} are clipped to int16_t range
before use.

Reviewed-by: Ronald S. Bultje <rsbultje@gmail.com>
Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun@googlemail.com>

1 files changed, 7 insertions, 2 deletions

diff --git a/libavcodec/vp8.h b/libavcodec/vp8.h
index b650892735..2135bd9d83 100644
--- a/libavcodec/vp8.h
+++ b/libavcodec/vp8.h
@@ -134,6 +134,11 @@ typedef struct VP8Frame {
     AVBufferRef *seg_map;
 } VP8Frame;
 
+typedef struct VP8intmv {
+    int x;
+    int y;
+} VP8intmv;
+
 #define MAX_THREADS 8
 typedef struct VP8Context {
     VP8ThreadData *thread_data;
@@ -152,8 +157,8 @@ typedef struct VP8Context {
     uint8_t deblock_filter;
     uint8_t mbskip_enabled;
     uint8_t profile;
-    VP56mv mv_min;
-    VP56mv mv_max;
+    VP8intmv mv_min;
+    VP8intmv mv_max;
 
     int8_t sign_bias[4]; ///< one state [0, 1] per ref frame type
     int ref_count[3];