Use an int MotionVector for find_block_motion.

Using the double variant causes several pointless conversions between
double and int.
Worse, one of the conversions is in an inner loop together with a
function using MMX, resulting in undefined behaviour.
Based on debugging by Ray Simard.

Signed-off-by: Reimar Döffinger <Reimar.Doeffinger@gmx.de>
Tested-by: Ray Simard <rhs.ffmpeg@sylvan-glade.com>

1 files changed, 9 insertions, 4 deletions

diff --git a/libavfilter/vf_deshake.c b/libavfilter/vf_deshake.c
index 7013f6ccc1..c2d8f9208e 100644
--- a/libavfilter/vf_deshake.c
+++ b/libavfilter/vf_deshake.c
@@ -67,6 +67,11 @@ enum SearchMethod {
 };
 
 typedef struct {
+    int x;             ///< Horizontal shift
+    int y;             ///< Vertical shift
+} IntMotionVector;
+
+typedef struct {
     double x;             ///< Horizontal shift
     double y;             ///< Vertical shift
 } MotionVector;
@@ -129,7 +134,7 @@ static double clean_mean(double *values, int count)
  */
 static void find_block_motion(DeshakeContext *deshake, uint8_t *src1,
                               uint8_t *src2, int cx, int cy, int stride,
-                              MotionVector *mv)
+                              IntMotionVector *mv)
 {
     int x, y;
     int diff;
@@ -222,7 +227,7 @@ static int block_contrast(uint8_t *src, int x, int y, int stride, int blocksize)
 /**
  * Find the rotation for a given block.
  */
-static double block_angle(int x, int y, int cx, int cy, MotionVector *shift)
+static double block_angle(int x, int y, int cx, int cy, IntMotionVector *shift)
 {
     double a1, a2, diff;
 
@@ -247,7 +252,7 @@ static void find_motion(DeshakeContext *deshake, uint8_t *src1, uint8_t *src2,
                         int width, int height, int stride, Transform *t)
 {
     int x, y;
-    MotionVector mv = {0, 0};
+    IntMotionVector mv = {0, 0};
     int counts[128][128];
     int count_max_value = 0;
     int contrast;
@@ -278,7 +283,7 @@ static void find_motion(DeshakeContext *deshake, uint8_t *src1, uint8_t *src2,
                 //av_log(NULL, AV_LOG_ERROR, "%d\n", contrast);
                 find_block_motion(deshake, src1, src2, x, y, stride, &mv);
                 if (mv.x != -1 && mv.y != -1) {
-                    counts[(int)(mv.x + deshake->rx)][(int)(mv.y + deshake->ry)] += 1;
+                    counts[mv.x + deshake->rx][mv.y + deshake->ry] += 1;
                     if (x > deshake->rx && y > deshake->ry)
                         angles[pos++] = block_angle(x, y, 0, 0, &mv);