avcodec/mp_cmp: reject invalid comparison function values

Fixes tickets #10306 and #10318.

Signed-off-by: James Almer <jamrial@gmail.com>
(cherry picked from commit 7c6e26a18403376987541f1ca801ae225f8ee6d4)

6 files changed, 26 insertions, 12 deletions

diff --git a/libavcodec/dvenc.c b/libavcodec/dvenc.c
index 11dd5763af..cd442b524d 100644
--- a/libavcodec/dvenc.c
+++ b/libavcodec/dvenc.c
@@ -104,7 +104,9 @@ static av_cold int dvvideo_encode_init(AVCodecContext *avctx)
     ff_fdctdsp_init(&fdsp, avctx);
     ff_me_cmp_init(&mecc, avctx);
     ff_pixblockdsp_init(&pdsp, avctx);
-    ff_set_cmp(&mecc, mecc.ildct_cmp, avctx->ildct_cmp);
+    ret = ff_set_cmp(&mecc, mecc.ildct_cmp, avctx->ildct_cmp);
+    if (ret < 0)
+        return AVERROR(EINVAL);
 
     s->get_pixels = pdsp.get_pixels;
     s->ildct_cmp  = mecc.ildct_cmp[5];
diff --git a/libavcodec/me_cmp.c b/libavcodec/me_cmp.c
index e2f9f84b05..cd05e63ffd 100644
--- a/libavcodec/me_cmp.c
+++ b/libavcodec/me_cmp.c
@@ -473,8 +473,9 @@ static int zero_cmp(MpegEncContext *s, const uint8_t *a, const uint8_t *b,
     return 0;
 }
 
-void ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type)
+int ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type)
 {
+    int ret = 0;
     int i;
 
     memset(cmp, 0, sizeof(void *) * 6);
@@ -533,9 +534,13 @@ void ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type)
 #endif
         default:
             av_log(NULL, AV_LOG_ERROR,
-                   "internal error in cmp function selection\n");
+                   "invalid cmp function selection\n");
+            ret = -1;
+            break;
         }
     }
+
+    return ret;
 }
 
 #define BUTTERFLY2(o1, o2, i1, i2)              \
diff --git a/libavcodec/me_cmp.h b/libavcodec/me_cmp.h
index 90ea76c891..aefd32a7dc 100644
--- a/libavcodec/me_cmp.h
+++ b/libavcodec/me_cmp.h
@@ -89,7 +89,7 @@ void ff_me_cmp_init_ppc(MECmpContext *c, AVCodecContext *avctx);
 void ff_me_cmp_init_x86(MECmpContext *c, AVCodecContext *avctx);
 void ff_me_cmp_init_mips(MECmpContext *c, AVCodecContext *avctx);
 
-void ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type);
+int ff_set_cmp(MECmpContext *c, me_cmp_func *cmp, int type);
 
 void ff_dsputil_init_dwt(MECmpContext *c);
 
diff --git a/libavcodec/motion_est.c b/libavcodec/motion_est.c
index d17ffe42b4..df9d1befa8 100644
--- a/libavcodec/motion_est.c
+++ b/libavcodec/motion_est.c
@@ -309,6 +309,7 @@ int ff_init_me(MpegEncContext *s){
     MotionEstContext * const c= &s->me;
     int cache_size= FFMIN(ME_MAP_SIZE>>ME_MAP_SHIFT, 1<<ME_MAP_SHIFT);
     int dia_size= FFMAX(FFABS(s->avctx->dia_size)&255, FFABS(s->avctx->pre_dia_size)&255);
+    int ret;
 
     if(FFMIN(s->avctx->dia_size, s->avctx->pre_dia_size) < -FFMIN(ME_MAP_SIZE, MAX_SAB_SIZE)){
         av_log(s->avctx, AV_LOG_ERROR, "ME_MAP size is too small for SAB diamond\n");
@@ -324,10 +325,12 @@ int ff_init_me(MpegEncContext *s){
         av_log(s->avctx, AV_LOG_INFO, "ME_MAP size may be a little small for the selected diamond size\n");
     }
 
-    ff_set_cmp(&s->mecc, s->mecc.me_pre_cmp, c->avctx->me_pre_cmp);
-    ff_set_cmp(&s->mecc, s->mecc.me_cmp,     c->avctx->me_cmp);
-    ff_set_cmp(&s->mecc, s->mecc.me_sub_cmp, c->avctx->me_sub_cmp);
-    ff_set_cmp(&s->mecc, s->mecc.mb_cmp,     c->avctx->mb_cmp);
+    ret  = ff_set_cmp(&s->mecc, s->mecc.me_pre_cmp, c->avctx->me_pre_cmp);
+    ret |= ff_set_cmp(&s->mecc, s->mecc.me_cmp,     c->avctx->me_cmp);
+    ret |= ff_set_cmp(&s->mecc, s->mecc.me_sub_cmp, c->avctx->me_sub_cmp);
+    ret |= ff_set_cmp(&s->mecc, s->mecc.mb_cmp,     c->avctx->mb_cmp);
+    if (ret < 0)
+        return ret;
 
     c->flags    = get_flags(c, 0, c->avctx->me_cmp    &FF_CMP_CHROMA);
     c->sub_flags= get_flags(c, 0, c->avctx->me_sub_cmp&FF_CMP_CHROMA);
diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c
index 7d3c8875f2..bcd2f9ef1d 100644
--- a/libavcodec/mpegvideo_enc.c
+++ b/libavcodec/mpegvideo_enc.c
@@ -902,8 +902,10 @@ av_cold int ff_mpv_encode_init(AVCodecContext *avctx)
 
     s->quant_precision = 5;
 
-    ff_set_cmp(&s->mecc, s->mecc.ildct_cmp,      avctx->ildct_cmp);
-    ff_set_cmp(&s->mecc, s->mecc.frame_skip_cmp, s->frame_skip_cmp);
+    ret  = ff_set_cmp(&s->mecc, s->mecc.ildct_cmp,      avctx->ildct_cmp);
+    ret |= ff_set_cmp(&s->mecc, s->mecc.frame_skip_cmp, s->frame_skip_cmp);
+    if (ret < 0)
+        return AVERROR(EINVAL);
 
     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263) {
         ff_h263_encode_init(s);
diff --git a/libavcodec/snowenc.c b/libavcodec/snowenc.c
index 658684c575..f61c2c9b13 100644
--- a/libavcodec/snowenc.c
+++ b/libavcodec/snowenc.c
@@ -129,8 +129,10 @@ static av_cold int encode_init(AVCodecContext *avctx)
     if (ret)
         return ret;
 
-    ff_set_cmp(&s->mecc, s->mecc.me_cmp, s->avctx->me_cmp);
-    ff_set_cmp(&s->mecc, s->mecc.me_sub_cmp, s->avctx->me_sub_cmp);
+    ret  = ff_set_cmp(&s->mecc, s->mecc.me_cmp, s->avctx->me_cmp);
+    ret |= ff_set_cmp(&s->mecc, s->mecc.me_sub_cmp, s->avctx->me_sub_cmp);
+    if (ret < 0)
+        return AVERROR(EINVAL);
 
     s->input_picture = av_frame_alloc();
     if (!s->input_picture)