libswresample/swresamplec: Err num(negative-size) was used as a function parameter

If memory allocation fails, ERROR(ENOMEM) '-12' will be returned.
When resample() is done first, the negative size param would cause buffer-overflow and SEGV in swri_rematrix().
When swri_rematrix() is run first, resample() would not cause an error but Err num as a wrong parameter passing.
Err num should be returned immediately. And remove an unneeded term from an assert.

coredump info:
    #0 0x499517 in posix_memalign (/home/r1/ffmpeg/ffmpeg_4.4.1+0x499517)
    #1 0x6c1f0b4 in av_malloc /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libavutil/mem.c:86:9
    #2 0x6c208fe in av_mallocz /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libavutil/mem.c:239:17
    #3 0x6c207ad in av_mallocz_array /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libavutil/mem.c:195:12
    #4 0x654b2e5 in swri_realloc_audio /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libswresample/swresample.c:418:14
    #5 0x654f9a1 in swr_convert_internal /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libswresample/swresample.c:601:17
    #6 0x654d2c0 in swr_convert /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libswresample/swresample.c:766:19
    #7 0x186cf56 in flush_frame /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libavfilter/af_aresample.c:251:13
    #8 0x186a454 in request_frame /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libavfilter/af_aresample.c:288:20
    #9 0x787d9c in ff_request_frame_to_filter /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libavfilter/avfilter.c:459:15
    #10 0x7877f1 in forward_status_change /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libavfilter/avfilter.c:1257:19
    #11 0x77ed7e in ff_filter_activate_default /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libavfilter/avfilter.c:1288:20
    #12 0x77e4e1 in ff_filter_activate /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libavfilter/avfilter.c:1441:11
    #13 0x793b3f in ff_filter_graph_run_once /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libavfilter/avfiltergraph.c:1403:12
    #14 0x7a7bee in get_frame_internal /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libavfilter/buffersink.c:131:19
    #15 0x7a7287 in av_buffersink_get_frame_flags /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libavfilter/buffersink.c:142:12
    #16 0x792888 in avfilter_graph_request_oldest /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/libavfilter/avfiltergraph.c:1356:17
    #17 0x5d07df in transcode_from_filter /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/fftools/ffmpeg.c:4639:11
    #18 0x59e557 in transcode_step /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/fftools/ffmpeg.c:4729:20
    #19 0x593970 in transcode /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/fftools/ffmpeg.c:4805:15
    #20 0x58f7a4 in main /home/r1/ffmpeg/ffmpeg-4.4.1/build/src/fftools/ffmpeg.c:5010:9
    #21 0x7f6fd2dee0b2 in __libc_start_main /build/glibc-eX1tMB/glibc-2.31/csu/../csu/libc-start.c:308:16

SUMMARY: AddressSanitizer: negative-size-param (/home/r1/ffmpeg/ffmpeg_4.4.1+0x497e67) in __asan_memcpy

Reported-by: TOTE Robot <oslab@tsinghua.edu.cn>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

1 files changed, 5 insertions, 3 deletions

diff --git a/libswresample/swresample.c b/libswresample/swresample.c
index c03fe5528f..16734c9df9 100644
--- a/libswresample/swresample.c
+++ b/libswresample/swresample.c
@@ -643,14 +643,16 @@ static int swr_convert_internal(struct SwrContext *s, AudioData *out, int out_co
 
     if(s->resample_first){
         if(postin != midbuf)
-            out_count= resample(s, midbuf, out_count, postin, in_count);
+            if ((out_count = resample(s, midbuf, out_count, postin, in_count)) < 0)
+                return out_count;
         if(midbuf != preout)
             swri_rematrix(s, preout, midbuf, out_count, preout==out);
     }else{
         if(postin != midbuf)
             swri_rematrix(s, midbuf, postin, in_count, midbuf==out);
         if(midbuf != preout)
-            out_count= resample(s, preout, out_count, midbuf, in_count);
+            if ((out_count = resample(s, preout, out_count, midbuf, in_count)) < 0)
+                return out_count;
     }
 
     if(preout != out && out_count){
@@ -769,7 +771,7 @@ int attribute_align_arg swr_convert(struct SwrContext *s,
         if(ret>0 && !s->drop_output)
             s->outpts += ret * (int64_t)s->in_sample_rate;
 
-        av_assert2(max_output < 0 || ret < 0 || ret <= max_output);
+        av_assert2(max_output < 0 || ret <= max_output);
 
         return ret;
     }else{