diff options
author | Loren Merritt <lorenm@u.washington.edu> | 2006-08-10 19:06:25 +0000 |
---|---|---|
committer | Loren Merritt <lorenm@u.washington.edu> | 2006-08-10 19:06:25 +0000 |
commit | eb4825b5d43bb6ecfae4d64688f9e2d2ac075263 (patch) | |
tree | 13acab1d16846859f7636ea19f796d02e2ea6da4 | |
parent | 60ae06e77e6e654b19529e89f8f067abd9a3d71a (diff) | |
download | ffmpeg-eb4825b5d43bb6ecfae4d64688f9e2d2ac075263.tar.gz |
sse and 3dnow implementations of float->int conversion and mdct windowing.
15% faster vorbis.
Originally committed as revision 5975 to svn://svn.ffmpeg.org/ffmpeg/trunk
-rw-r--r-- | libavcodec/dsputil.c | 37 | ||||
-rw-r--r-- | libavcodec/dsputil.h | 14 | ||||
-rw-r--r-- | libavcodec/i386/dsputil_mmx.c | 213 | ||||
-rw-r--r-- | libavcodec/vorbis.c | 99 | ||||
-rw-r--r-- | libavcodec/vorbis.h | 2 |
5 files changed, 317 insertions, 48 deletions
diff --git a/libavcodec/dsputil.c b/libavcodec/dsputil.c index 937dceb2c4..bba233b1c6 100644 --- a/libavcodec/dsputil.c +++ b/libavcodec/dsputil.c @@ -3753,6 +3753,39 @@ WARPER8_16_SQ(quant_psnr8x8_c, quant_psnr16_c) WARPER8_16_SQ(rd8x8_c, rd16_c) WARPER8_16_SQ(bit8x8_c, bit16_c) +static void vector_fmul_c(float *dst, const float *src, int len){ + int i; + for(i=0; i<len; i++) + dst[i] *= src[i]; +} + +static void vector_fmul_reverse_c(float *dst, const float *src0, const float *src1, int len){ + int i; + src1 += len-1; + for(i=0; i<len; i++) + dst[i] = src0[i] * src1[-i]; +} + +void ff_vector_fmul_add_add_c(float *dst, const float *src0, const float *src1, const float *src2, int src3, int len, int step){ + int i; + for(i=0; i<len; i++) + dst[i*step] = src0[i] * src1[i] + src2[i] + src3; +} + +void ff_float_to_int16_c(int16_t *dst, const float *src, int len){ + int i; + for(i=0; i<len; i++) { + int_fast32_t tmp = ((int32_t*)src)[i]; + if(tmp & 0xf0000){ + tmp = (0x43c0ffff - tmp)>>31; + // is this faster on some gcc/cpu combinations? +// if(tmp > 0x43c0ffff) tmp = 0xFFFF; +// else tmp = 0; + } + dst[i] = tmp - 0x8000; + } +} + /* XXX: those functions should be suppressed ASAP when all IDCTs are converted */ static void ff_jref_idct_put(uint8_t *dest, int line_size, DCTELEM *block) @@ -4096,6 +4129,10 @@ void dsputil_init(DSPContext* c, AVCodecContext *avctx) #ifdef CONFIG_VORBIS_DECODER c->vorbis_inverse_coupling = vorbis_inverse_coupling; #endif + c->vector_fmul = vector_fmul_c; + c->vector_fmul_reverse = vector_fmul_reverse_c; + c->vector_fmul_add_add = ff_vector_fmul_add_add_c; + c->float_to_int16 = ff_float_to_int16_c; c->shrink[0]= ff_img_copy_plane; c->shrink[1]= ff_shrink22; diff --git a/libavcodec/dsputil.h b/libavcodec/dsputil.h index 1083c39c71..5c31a87dcf 100644 --- a/libavcodec/dsputil.h +++ b/libavcodec/dsputil.h @@ -58,6 +58,10 @@ void ff_h264_idct_dc_add_c(uint8_t *dst, DCTELEM *block, int stride); void ff_h264_lowres_idct_add_c(uint8_t *dst, int stride, DCTELEM *block); void ff_h264_lowres_idct_put_c(uint8_t *dst, int stride, DCTELEM *block); +void ff_vector_fmul_add_add_c(float *dst, const float *src0, const float *src1, + const float *src2, int src3, int blocksize, int step); +void ff_float_to_int16_c(int16_t *dst, const float *src, int len); + /* encoding scans */ extern const uint8_t ff_alternate_horizontal_scan[64]; extern const uint8_t ff_alternate_vertical_scan[64]; @@ -307,7 +311,17 @@ typedef struct DSPContext { void (*h261_loop_filter)(uint8_t *src, int stride); + /* assume len is a multiple of 4, and arrays are 16-byte aligned */ void (*vorbis_inverse_coupling)(float *mag, float *ang, int blocksize); + void (*vector_fmul)(float *dst, const float *src, int len); + /* assume len is a multiple of 8, and arrays are 16-byte aligned */ + void (*vector_fmul_reverse)(float *dst, const float *src0, const float *src1, int len); + /* assume len is a multiple of 8, and src arrays are 16-byte aligned */ + void (*vector_fmul_add_add)(float *dst, const float *src0, const float *src1, const float *src2, int src3, int len, int step); + + /* C version: convert floats from the range [384.0,386.0] to ints in [-32768,32767] + * asm versions: convert floats from [-32768.0,32767.0] without rescaling */ + void (*float_to_int16)(int16_t *dst, const float *src, int len); /* (I)DCT */ void (*fdct)(DCTELEM *block/* align 16*/); diff --git a/libavcodec/i386/dsputil_mmx.c b/libavcodec/i386/dsputil_mmx.c index 6c71640cbf..9686b9ed0f 100644 --- a/libavcodec/i386/dsputil_mmx.c +++ b/libavcodec/i386/dsputil_mmx.c @@ -2772,6 +2772,198 @@ static void vorbis_inverse_coupling_sse(float *mag, float *ang, int blocksize) } } +static void vector_fmul_3dnow(float *dst, const float *src, int len){ + long i; + len >>= 1; + for(i=0; i<len; i++) { + asm volatile( + "movq %0, %%mm0 \n\t" + "pfmul %1, %%mm0 \n\t" + "movq %%mm0, %0 \n\t" + :"+m"(dst[i*2]) + :"m"(src[i*2]) + :"memory" + ); + } + asm volatile("femms"); +} +static void vector_fmul_sse(float *dst, const float *src, int len){ + long i; + len >>= 2; + for(i=0; i<len; i++) { + asm volatile( + "movaps %0, %%xmm0 \n\t" + "mulps %1, %%xmm0 \n\t" + "movaps %%xmm0, %0 \n\t" + :"+m"(dst[i*4]) + :"m"(src[i*4]) + :"memory" + ); + } +} + +static void vector_fmul_reverse_3dnow2(float *dst, const float *src0, const float *src1, int len){ + long i = len*4-16; + asm volatile( + "1: \n\t" + "pswapd 8(%1), %%mm0 \n\t" + "pswapd (%1), %%mm1 \n\t" + "pfmul (%3,%0), %%mm0 \n\t" + "pfmul 8(%3,%0), %%mm1 \n\t" + "movq %%mm0, (%2,%0) \n\t" + "movq %%mm1, 8(%2,%0) \n\t" + "add $16, %1 \n\t" + "sub $16, %0 \n\t" + "jge 1b \n\t" + :"+r"(i), "+r"(src1) + :"r"(dst), "r"(src0) + ); + asm volatile("femms"); +} +static void vector_fmul_reverse_sse2(float *dst, const float *src0, const float *src1, int len){ + long i = len*4-32; + asm volatile( + "1: \n\t" + "pshufd $0x1b, 16(%1), %%xmm0 \n\t" + "pshufd $0x1b, (%1), %%xmm1 \n\t" + "mulps (%3,%0), %%xmm0 \n\t" + "mulps 16(%3,%0), %%xmm1 \n\t" + "movaps %%xmm0, (%2,%0) \n\t" + "movaps %%xmm1, 16(%2,%0) \n\t" + "add $32, %1 \n\t" + "sub $32, %0 \n\t" + "jge 1b \n\t" + :"+r"(i), "+r"(src1) + :"r"(dst), "r"(src0) + ); +} + +static void vector_fmul_add_add_3dnow(float *dst, const float *src0, const float *src1, + const float *src2, int src3, int len, int step){ + long i; + if(step == 2 && src3 == 0){ + i = (len-4)*4; + dst += (len-4)*2; + asm volatile( + "1: \n\t" + "movq (%2,%0), %%mm0 \n\t" + "movq 8(%2,%0), %%mm1 \n\t" + "pfmul (%3,%0), %%mm0 \n\t" + "pfmul 8(%3,%0), %%mm1 \n\t" + "pfadd (%4,%0), %%mm0 \n\t" + "pfadd 8(%4,%0), %%mm1 \n\t" + "movd %%mm0, (%1) \n\t" + "movd %%mm1, 16(%1) \n\t" + "psrlq $32, %%mm0 \n\t" + "psrlq $32, %%mm1 \n\t" + "movd %%mm0, 8(%1) \n\t" + "movd %%mm1, 24(%1) \n\t" + "sub $32, %1 \n\t" + "sub $16, %0 \n\t" + "jge 1b \n\t" + :"+r"(i), "+r"(dst) + :"r"(src0), "r"(src1), "r"(src2) + :"memory" + ); + } + else if(step == 1 && src3 == 0){ + for(i=0; i<len; i+=2){ + asm volatile( + "movq %1, %%mm0 \n\t" + "pfmul %2, %%mm0 \n\t" + "pfadd %3, %%mm0 \n\t" + "movq %%mm0, %0 \n\t" + :"=m"(dst[i]) + :"m"(src0[i]), "m"(src1[i]), "m"(src2[i]) + ); + } + } + else + ff_vector_fmul_add_add_c(dst, src0, src1, src2, src3, len, step); + asm volatile("femms"); +} +static void vector_fmul_add_add_sse2(float *dst, const float *src0, const float *src1, + const float *src2, float src3, int len, int step){ + long i; + if(step == 2 && src3 == 0){ + i = (len-8)*4; + dst += (len-8)*2; + asm volatile( + "1: \n\t" + "movaps (%2,%0), %%xmm0 \n\t" + "movaps 16(%2,%0), %%xmm1 \n\t" + "mulps (%3,%0), %%xmm0 \n\t" + "mulps 16(%3,%0), %%xmm1 \n\t" + "addps (%4,%0), %%xmm0 \n\t" + "addps 16(%4,%0), %%xmm1 \n\t" + "movd %%xmm0, (%1) \n\t" + "movd %%xmm1, 32(%1) \n\t" + "psrldq $4, %%xmm0 \n\t" + "psrldq $4, %%xmm1 \n\t" + "movd %%xmm0, 8(%1) \n\t" + "movd %%xmm1, 40(%1) \n\t" + "psrldq $4, %%xmm0 \n\t" + "psrldq $4, %%xmm1 \n\t" + "movd %%xmm0, 16(%1) \n\t" + "movd %%xmm1, 48(%1) \n\t" + "psrldq $4, %%xmm0 \n\t" + "psrldq $4, %%xmm1 \n\t" + "movd %%xmm0, 24(%1) \n\t" + "movd %%xmm1, 56(%1) \n\t" + "sub $64, %1 \n\t" + "sub $32, %0 \n\t" + "jge 1b \n\t" + :"+r"(i), "+r"(dst) + :"r"(src0), "r"(src1), "r"(src2) + :"memory" + ); + } + else if(step == 1 && src3 == 0){ + for(i=0; i<len; i+=4){ + asm volatile( + "movaps %1, %%xmm0 \n\t" + "mulps %2, %%xmm0 \n\t" + "addps %3, %%xmm0 \n\t" + "movaps %%xmm0, %0 \n\t" + :"=m"(dst[i]) + :"m"(src0[i]), "m"(src1[i]), "m"(src2[i]) + ); + } + } + else + ff_vector_fmul_add_add_c(dst, src0, src1, src2, src3, len, step); +} + +void float_to_int16_3dnow(int16_t *dst, const float *src, int len){ + // not bit-exact: pf2id uses different rounding than C and SSE + int i; + for(i=0; i<len; i+=4) { + asm volatile( + "pf2id %1, %%mm0 \n\t" + "pf2id %2, %%mm1 \n\t" + "packssdw %%mm1, %%mm0 \n\t" + "movq %%mm0, %0 \n\t" + :"=m"(dst[i]) + :"m"(src[i]), "m"(src[i+2]) + ); + } + asm volatile("femms"); +} +void float_to_int16_sse(int16_t *dst, const float *src, int len){ + int i; + for(i=0; i<len; i+=4) { + asm volatile( + "cvtps2pi %1, %%mm0 \n\t" + "cvtps2pi %2, %%mm1 \n\t" + "packssdw %%mm1, %%mm0 \n\t" + "movq %%mm0, %0 \n\t" + :"=m"(dst[i]) + :"m"(src[i]), "m"(src[i+2]) + ); + } + asm volatile("emms"); +} + #ifdef CONFIG_SNOW_ENCODER extern void ff_snow_horizontal_compose97i_sse2(DWTELEM *b, int width); extern void ff_snow_horizontal_compose97i_mmx(DWTELEM *b, int width); @@ -3199,10 +3391,25 @@ void dsputil_init_mmx(DSPContext* c, AVCodecContext *avctx) } #endif - if(mm_flags & MM_SSE) - c->vorbis_inverse_coupling = vorbis_inverse_coupling_sse; - else if(mm_flags & MM_3DNOW) + if(mm_flags & MM_3DNOW){ c->vorbis_inverse_coupling = vorbis_inverse_coupling_3dnow; + c->vector_fmul = vector_fmul_3dnow; + if(!(avctx->flags & CODEC_FLAG_BITEXACT)) + c->float_to_int16 = float_to_int16_3dnow; + } + if(mm_flags & MM_3DNOWEXT) + c->vector_fmul_reverse = vector_fmul_reverse_3dnow2; + if(mm_flags & MM_SSE){ + c->vorbis_inverse_coupling = vorbis_inverse_coupling_sse; + c->vector_fmul = vector_fmul_sse; + c->float_to_int16 = float_to_int16_sse; + } + if(mm_flags & MM_SSE2){ + c->vector_fmul_reverse = vector_fmul_reverse_sse2; + c->vector_fmul_add_add = vector_fmul_add_add_sse2; + } + if(mm_flags & MM_3DNOW) + c->vector_fmul_add_add = vector_fmul_add_add_3dnow; // faster than sse2 } #ifdef CONFIG_ENCODERS diff --git a/libavcodec/vorbis.c b/libavcodec/vorbis.c index bf2cb358bf..97214a6306 100644 --- a/libavcodec/vorbis.c +++ b/libavcodec/vorbis.c @@ -192,6 +192,11 @@ static void vorbis_free(vorbis_context *vc) { av_free(vc->mappings[i].mux); } av_freep(&vc->mappings); + + if(vc->exp_bias){ + av_freep(&vc->swin); + av_freep(&vc->lwin); + } } // Parse setup header ------------------------------------------------- @@ -888,6 +893,19 @@ static int vorbis_parse_id_hdr(vorbis_context *vc){ vc->swin=vwin[bl0-6]; vc->lwin=vwin[bl1-6]; + if(vc->exp_bias){ + int i; + float *win; + win = av_malloc(vc->blocksize_0/2 * sizeof(float)); + for(i=0; i<vc->blocksize_0/2; i++) + win[i] = vc->swin[i] * (1<<15); + vc->swin = win; + win = av_malloc(vc->blocksize_1/2 * sizeof(float)); + for(i=0; i<vc->blocksize_1/2; i++) + win[i] = vc->lwin[i] * (1<<15); + vc->lwin = win; + } + if ((get_bits1(gb)) == 0) { av_log(vc->avccontext, AV_LOG_ERROR, " Vorbis id header packet corrupt (framing flag not set). \n"); return 2; @@ -931,6 +949,14 @@ static int vorbis_decode_init(AVCodecContext *avccontext) { vc->avccontext = avccontext; dsputil_init(&vc->dsp, avccontext); + if(vc->dsp.float_to_int16 == ff_float_to_int16_c) { + vc->add_bias = 385; + vc->exp_bias = 0; + } else { + vc->add_bias = 0; + vc->exp_bias = 15<<23; + } + if (!headers_len) { av_log(avccontext, AV_LOG_ERROR, "Extradata corrupt.\n"); return -1; @@ -1472,7 +1498,6 @@ void vorbis_inverse_coupling(float *mag, float *ang, int blocksize) } // Decode the audio packet using the functions above -#define BIAS 385 static int vorbis_parse_audio_packet(vorbis_context *vc) { GetBitContext *gb=&vc->gb; @@ -1490,6 +1515,7 @@ static int vorbis_parse_audio_packet(vorbis_context *vc) { uint_fast8_t res_num=0; int_fast16_t retlen=0; uint_fast16_t saved_start=0; + float fadd_bias = vc->add_bias; if (get_bits1(gb)) { av_log(vc->avccontext, AV_LOG_ERROR, "Not a Vorbis I audio packet.\n"); @@ -1576,10 +1602,7 @@ static int vorbis_parse_audio_packet(vorbis_context *vc) { for(j=0, ch_floor_ptr=vc->channel_floors;j<vc->audio_channels;++j,ch_floor_ptr+=blocksize/2) { ch_res_ptr=vc->channel_residues+res_chan[j]*blocksize/2; - - for(i=0;i<blocksize/2;++i) { - ch_floor_ptr[i]*=ch_res_ptr[i]; //FPMATH - } + vc->dsp.vector_fmul(ch_floor_ptr, ch_res_ptr, blocksize/2); } // MDCT, overlap/add, save data for next overlapping FPMATH @@ -1600,59 +1623,54 @@ static int vorbis_parse_audio_packet(vorbis_context *vc) { vc->mdct0.fft.imdct_calc(vc->modes[mode_number].blockflag ? &vc->mdct1 : &vc->mdct0, buf, ch_floor_ptr, buf_tmp); + //FIXME process channels together, to allow faster simd vector_fmul_add_add? if (vc->modes[mode_number].blockflag) { // -- overlap/add if (previous_window) { - for(k=j, i=0;i<vc->blocksize_1/2;++i, k+=step) { - ret[k]=saved[i]+buf[i]*lwin[i]+BIAS; - } + vc->dsp.vector_fmul_add_add(ret+j, buf, lwin, saved, vc->add_bias, vc->blocksize_1/2, step); retlen=vc->blocksize_1/2; } else { - buf += (vc->blocksize_1-vc->blocksize_0)/4; - for(k=j, i=0;i<vc->blocksize_0/2;++i, k+=step) { - ret[k]=saved[i]+buf[i]*swin[i]+BIAS; - } + int len = (vc->blocksize_1-vc->blocksize_0)/4; + buf += len; + vc->dsp.vector_fmul_add_add(ret+j, buf, swin, saved, vc->add_bias, vc->blocksize_0/2, step); + k = vc->blocksize_0/2*step + j; buf += vc->blocksize_0/2; - for(i=0;i<(vc->blocksize_1-vc->blocksize_0)/4;++i, k+=step) { - ret[k]=buf[i]+BIAS; + if(vc->exp_bias){ + for(i=0; i<len; i++, k+=step) + ((uint32_t*)ret)[k] = ((uint32_t*)buf)[i] + vc->exp_bias; // ret[k]=buf[i]*(1<<bias) + } else { + for(i=0; i<len; i++, k+=step) + ret[k] = buf[i] + fadd_bias; } buf=vc->buf; - retlen=vc->blocksize_0/2+(vc->blocksize_1-vc->blocksize_0)/4; + retlen=vc->blocksize_0/2+len; } // -- save if (next_window) { buf += vc->blocksize_1/2; - lwin += vc->blocksize_1/2-1; - for(i=0;i<vc->blocksize_1/2;++i) { - saved[i]=buf[i]*lwin[-i]; - } + vc->dsp.vector_fmul_reverse(saved, buf, lwin, vc->blocksize_1/2); saved_start=0; } else { saved_start=(vc->blocksize_1-vc->blocksize_0)/4; buf += vc->blocksize_1/2; - for(i=0;i<saved_start;++i) { - saved[i]=buf[i]; - } - swin += vc->blocksize_0/2-1; - for(i=0;i<vc->blocksize_0/2;++i) { - saved[saved_start+i]=buf[saved_start+i]*swin[-i]; - } + for(i=0; i<saved_start; i++) + ((uint32_t*)saved)[i] = ((uint32_t*)buf)[i] + vc->exp_bias; + vc->dsp.vector_fmul_reverse(saved+saved_start, buf+saved_start, swin, vc->blocksize_0/2); } } else { // --overlap/add - for(k=j, i=0;i<saved_start;++i, k+=step) { - ret[k]=saved[i]+BIAS; - } - for(i=0;i<vc->blocksize_0/2;++i, k+=step) { - ret[k]=saved[saved_start+i]+buf[i]*swin[i]+BIAS; + if(vc->add_bias) { + for(k=j, i=0;i<saved_start;++i, k+=step) + ret[k] = saved[i] + fadd_bias; + } else { + for(k=j, i=0;i<saved_start;++i, k+=step) + ret[k] = saved[i]; } + vc->dsp.vector_fmul_add_add(ret+k, buf, swin, saved+saved_start, vc->add_bias, vc->blocksize_0/2, step); retlen=saved_start+vc->blocksize_0/2; // -- save buf += vc->blocksize_0/2; - swin += vc->blocksize_0/2-1; - for(i=0;i<vc->blocksize_0/2;++i) { - saved[i]=buf[i]*swin[-i]; - } + vc->dsp.vector_fmul_reverse(saved, buf, swin, vc->blocksize_0/2); saved_start=0; } } @@ -1695,16 +1713,7 @@ static int vorbis_decode_frame(AVCodecContext *avccontext, AV_DEBUG("parsed %d bytes %d bits, returned %d samples (*ch*bits) \n", get_bits_count(gb)/8, get_bits_count(gb)%8, len); - for(i=0;i<len;++i) { - int_fast32_t tmp= ((int32_t*)vc->ret)[i]; - if(tmp & 0xf0000){ -// tmp= (0x43c0ffff - tmp)>>31; //ask gcc devs why this is slower - if(tmp > 0x43c0ffff) tmp= 0xFFFF; - else tmp= 0; - } - ((int16_t*)data)[i]=tmp - 0x8000; - } - + vc->dsp.float_to_int16(data, vc->ret, len); *data_size=len*2; return buf_size ; diff --git a/libavcodec/vorbis.h b/libavcodec/vorbis.h index 1274f1891f..de150f9940 100644 --- a/libavcodec/vorbis.h +++ b/libavcodec/vorbis.h @@ -120,6 +120,8 @@ typedef struct vorbis_context_s { float *ret; float *buf; float *buf_tmp; + uint_fast32_t add_bias; // for float->int conversion + uint_fast32_t exp_bias; } vorbis_context; |