Merge remote-tracking branch 'qatar/master'

* qatar/master:
  proresdsp: port x86 assembly to cpuflags.
  lavr: x86: improve non-SSE4 version of S16_TO_S32_SX macro
  lavfi: better channel layout negotiation
  alac: check for truncated packets
  alac: reverse lpc coeff order, simplify filter
  lavr: add x86-optimized mixing functions
  x86: add support for fmaddps fma4 instruction with abstraction to avx/sse
  tscc2: fix typo in array index
  build: use COMPILE template for HOSTOBJS
  build: do full flag handling for all compiler-type tools
  eval: fix printing of NaN in eval fate test.
  build: Rename aandct component to more descriptive aandcttables
  mpegaudio: bury inline asm under HAVE_INLINE_ASM.
  x86inc: automatically insert vzeroupper for YMM functions.
  rtmp: Check the buffer length of ping packets
  rtmp: Allow having more unknown data at the end of a chunk size packet without failing
  rtmp: Prevent reading outside of an allocate buffer when receiving server bandwidth packets

Conflicts:
	Makefile
	configure
	libavcodec/x86/proresdsp.asm
	libavutil/eval.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

7 files changed, 56 insertions, 52 deletions

diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 041df92e33..b4138e88d9 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -32,7 +32,7 @@ OBJS = allcodecs.o                                                      \
        utils.o                                                          \
 
 # parts needed for many different codecs
-OBJS-$(CONFIG_AANDCT)                  += aandcttab.o
+OBJS-$(CONFIG_AANDCTTABLES)            += aandcttab.o
 OBJS-$(CONFIG_AC3DSP)                  += ac3dsp.o
 OBJS-$(CONFIG_CRYSTALHD)               += crystalhd.o
 OBJS-$(CONFIG_ENCODERS)                += faandct.o jfdctfst.o jfdctint.o
diff --git a/libavcodec/alac.c b/libavcodec/alac.c
index 4fa328539c..2d98456f8b 100644
--- a/libavcodec/alac.c
+++ b/libavcodec/alac.c
@@ -200,6 +200,7 @@ static void lpc_prediction(int32_t *error_buffer, int32_t *buffer_out,
                            int lpc_order, int lpc_quant)
 {
     int i;
+    int32_t *pred = buffer_out;
 
     /* first sample always copies */
     *buffer_out = *error_buffer;
@@ -223,37 +224,35 @@ static void lpc_prediction(int32_t *error_buffer, int32_t *buffer_out,
     }
 
     /* read warm-up samples */
-    for (i = 0; i < lpc_order; i++) {
-        buffer_out[i + 1] = sign_extend(buffer_out[i] + error_buffer[i + 1],
-                                        bps);
-    }
+    for (i = 1; i <= lpc_order; i++)
+        buffer_out[i] = sign_extend(buffer_out[i - 1] + error_buffer[i], bps);
 
     /* NOTE: 4 and 8 are very common cases that could be optimized. */
 
-    for (i = lpc_order; i < nb_samples - 1; i++) {
+    for (; i < nb_samples; i++) {
         int j;
         int val = 0;
-        int error_val = error_buffer[i + 1];
+        int error_val = error_buffer[i];
         int error_sign;
-        int d = buffer_out[i - lpc_order];
+        int d = *pred++;
 
         /* LPC prediction */
         for (j = 0; j < lpc_order; j++)
-            val += (buffer_out[i - j] - d) * lpc_coefs[j];
+            val += (pred[j] - d) * lpc_coefs[j];
         val = (val + (1 << (lpc_quant - 1))) >> lpc_quant;
         val += d + error_val;
-        buffer_out[i + 1] = sign_extend(val, bps);
+        buffer_out[i] = sign_extend(val, bps);
 
         /* adapt LPC coefficients */
         error_sign = sign_only(error_val);
         if (error_sign) {
-            for (j = lpc_order - 1; j >= 0 && error_val * error_sign > 0; j--) {
+            for (j = 0; j < lpc_order && error_val * error_sign > 0; j++) {
                 int sign;
-                val  = d - buffer_out[i - j];
+                val  = d - pred[j];
                 sign = sign_only(val) * error_sign;
                 lpc_coefs[j] -= sign;
                 val *= sign;
-                error_val -= (val >> lpc_quant) * (lpc_order - j);
+                error_val -= (val >> lpc_quant) * (j + 1);
             }
         }
     }
@@ -356,7 +355,7 @@ static int decode_element(AVCodecContext *avctx, void *data, int ch_index,
             lpc_order[ch]         = get_bits(&alac->gb, 5);
 
             /* read the predictor table */
-            for (i = 0; i < lpc_order[ch]; i++)
+            for (i = lpc_order[ch] - 1; i >= 0; i--)
                 lpc_coefs[ch][i] = get_sbits(&alac->gb, 16);
         }
 
@@ -477,16 +476,19 @@ static int alac_decode_frame(AVCodecContext *avctx, void *data,
     ALACContext *alac = avctx->priv_data;
     enum RawDataBlockType element;
     int channels;
-    int ch, ret;
+    int ch, ret, got_end;
 
     init_get_bits(&alac->gb, avpkt->data, avpkt->size * 8);
 
+    got_end = 0;
     alac->nb_samples = 0;
     ch = 0;
-    while (get_bits_left(&alac->gb)) {
+    while (get_bits_left(&alac->gb) >= 3) {
         element = get_bits(&alac->gb, 3);
-        if (element == TYPE_END)
+        if (element == TYPE_END) {
+            got_end = 1;
             break;
+        }
         if (element > TYPE_CPE && element != TYPE_LFE) {
             av_log(avctx, AV_LOG_ERROR, "syntax element unsupported: %d", element);
             return AVERROR_PATCHWELCOME;
@@ -501,11 +503,15 @@ static int alac_decode_frame(AVCodecContext *avctx, void *data,
         ret = decode_element(avctx, data,
                              alac_channel_layout_offsets[alac->channels - 1][ch],
                              channels);
-        if (ret < 0)
+        if (ret < 0 && get_bits_left(&alac->gb))
             return ret;
 
         ch += channels;
     }
+    if (!got_end) {
+        av_log(avctx, AV_LOG_ERROR, "no end tag found. incomplete packet.\n");
+        return AVERROR_INVALIDDATA;
+    }
 
     if (avpkt->size * 8 - get_bits_count(&alac->gb) > 8) {
         av_log(avctx, AV_LOG_ERROR, "Error : %d bits left\n",
diff --git a/libavcodec/tscc2.c b/libavcodec/tscc2.c
index a8fd652e59..5e2fd02745 100644
--- a/libavcodec/tscc2.c
+++ b/libavcodec/tscc2.c
@@ -298,8 +298,8 @@ static int tscc2_decode_frame(AVCodecContext *avctx, void *data,
         if (!size) {
             int skip_row = 1, j, off = i * c->mb_width;
             for (j = 0; j < c->mb_width; j++) {
-                if (c->slice_quants[off + i] == 1 ||
-                    c->slice_quants[off + i] == 2) {
+                if (c->slice_quants[off + j] == 1 ||
+                    c->slice_quants[off + j] == 2) {
                     skip_row = 0;
                     break;
                 }
diff --git a/libavcodec/x86/dsputil_yasm.asm b/libavcodec/x86/dsputil_yasm.asm
index d2e5439e61..5a6c3d1eae 100644
--- a/libavcodec/x86/dsputil_yasm.asm
+++ b/libavcodec/x86/dsputil_yasm.asm
@@ -1158,12 +1158,7 @@ ALIGN 16
     add     src1q, 2*mmsize
     sub     lenq,  2*mmsize
     jge     .loop
-%if mmsize == 32
-    vzeroupper
-    RET
-%else
     REP_RET
-%endif
 %endmacro
 
 INIT_XMM sse
@@ -1193,12 +1188,7 @@ ALIGN 16
 
     sub     lenq,   2*mmsize
     jge     .loop
-%if mmsize == 32
-    vzeroupper
-    RET
-%else
     REP_RET
-%endif
 %endmacro
 
 INIT_XMM sse
@@ -1243,10 +1233,6 @@ cglobal butterflies_float_interleave, 4,4,3, dst, src0, src1, len
 %endif
     add       lenq, mmsize
     jl .loop
-%if mmsize == 32
-    vzeroupper
-    RET
-%endif
 .end:
     REP_RET
 %endmacro
diff --git a/libavcodec/x86/fft_mmx.asm b/libavcodec/x86/fft_mmx.asm
index 5bf0e2f259..5e5004ca97 100644
--- a/libavcodec/x86/fft_mmx.asm
+++ b/libavcodec/x86/fft_mmx.asm
@@ -750,9 +750,6 @@ section .text
 ; The others pass args in registers and don't spill anything.
 cglobal fft_dispatch%2, 2,5,8, z, nbits
     FFT_DISPATCH fullsuffix, nbits
-%if mmsize == 32
-    vzeroupper
-%endif
     RET
 %endmacro ; DECL_FFT
 
@@ -958,9 +955,6 @@ cglobal imdct_half, 3,12,8; FFTContext *s, FFTSample *output, const FFTSample *i
 %if ARCH_X86_64 == 0
     add esp, 12
 %endif
-%if mmsize == 32
-    vzeroupper
-%endif
     RET
 %endmacro
 
diff --git a/libavcodec/x86/mpegaudiodec_mmx.c b/libavcodec/x86/mpegaudiodec_mmx.c
index 939b441277..0d6cc08305 100644
--- a/libavcodec/x86/mpegaudiodec_mmx.c
+++ b/libavcodec/x86/mpegaudiodec_mmx.c
@@ -36,6 +36,8 @@ void ff_four_imdct36_float_avx(float *out, float *buf, float *in, float *win,
 
 DECLARE_ALIGNED(16, static float, mdct_win_sse)[2][4][4*40];
 
+#if HAVE_INLINE_ASM
+
 #define MACS(rt, ra, rb) rt+=(ra)*(rb)
 #define MLSS(rt, ra, rb) rt-=(ra)*(rb)
 
@@ -178,6 +180,7 @@ static void apply_window_mp3(float *in, float *win, int *unused, float *out,
     *out = sum;
 }
 
+#endif /* HAVE_INLINE_ASM */
 
 #define DECL_IMDCT_BLOCKS(CPU1, CPU2)                                       \
 static void imdct36_blocks_ ## CPU1(float *out, float *buf, float *in,      \
@@ -241,9 +244,11 @@ void ff_mpadsp_init_mmx(MPADSPContext *s)
         }
     }
 
+#if HAVE_INLINE_ASM
     if (mm_flags & AV_CPU_FLAG_SSE2) {
         s->apply_window_float = apply_window_mp3;
     }
+#endif /* HAVE_INLINE_ASM */
 #if HAVE_YASM
     if (0) {
 #if HAVE_AVX
diff --git a/libavcodec/x86/proresdsp.asm b/libavcodec/x86/proresdsp.asm
index a09d871fda..01e5deec93 100644
--- a/libavcodec/x86/proresdsp.asm
+++ b/libavcodec/x86/proresdsp.asm
@@ -83,8 +83,7 @@ section .text align=16
 
 ; %1 = row or col (for rounding variable)
 ; %2 = number of bits to shift at the end
-; %3 = optimization
-%macro IDCT_1D 3
+%macro IDCT_1D 2
     ; a0 = (W4 * row[0]) + (1 << (15 - 1));
     ; a1 = a0;
     ; a2 = a0;
@@ -235,8 +234,8 @@ section .text align=16
 
 ; void prores_idct_put_10_<opt>(uint8_t *pixels, int stride,
 ;                               DCTELEM *block, const int16_t *qmat);
-%macro idct_put_fn 2
-cglobal prores_idct_put_10_%1, 4, 4, %2
+%macro idct_put_fn 1
+cglobal prores_idct_put_10, 4, 4, %1
     movsxd      r1,  r1d
     pxor        m15, m15           ; zero
 
@@ -252,7 +251,7 @@ cglobal prores_idct_put_10_%1, 4, 4, %2
     pmullw      m13,[r3+64]
     pmullw      m12,[r3+96]
 
-    IDCT_1D     row, 15,  %1
+    IDCT_1D     row, 15
 
     ; transpose for second part of IDCT
     TRANSPOSE8x8W 8, 0, 1, 2, 4, 11, 9, 10, 3
@@ -267,7 +266,7 @@ cglobal prores_idct_put_10_%1, 4, 4, %2
 
     ; for (i = 0; i < 8; i++)
     ;     idctSparseColAdd(dest + i, line_size, block + i);
-    IDCT_1D     col, 18,  %1
+    IDCT_1D     col, 18
 
     ; clip/store
     mova        m3, [pw_4]
@@ -302,13 +301,27 @@ cglobal prores_idct_put_10_%1, 4, 4, %2
     RET
 %endmacro
 
-INIT_XMM
-idct_put_fn sse2, 16
-INIT_XMM
-idct_put_fn sse4, 16
+%macro SIGNEXTEND 2-3 ; dstlow, dsthigh, tmp
+%if cpuflag(sse4)
+    movhlps     %2,  %1
+    pmovsxwd    %1,  %1
+    pmovsxwd    %2,  %2
+%else ; sse2
+    pxor        %3,  %3
+    pcmpgtw     %3,  %1
+    mova        %2,  %1
+    punpcklwd   %1,  %3
+    punpckhwd   %2,  %3
+%endif
+%endmacro
+
+INIT_XMM sse2
+idct_put_fn 16
+INIT_XMM sse4
+idct_put_fn 16
 %if HAVE_AVX
-INIT_AVX
-idct_put_fn avx,  16
+INIT_XMM avx
+idct_put_fn 16
 %endif
 
 %endif