Merge commit 'c50241080d7599c90fc8b4e74c5f8d62a4caae52' into release/1.1

* commit 'c50241080d7599c90fc8b4e74c5f8d62a4caae52':
  vf_hqdn3d: fix uninitialized variable use
  lzo: fix overflow checking in copy_backptr()
  flacdec: simplify bounds checking in flac_probe()
  atrac3: avoid oversized shifting in decode_bytes()
  shorten: use the unsigned type where needed
  shorten: report meaningful errors
  shorten: K&R formatting cosmetics
  shorten: set invalid channels count to 0

Conflicts:
	libavcodec/shorten.c
	libavformat/flacdec.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

5 files changed, 137 insertions, 123 deletions

diff --git a/libavcodec/atrac3.c b/libavcodec/atrac3.c
index 84c5c11fca..c1518a733f 100644
--- a/libavcodec/atrac3.c
+++ b/libavcodec/atrac3.c
@@ -165,7 +165,10 @@ static int decode_bytes(const uint8_t *input, uint8_t *out, int bytes)
 
     off = (intptr_t)input & 3;
     buf = (const uint32_t *)(input - off);
-    c   = av_be2ne32((0x537F6103 >> (off * 8)) | (0x537F6103 << (32 - (off * 8))));
+    if (off)
+        c = av_be2ne32((0x537F6103U >> (off * 8)) | (0x537F6103U << (32 - (off * 8))));
+    else
+        c = av_be2ne32(0x537F6103U);
     bytes += 3 + off;
     for (i = 0; i < bytes / 4; i++)
         output[i] = c ^ buf[i];
diff --git a/libavcodec/shorten.c b/libavcodec/shorten.c
index 1d1f7eca07..991141533a 100644
--- a/libavcodec/shorten.c
+++ b/libavcodec/shorten.c
@@ -88,7 +88,7 @@ typedef struct ShortenContext {
     GetBitContext gb;
 
     int min_framesize, max_framesize;
-    int channels;
+    unsigned channels;
 
     int32_t *decoded[MAX_CHANNELS];
     int32_t *decoded_base[MAX_CHANNELS];
@@ -113,10 +113,10 @@ typedef struct ShortenContext {
     int got_quit_command;
 } ShortenContext;
 
-static av_cold int shorten_decode_init(AVCodecContext * avctx)
+static av_cold int shorten_decode_init(AVCodecContext *avctx)
 {
     ShortenContext *s = avctx->priv_data;
-    s->avctx = avctx;
+    s->avctx          = avctx;
 
     avcodec_get_frame_defaults(&s->frame);
     avctx->coded_frame = &s->frame;
@@ -130,17 +130,20 @@ static int allocate_buffers(ShortenContext *s)
     int *coeffs;
     void *tmp_ptr;
 
-    for (chan=0; chan<s->channels; chan++) {
-        if(FFMAX(1, s->nmean) >= UINT_MAX/sizeof(int32_t)){
+    for (chan = 0; chan < s->channels; chan++) {
+        if (FFMAX(1, s->nmean) >= UINT_MAX / sizeof(int32_t)) {
             av_log(s->avctx, AV_LOG_ERROR, "nmean too large\n");
             return AVERROR_INVALIDDATA;
         }
-        if(s->blocksize + s->nwrap >= UINT_MAX/sizeof(int32_t) || s->blocksize + s->nwrap <= (unsigned)s->nwrap){
-            av_log(s->avctx, AV_LOG_ERROR, "s->blocksize + s->nwrap too large\n");
+        if (s->blocksize + s->nwrap >= UINT_MAX / sizeof(int32_t) ||
+            s->blocksize + s->nwrap <= (unsigned)s->nwrap) {
+            av_log(s->avctx, AV_LOG_ERROR,
+                   "s->blocksize + s->nwrap too large\n");
             return AVERROR_INVALIDDATA;
         }
 
-        tmp_ptr = av_realloc(s->offset[chan], sizeof(int32_t)*FFMAX(1, s->nmean));
+        tmp_ptr =
+            av_realloc(s->offset[chan], sizeof(int32_t) * FFMAX(1, s->nmean));
         if (!tmp_ptr)
             return AVERROR(ENOMEM);
         s->offset[chan] = tmp_ptr;
@@ -150,7 +153,7 @@ static int allocate_buffers(ShortenContext *s)
         if (!tmp_ptr)
             return AVERROR(ENOMEM);
         s->decoded_base[chan] = tmp_ptr;
-        for (i=0; i<s->nwrap; i++)
+        for (i = 0; i < s->nwrap; i++)
             s->decoded_base[chan][i] = 0;
         s->decoded[chan] = s->decoded_base[chan] + s->nwrap;
     }
@@ -163,7 +166,6 @@ static int allocate_buffers(ShortenContext *s)
     return 0;
 }
 
-
 static inline unsigned int get_uint(ShortenContext *s, int k)
 {
     if (s->version != 0)
@@ -171,7 +173,6 @@ static inline unsigned int get_uint(ShortenContext *s, int k)
     return get_ur_golomb_shorten(&s->gb, k);
 }
 
-
 static void fix_bitshift(ShortenContext *s, int32_t *buffer)
 {
     int i;
@@ -181,26 +182,24 @@ static void fix_bitshift(ShortenContext *s, int32_t *buffer)
             buffer[i] <<= s->bitshift;
 }
 
-
 static int init_offset(ShortenContext *s)
 {
     int32_t mean = 0;
-    int  chan, i;
+    int chan, i;
     int nblock = FFMAX(1, s->nmean);
     /* initialise offset */
-    switch (s->internal_ftype)
-    {
-        case TYPE_U8:
-            s->avctx->sample_fmt = AV_SAMPLE_FMT_U8P;
-            mean = 0x80;
-            break;
-        case TYPE_S16HL:
-        case TYPE_S16LH:
-            s->avctx->sample_fmt = AV_SAMPLE_FMT_S16P;
-            break;
-        default:
-            av_log(s->avctx, AV_LOG_ERROR, "unknown audio type\n");
-            return AVERROR_PATCHWELCOME;
+    switch (s->internal_ftype) {
+    case TYPE_U8:
+        s->avctx->sample_fmt = AV_SAMPLE_FMT_U8P;
+        mean = 0x80;
+        break;
+    case TYPE_S16HL:
+    case TYPE_S16LH:
+        s->avctx->sample_fmt = AV_SAMPLE_FMT_S16P;
+        break;
+    default:
+        av_log(s->avctx, AV_LOG_ERROR, "unknown audio type\n");
+        return AVERROR_PATCHWELCOME;
     }
 
     for (chan = 0; chan < s->channels; chan++)
@@ -216,21 +215,21 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header,
     short wave_format;
     const uint8_t *end= header + header_size;
 
-    if (bytestream_get_le32(&header) != MKTAG('R','I','F','F')) {
+    if (bytestream_get_le32(&header) != MKTAG('R', 'I', 'F', 'F')) {
         av_log(avctx, AV_LOG_ERROR, "missing RIFF tag\n");
         return AVERROR_INVALIDDATA;
     }
 
-    header += 4; /* chunk size */;
+    header += 4; /* chunk size */
 
-    if (bytestream_get_le32(&header) != MKTAG('W','A','V','E')) {
+    if (bytestream_get_le32(&header) != MKTAG('W', 'A', 'V', 'E')) {
         av_log(avctx, AV_LOG_ERROR, "missing WAVE tag\n");
         return AVERROR_INVALIDDATA;
     }
 
-    while (bytestream_get_le32(&header) != MKTAG('f','m','t',' ')) {
-        len = bytestream_get_le32(&header);
-        if(len<0 || end - header - 8 < len)
+    while (bytestream_get_le32(&header) != MKTAG('f', 'm', 't', ' ')) {
+        len     = bytestream_get_le32(&header);
+        if (len<0 || end - header - 8 < len)
             return AVERROR_INVALIDDATA;
         header += len;
     }
@@ -244,11 +243,11 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header,
     wave_format = bytestream_get_le16(&header);
 
     switch (wave_format) {
-        case WAVE_FORMAT_PCM:
-            break;
-        default:
-            av_log(avctx, AV_LOG_ERROR, "unsupported wave format\n");
-            return AVERROR_PATCHWELCOME;
+    case WAVE_FORMAT_PCM:
+        break;
+    default:
+        av_log(avctx, AV_LOG_ERROR, "unsupported wave format\n");
+        return AVERROR(ENOSYS);
     }
 
     header += 2;        // skip channels    (already got from shorten header)
@@ -260,7 +259,7 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header,
 
     if (bps != 16 && bps != 8) {
         av_log(avctx, AV_LOG_ERROR, "unsupported number of bits per sample: %d\n", bps);
-        return AVERROR_INVALIDDATA;
+        return AVERROR(ENOSYS);
     }
 
     len -= 16;
@@ -286,11 +285,12 @@ static int decode_subframe_lpc(ShortenContext *s, int command, int channel,
         /* read/validate prediction order */
         pred_order = get_ur_golomb_shorten(&s->gb, LPCQSIZE);
         if (pred_order > s->nwrap) {
-            av_log(s->avctx, AV_LOG_ERROR, "invalid pred_order %d\n", pred_order);
+            av_log(s->avctx, AV_LOG_ERROR, "invalid pred_order %d\n",
+                   pred_order);
             return AVERROR(EINVAL);
         }
         /* read LPC coefficients */
-        for (i=0; i<pred_order; i++)
+        for (i = 0; i < pred_order; i++)
             s->coeffs[i] = get_sr_golomb_shorten(&s->gb, LPCQUANT);
         coeffs = s->coeffs;
 
@@ -298,7 +298,7 @@ static int decode_subframe_lpc(ShortenContext *s, int command, int channel,
     } else {
         /* fixed LPC coeffs */
         pred_order = command;
-        coeffs     = fixed_coeffs[pred_order-1];
+        coeffs     = fixed_coeffs[pred_order - 1];
         qshift     = 0;
     }
 
@@ -309,11 +309,12 @@ static int decode_subframe_lpc(ShortenContext *s, int command, int channel,
 
     /* decode residual and do LPC prediction */
     init_sum = pred_order ? (command == FN_QLPC ? s->lpcqoffset : 0) : coffset;
-    for (i=0; i < s->blocksize; i++) {
+    for (i = 0; i < s->blocksize; i++) {
         sum = init_sum;
-        for (j=0; j<pred_order; j++)
-            sum += coeffs[j] * s->decoded[channel][i-j-1];
-        s->decoded[channel][i] = get_sr_golomb_shorten(&s->gb, residual_size) + (sum >> qshift);
+        for (j = 0; j < pred_order; j++)
+            sum += coeffs[j] * s->decoded[channel][i - j - 1];
+        s->decoded[channel][i] = get_sr_golomb_shorten(&s->gb, residual_size) +
+                                 (sum >> qshift);
     }
 
     /* add offset to current samples */
@@ -334,14 +335,18 @@ static int read_header(ShortenContext *s)
         return AVERROR_INVALIDDATA;
     }
 
-    s->lpcqoffset = 0;
-    s->blocksize = DEFAULT_BLOCK_SIZE;
-    s->nmean = -1;
-    s->version = get_bits(&s->gb, 8);
+    s->lpcqoffset     = 0;
+    s->blocksize      = DEFAULT_BLOCK_SIZE;
+    s->nmean          = -1;
+    s->version        = get_bits(&s->gb, 8);
     s->internal_ftype = get_uint(s, TYPESIZE);
 
     s->channels = get_uint(s, CHANSIZE);
-    if (s->channels <= 0 || s->channels > MAX_CHANNELS) {
+    if (!s->channels) {
+        av_log(s->avctx, AV_LOG_ERROR, "No channels reported\n");
+        return AVERROR_INVALIDDATA;
+    }
+    if (s->channels > MAX_CHANNELS) {
         av_log(s->avctx, AV_LOG_ERROR, "too many channels: %d\n", s->channels);
         s->channels = 0;
         return AVERROR_INVALIDDATA;
@@ -350,23 +355,24 @@ static int read_header(ShortenContext *s)
 
     /* get blocksize if version > 0 */
     if (s->version > 0) {
-        int skip_bytes, blocksize;
+        int skip_bytes;
+        unsigned blocksize;
 
         blocksize = get_uint(s, av_log2(DEFAULT_BLOCK_SIZE));
         if (!blocksize || blocksize > MAX_BLOCKSIZE) {
-            av_log(s->avctx, AV_LOG_ERROR, "invalid or unsupported block size: %d\n",
+            av_log(s->avctx, AV_LOG_ERROR,
+                   "invalid or unsupported block size: %d\n",
                    blocksize);
             return AVERROR(EINVAL);
         }
         s->blocksize = blocksize;
 
-        maxnlpc = get_uint(s, LPCQSIZE);
+        maxnlpc  = get_uint(s, LPCQSIZE);
         s->nmean = get_uint(s, 0);
 
         skip_bytes = get_uint(s, NSKIPSIZE);
-        for (i=0; i<skip_bytes; i++) {
+        for (i = 0; i < skip_bytes; i++)
             skip_bits(&s->gb, 8);
-        }
     }
     s->nwrap = FFMAX(NWRAP, maxnlpc);
 
@@ -380,17 +386,20 @@ static int read_header(ShortenContext *s)
         s->lpcqoffset = V2LPCQOFFSET;
 
     if (get_ur_golomb_shorten(&s->gb, FNSIZE) != FN_VERBATIM) {
-        av_log(s->avctx, AV_LOG_ERROR, "missing verbatim section at beginning of stream\n");
+        av_log(s->avctx, AV_LOG_ERROR,
+               "missing verbatim section at beginning of stream\n");
         return AVERROR_INVALIDDATA;
     }
 
     s->header_size = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE);
-    if (s->header_size >= OUT_BUFFER_SIZE || s->header_size < CANONICAL_HEADER_SIZE) {
-        av_log(s->avctx, AV_LOG_ERROR, "header is wrong size: %d\n", s->header_size);
+    if (s->header_size >= OUT_BUFFER_SIZE ||
+        s->header_size < CANONICAL_HEADER_SIZE) {
+        av_log(s->avctx, AV_LOG_ERROR, "header is wrong size: %d\n",
+               s->header_size);
         return AVERROR_INVALIDDATA;
     }
 
-    for (i=0; i<s->header_size; i++)
+    for (i = 0; i < s->header_size; i++)
         s->header[i] = (char)get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE);
 
     if ((ret = decode_wave_header(s->avctx, s->header, s->header_size)) < 0)
@@ -408,15 +417,15 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
                                 int *got_frame_ptr, AVPacket *avpkt)
 {
     const uint8_t *buf = avpkt->data;
-    int buf_size = avpkt->size;
-    ShortenContext *s = avctx->priv_data;
+    int buf_size       = avpkt->size;
+    ShortenContext *s  = avctx->priv_data;
     int i, input_buf_size = 0;
     int ret;
 
     /* allocate internal bitstream buffer */
-    if(s->max_framesize == 0){
+    if (s->max_framesize == 0) {
         void *tmp_ptr;
-        s->max_framesize= 8192; // should hopefully be enough for the first header
+        s->max_framesize = 8192; // should hopefully be enough for the first header
         tmp_ptr = av_fast_realloc(s->bitstream, &s->allocated_bitstream_size,
                                   s->max_framesize);
         if (!tmp_ptr) {
@@ -427,29 +436,32 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
     }
 
     /* append current packet data to bitstream buffer */
-    if(1 && s->max_framesize){//FIXME truncated
-        buf_size= FFMIN(buf_size, s->max_framesize - s->bitstream_size);
-        input_buf_size= buf_size;
-
-        if(s->bitstream_index + s->bitstream_size + buf_size > s->allocated_bitstream_size){
-            memmove(s->bitstream, &s->bitstream[s->bitstream_index], s->bitstream_size);
-            s->bitstream_index=0;
+    if (1 && s->max_framesize) { //FIXME truncated
+        buf_size       = FFMIN(buf_size, s->max_framesize - s->bitstream_size);
+        input_buf_size = buf_size;
+
+        if (s->bitstream_index + s->bitstream_size + buf_size >
+            s->allocated_bitstream_size) {
+            memmove(s->bitstream, &s->bitstream[s->bitstream_index],
+                    s->bitstream_size);
+            s->bitstream_index = 0;
         }
         if (buf)
-            memcpy(&s->bitstream[s->bitstream_index + s->bitstream_size], buf, buf_size);
-        buf= &s->bitstream[s->bitstream_index];
-        buf_size += s->bitstream_size;
-        s->bitstream_size= buf_size;
+            memcpy(&s->bitstream[s->bitstream_index + s->bitstream_size], buf,
+                   buf_size);
+        buf               = &s->bitstream[s->bitstream_index];
+        buf_size         += s->bitstream_size;
+        s->bitstream_size = buf_size;
 
         /* do not decode until buffer has at least max_framesize bytes or
-           the end of the file has been reached */
+         * the end of the file has been reached */
         if (buf_size < s->max_framesize && avpkt->data) {
             *got_frame_ptr = 0;
             return input_buf_size;
         }
     }
     /* init and position bitstream reader */
-    init_get_bits(&s->gb, buf, buf_size*8);
+    init_get_bits(&s->gb, buf, buf_size * 8);
     skip_bits(&s->gb, s->bitindex);
 
     /* process header or next subblock */
@@ -471,7 +483,7 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
         unsigned int cmd;
         int len;
 
-        if (get_bits_left(&s->gb) < 3+FNSIZE) {
+        if (get_bits_left(&s->gb) < 3 + FNSIZE) {
             *got_frame_ptr = 0;
             break;
         }
@@ -487,32 +499,32 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
         if (!is_audio_command[cmd]) {
             /* process non-audio command */
             switch (cmd) {
-                case FN_VERBATIM:
-                    len = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE);
-                    while (len--) {
-                        get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE);
-                    }
-                    break;
-                case FN_BITSHIFT:
-                    s->bitshift = get_ur_golomb_shorten(&s->gb, BITSHIFTSIZE);
-                    break;
-                case FN_BLOCKSIZE: {
-                    int blocksize = get_uint(s, av_log2(s->blocksize));
-                    if (blocksize > s->blocksize) {
-                        av_log(avctx, AV_LOG_ERROR, "Increasing block size is not supported\n");
-                        return AVERROR_PATCHWELCOME;
-                    }
-                    if (!blocksize || blocksize > MAX_BLOCKSIZE) {
-                        av_log(avctx, AV_LOG_ERROR, "invalid or unsupported "
-                               "block size: %d\n", blocksize);
-                        return AVERROR(EINVAL);
-                    }
-                    s->blocksize = blocksize;
-                    break;
+            case FN_VERBATIM:
+                len = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE);
+                while (len--)
+                    get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE);
+                break;
+            case FN_BITSHIFT:
+                s->bitshift = get_ur_golomb_shorten(&s->gb, BITSHIFTSIZE);
+                break;
+            case FN_BLOCKSIZE: {
+                unsigned blocksize = get_uint(s, av_log2(s->blocksize));
+                if (blocksize > s->blocksize) {
+                    av_log(avctx, AV_LOG_ERROR,
+                           "Increasing block size is not supported\n");
+                    return AVERROR_PATCHWELCOME;
+                }
+                if (!blocksize || blocksize > MAX_BLOCKSIZE) {
+                    av_log(avctx, AV_LOG_ERROR, "invalid or unsupported "
+                                                "block size: %d\n", blocksize);
+                    return AVERROR(EINVAL);
                 }
-                case FN_QUIT:
-                    s->got_quit_command = 1;
-                    break;
+                s->blocksize = blocksize;
+                break;
+            }
+            case FN_QUIT:
+                s->got_quit_command = 1;
+                break;
             }
             if (cmd == FN_BLOCKSIZE || cmd == FN_QUIT) {
                 *got_frame_ptr = 0;
@@ -538,7 +550,7 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
                 coffset = s->offset[channel][0];
             else {
                 int32_t sum = (s->version < 2) ? 0 : s->nmean / 2;
-                for (i=0; i<s->nmean; i++)
+                for (i = 0; i < s->nmean; i++)
                     sum += s->offset[channel][i];
                 coffset = sum / s->nmean;
                 if (s->version >= 2)
@@ -547,21 +559,22 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
 
             /* decode samples for this channel */
             if (cmd == FN_ZERO) {
-                for (i=0; i<s->blocksize; i++)
+                for (i = 0; i < s->blocksize; i++)
                     s->decoded[channel][i] = 0;
             } else {
-                if ((ret = decode_subframe_lpc(s, cmd, channel, residual_size, coffset)) < 0)
+                if ((ret = decode_subframe_lpc(s, cmd, channel,
+                                               residual_size, coffset)) < 0)
                     return ret;
             }
 
             /* update means with info from the current block */
             if (s->nmean > 0) {
                 int32_t sum = (s->version < 2) ? 0 : s->blocksize / 2;
-                for (i=0; i<s->blocksize; i++)
+                for (i = 0; i < s->blocksize; i++)
                     sum += s->decoded[channel][i];
 
-                for (i=1; i<s->nmean; i++)
-                    s->offset[channel][i-1] = s->offset[channel][i];
+                for (i = 1; i < s->nmean; i++)
+                    s->offset[channel][i - 1] = s->offset[channel][i];
 
                 if (s->version < 2)
                     s->offset[channel][s->nmean - 1] = sum / s->blocksize;
@@ -570,11 +583,11 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
             }
 
             /* copy wrap samples for use with next block */
-            for (i=-s->nwrap; i<0; i++)
+            for (i = -s->nwrap; i < 0; i++)
                 s->decoded[channel][i] = s->decoded[channel][i + s->blocksize];
 
             /* shift samples to add in unused zero bits which were removed
-               during encoding */
+             * during encoding */
             fix_bitshift(s, s->decoded[channel]);
 
             /* if this is the last channel in the block, output the samples */
@@ -617,12 +630,12 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
         *got_frame_ptr = 0;
 
 finish_frame:
-    s->bitindex = get_bits_count(&s->gb) - 8*((get_bits_count(&s->gb))/8);
-    i= (get_bits_count(&s->gb))/8;
+    s->bitindex = get_bits_count(&s->gb) - 8 * (get_bits_count(&s->gb) / 8);
+    i           = get_bits_count(&s->gb) / 8;
     if (i > buf_size) {
         av_log(s->avctx, AV_LOG_ERROR, "overread: %d\n", i - buf_size);
-        s->bitstream_size=0;
-        s->bitstream_index=0;
+        s->bitstream_size  = 0;
+        s->bitstream_index = 0;
         return AVERROR_INVALIDDATA;
     }
     if (s->bitstream_size) {
diff --git a/libavfilter/vf_hqdn3d.c b/libavfilter/vf_hqdn3d.c
index b30f8ae8ad..4da80b4858 100644
--- a/libavfilter/vf_hqdn3d.c
+++ b/libavfilter/vf_hqdn3d.c
@@ -327,12 +327,13 @@ static int filter_frame(AVFilterLink *inlink, AVFilterBufferRef *in)
     AVFilterLink *outlink = inlink->dst->outputs[0];
 
     AVFilterBufferRef *out;
-    int direct = 0, c;
+    int direct, c;
 
     if (in->perms & AV_PERM_WRITE) {
         direct = 1;
         out = in;
     } else {
+        direct = 0;
         out = ff_get_video_buffer(outlink, AV_PERM_WRITE, outlink->w, outlink->h);
         if (!out) {
             avfilter_unref_bufferp(&in);
diff --git a/libavformat/flacdec.c b/libavformat/flacdec.c
index ecc47e6ccd..6b19cd5923 100644
--- a/libavformat/flacdec.c
+++ b/libavformat/flacdec.c
@@ -277,11 +277,9 @@ fail:
 
 static int flac_probe(AVProbeData *p)
 {
-    const uint8_t *bufptr = p->buf;
-    const uint8_t *end    = p->buf + p->buf_size;
-
-    if(bufptr > end-4 || memcmp(bufptr, "fLaC", 4)) return 0;
-    else                                            return AVPROBE_SCORE_MAX/2;
+    if (p->buf_size < 4 || memcmp(p->buf, "fLaC", 4))
+        return 0;
+    return AVPROBE_SCORE_MAX/2;
 }
 
 AVInputFormat ff_flac_demuxer = {
diff --git a/libavutil/lzo.c b/libavutil/lzo.c
index c723257212..221a66b9ab 100644
--- a/libavutil/lzo.c
+++ b/libavutil/lzo.c
@@ -110,9 +110,8 @@ static inline void copy(LZOContext *c, int cnt)
  */
 static inline void copy_backptr(LZOContext *c, int back, int cnt)
 {
-    register const uint8_t *src = &c->out[-back];
     register uint8_t *dst       = c->out;
-    if (src < c->out_start || src > dst) {
+    if (dst - c->out_start < back) {
         c->error |= AV_LZO_INVALID_BACKPTR;
         return;
     }