From 3a2731cbd31d0c5681ddbc7c78edd5c53c4d0032 Mon Sep 17 00:00:00 2001
From: Justin Ruggles <justin.ruggles@gmail.com>
Date: Wed, 7 Nov 2012 14:48:28 -0500
Subject: flacenc: ensure the order is within the min/max range in LPC order
 search

This fixes use of uninitialized values when the FLAC encoder uses the
2-level, 4-level, and 8-level search methods. Fixes failure of the
fate-flac-24-comp-8 test when run using valgrind.
---
 libavcodec/flacenc.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'libavcodec/flacenc.c')

diff --git a/libavcodec/flacenc.c b/libavcodec/flacenc.c
index 93d4646604..54bc64c9de 100644
--- a/libavcodec/flacenc.c
+++ b/libavcodec/flacenc.c
@@ -799,14 +799,16 @@ static int encode_residual_ch(FlacEncodeContext *s, int ch)
         omethod == ORDER_METHOD_8LEVEL) {
         int levels = 1 << omethod;
         uint64_t bits[1 << ORDER_METHOD_8LEVEL];
-        int order;
+        int order       = -1;
         int opt_index   = levels-1;
         opt_order       = max_order-1;
         bits[opt_index] = UINT32_MAX;
         for (i = levels-1; i >= 0; i--) {
+            int last_order = order;
             order = min_order + (((max_order-min_order+1) * (i+1)) / levels)-1;
-            if (order < 0)
-                order = 0;
+            order = av_clip(order, min_order - 1, max_order - 1);
+            if (order == last_order)
+                continue;
             s->flac_dsp.lpc_encode(res, smp, n, order+1, coefs[order],
                                    shift[order]);
             bits[i] = find_subframe_rice_params(s, sub, order+1);
-- 
cgit v1.2.3