vorbis: Validate that the floor 1 X values contain no duplicates.

Duplicate values in this vector are explicitly banned by the Vorbis I spec
and cause divide-by-zero crashes later on.
(cherry picked from commit ecf79c4d3e8baaf2f303278ef81db6f8407656bc)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>
(cherry picked from commit 9aaaeba45c41cf2b3fa4100abbdee7437428f93c)

Signed-off-by: Anton Khirnov <anton@khirnov.net>
(cherry picked from commit d6e250abfc36b239ef0c1fc9d45d588b853bfcb9)

Signed-off-by: Anton Khirnov <anton@khirnov.net>

4 files changed, 17 insertions, 4 deletions

diff --git a/libavcodec/vorbis.c b/libavcodec/vorbis.c
index 109737976c..5f0ddda32f 100644
--- a/libavcodec/vorbis.c
+++ b/libavcodec/vorbis.c
@@ -123,7 +123,8 @@ int ff_vorbis_len2vlc(uint8_t *bits, uint32_t *codes, uint_fast32_t num)
     return 0;
 }
 
-void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
+int ff_vorbis_ready_floor1_list(AVCodecContext *avccontext,
+                                vorbis_floor1_entry *list, int values)
 {
     int i;
     list[0].sort = 0;
@@ -147,6 +148,11 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
     for (i = 0; i < values - 1; i++) {
         int j;
         for (j = i + 1; j < values; j++) {
+            if (list[i].x == list[j].x) {
+                av_log(avccontext, AV_LOG_ERROR,
+                       "Duplicate value found in floor 1 X coordinates\n");
+                return AVERROR_INVALIDDATA;
+            }
             if (list[list[i].sort].x > list[list[j].sort].x) {
                 int tmp = list[i].sort;
                 list[i].sort = list[j].sort;
@@ -154,6 +160,7 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
             }
         }
     }
+    return 0;
 }
 
 static inline void render_line_unrolled(intptr_t x, uint8_t y, int x1,
diff --git a/libavcodec/vorbis.h b/libavcodec/vorbis.h
index ce9bead425..18a826eee8 100644
--- a/libavcodec/vorbis.h
+++ b/libavcodec/vorbis.h
@@ -35,7 +35,8 @@ typedef struct {
     uint_fast16_t high;
 } vorbis_floor1_entry;
 
-void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values);
+int ff_vorbis_ready_floor1_list(AVCodecContext *avccontext,
+                                vorbis_floor1_entry *list, int values);
 unsigned int ff_vorbis_nth_root(unsigned int x, unsigned int n); // x^(1/n)
 int ff_vorbis_len2vlc(uint8_t *bits, uint32_t *codes, uint_fast32_t num);
 void ff_vorbis_floor1_render_list(vorbis_floor1_entry * list, int values,
diff --git a/libavcodec/vorbis_dec.c b/libavcodec/vorbis_dec.c
index e5ad4aaae3..a2ae854699 100644
--- a/libavcodec/vorbis_dec.c
+++ b/libavcodec/vorbis_dec.c
@@ -550,7 +550,11 @@ static int vorbis_parse_setup_hdr_floors(vorbis_context *vc)
             }
 
 // Precalculate order of x coordinates - needed for decode
-            ff_vorbis_ready_floor1_list(floor_setup->data.t1.list, floor_setup->data.t1.x_list_dim);
+            if (ff_vorbis_ready_floor1_list(vc->avccontext,
+                                            floor_setup->data.t1.list,
+                                            floor_setup->data.t1.x_list_dim)) {
+                return AVERROR_INVALIDDATA;
+            }
         } else if (floor_setup->floor_type == 0) {
             uint_fast8_t max_codebook_dim = 0;
 
diff --git a/libavcodec/vorbis_enc.c b/libavcodec/vorbis_enc.c
index 934463d1e1..2fb41d16eb 100644
--- a/libavcodec/vorbis_enc.c
+++ b/libavcodec/vorbis_enc.c
@@ -302,7 +302,8 @@ static void create_vorbis_context(vorbis_enc_context *venc,
         };
         fc->list[i].x = a[i - 2];
     }
-    ff_vorbis_ready_floor1_list(fc->list, fc->values);
+    if (ff_vorbis_ready_floor1_list(avccontext, fc->list, fc->values))
+        return AVERROR(EINVAL);
 
     venc->nresidues = 1;
     venc->residues  = av_malloc(sizeof(vorbis_enc_residue) * venc->nresidues);