avcodec/rpzaenc: stop accessing out of bounds frame

(cherry picked from commit 92f9b28ed84a77138105475beba16c146bdaf984)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

1 files changed, 15 insertions, 7 deletions

diff --git a/libavcodec/rpzaenc.c b/libavcodec/rpzaenc.c
index 337b1faf5b..3e97c87db2 100644
--- a/libavcodec/rpzaenc.c
+++ b/libavcodec/rpzaenc.c
@@ -205,7 +205,7 @@ static void get_max_component_diff(BlockInfo *bi, uint16_t *block_ptr,
 
     // loop thru and compare pixels
     for (y = 0; y < bi->block_height; y++) {
-        for (x = 0; x < bi->block_width; x++){
+        for (x = 0; x < bi->block_width; x++) {
             // TODO:  optimize
             min_r = FFMIN(R(block_ptr[x]), min_r);
             min_g = FFMIN(G(block_ptr[x]), min_g);
@@ -277,7 +277,7 @@ static int leastsquares(uint16_t *block_ptr, BlockInfo *bi,
         return -1;
 
     for (i = 0; i < bi->block_height; i++) {
-        for (j = 0; j < bi->block_width; j++){
+        for (j = 0; j < bi->block_width; j++) {
             x = GET_CHAN(block_ptr[j], xchannel);
             y = GET_CHAN(block_ptr[j], ychannel);
             sumx += x;
@@ -324,7 +324,7 @@ static int calc_lsq_max_fit_error(uint16_t *block_ptr, BlockInfo *bi,
     int max_err = 0;
 
     for (i = 0; i < bi->block_height; i++) {
-        for (j = 0; j < bi->block_width; j++){
+        for (j = 0; j < bi->block_width; j++) {
             int x_inc, lin_y, lin_x;
             x = GET_CHAN(block_ptr[j], xchannel);
             y = GET_CHAN(block_ptr[j], ychannel);
@@ -419,7 +419,9 @@ static void update_block_in_prev_frame(const uint16_t *src_pixels,
                                        uint16_t *dest_pixels,
                                        const BlockInfo *bi, int block_counter)
 {
-    for (int y = 0; y < 4; y++) {
+    const int y_size = FFMIN(4, bi->image_height - bi->row * 4);
+
+    for (int y = 0; y < y_size; y++) {
         memcpy(dest_pixels, src_pixels, 8);
         dest_pixels += bi->rowstride;
         src_pixels += bi->rowstride;
@@ -729,14 +731,15 @@ post_skip :
 
             if (err > s->sixteen_color_thresh) { // DO SIXTEEN COLOR BLOCK
                 uint16_t *row_ptr;
-                int rgb555;
+                int y_size, rgb555;
 
                 block_offset = get_block_info(&bi, block_counter);
 
                 row_ptr = &src_pixels[block_offset];
+                y_size = FFMIN(4, bi.image_height - bi.row * 4);
 
-                for (int y = 0; y < 4; y++) {
-                    for (int x = 0; x < 4; x++){
+                for (int y = 0; y < y_size; y++) {
+                    for (int x = 0; x < 4; x++) {
                         rgb555 = row_ptr[x] & ~0x8000;
 
                         put_bits(&s->pb, 16, rgb555);
@@ -744,6 +747,11 @@ post_skip :
                     row_ptr += bi.rowstride;
                 }
 
+                for (int y = y_size; y < 4; y++) {
+                    for (int x = 0; x < 4; x++)
+                        put_bits(&s->pb, 16, 0);
+                }
+
                 block_counter++;
             } else { // FOUR COLOR BLOCK
                 block_counter += encode_four_color_block(min_color, max_color,