avcodec/smcenc: fix encoding of width not multiple of 4

1 files changed, 36 insertions, 18 deletions

diff --git a/libavcodec/smcenc.c b/libavcodec/smcenc.c
index a2d224bd09..39abf2407b 100644
--- a/libavcodec/smcenc.c
+++ b/libavcodec/smcenc.c
@@ -57,11 +57,13 @@ typedef struct SMCContext {
 { \
     for (int block = 0; block < nb_blocks && pixel_ptr && row_ptr; block++) { \
         pixel_ptr += 4; \
+        cur_x += 4; \
         if (pixel_ptr - row_ptr >= width) \
         { \
             row_ptr += stride * 4; \
             pixel_ptr = row_ptr; \
             cur_y += 4; \
+            cur_x  = 0; \
         } \
     } \
 }
@@ -121,7 +123,6 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
     const uint8_t *pixel_ptr, *row_ptr;
     const int height = frame->height;
     const int width = frame->width;
-    uint8_t block_values[16];
     int block_counter = 0;
     int color_pair_index = 0;
     int color_quad_index = 0;
@@ -129,6 +130,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
     int color_table_index;  /* indexes to color pair, quad, or octet tables */
     int total_blocks;
     int cur_y = 0;
+    int cur_x = 0;
 
     memset(s->color_pairs, 0, sizeof(s->color_pairs));
     memset(s->color_quads, 0, sizeof(s->color_quads));
@@ -150,16 +152,18 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
         int distinct = 0;
         int blocks = 0;
         int frame_y = cur_y;
+        int frame_x = cur_x;
 
         while (prev_pixels && s->key_frame == 0 && block_counter + inter_skip_blocks < total_blocks) {
             const int y_size = FFMIN(4, height - cur_y);
+            const int x_size = FFMIN(4, width  - cur_x);
             int compare = 0;
 
             for (int y = 0; y < y_size; y++) {
                 const ptrdiff_t offset = pixel_ptr - row_ptr;
                 const uint8_t *prev_pixel_ptr = prev_pixels + cur_y * prev_stride + offset;
 
-                compare |= memcmp(prev_pixel_ptr + y * prev_stride, pixel_ptr + y * stride, 4);
+                compare |= memcmp(prev_pixel_ptr + y * prev_stride, pixel_ptr + y * stride, x_size);
                 if (compare)
                     break;
             }
@@ -177,9 +181,11 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
         pixel_ptr = xpixel_ptr;
         row_ptr = xrow_ptr;
         cur_y = frame_y;
+        cur_x = frame_x;
 
         while (block_counter > 0 && block_counter + intra_skip_blocks < total_blocks) {
             const int y_size = FFMIN(4, height - cur_y);
+            const int x_size = FFMIN(4, width  - cur_x);
             const ptrdiff_t offset = pixel_ptr - src_pixels;
             const int sy = offset / stride;
             const int sx = offset % stride;
@@ -189,7 +195,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
             int compare = 0;
 
             for (int y = 0; y < y_size; y++) {
-                compare |= memcmp(old_pixel_ptr + y * stride, pixel_ptr + y * stride, 4);
+                compare |= memcmp(old_pixel_ptr + y * stride, pixel_ptr + y * stride, x_size);
                 if (compare)
                     break;
             }
@@ -206,14 +212,18 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
         pixel_ptr = xpixel_ptr;
         row_ptr = xrow_ptr;
         cur_y = frame_y;
+        cur_x = frame_x;
 
         while (block_counter + coded_blocks < total_blocks && coded_blocks < 256) {
             const int y_size = FFMIN(4, height - cur_y);
+            const int x_size = FFMIN(4, width  - cur_y);
+            const int nb_elements = x_size * y_size;
+            uint8_t block_values[16] = { 0 };
             for (int y = 0; y < y_size; y++)
-                memcpy(block_values + y * 4, pixel_ptr + y * stride, 4);
+                memcpy(block_values + y * x_size, pixel_ptr + y * stride, x_size);
 
-            qsort(block_values, 16, sizeof(block_values[0]), smc_cmp_values);
-            s->next_nb_distinct = count_distinct_items(block_values, s->next_distinct_values, 16);
+            qsort(block_values, nb_elements, sizeof(block_values[0]), smc_cmp_values);
+            s->next_nb_distinct = count_distinct_items(block_values, s->next_distinct_values, nb_elements);
             if (coded_blocks == 0) {
                 memcpy(distinct_values, s->next_distinct_values, sizeof(s->distinct_values));
                 s->nb_distinct = s->next_nb_distinct;
@@ -235,30 +245,27 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
         pixel_ptr = xpixel_ptr;
         row_ptr = xrow_ptr;
         cur_y = frame_y;
+        cur_x = frame_x;
 
         blocks = coded_blocks;
         distinct = coded_distinct;
 
-        if (intra_skip_blocks > 0 && intra_skip_blocks >= inter_skip_blocks &&
-            intra_skip_blocks > 0) {
+        if (intra_skip_blocks > 0 && intra_skip_blocks >= inter_skip_blocks) {
             distinct = 17;
             blocks = intra_skip_blocks;
         }
 
-        if (intra_skip_blocks > 16 && intra_skip_blocks >= inter_skip_blocks &&
-            intra_skip_blocks > 0) {
+        if (intra_skip_blocks > 16 && intra_skip_blocks >= inter_skip_blocks) {
             distinct = 18;
             blocks = intra_skip_blocks;
         }
 
-        if (inter_skip_blocks > 0 && inter_skip_blocks > intra_skip_blocks &&
-            inter_skip_blocks > 0) {
+        if (inter_skip_blocks > 0 && inter_skip_blocks > intra_skip_blocks) {
             distinct = 19;
             blocks = inter_skip_blocks;
         }
 
-        if (inter_skip_blocks > 16 && inter_skip_blocks > intra_skip_blocks &&
-            inter_skip_blocks > 0) {
+        if (inter_skip_blocks > 16 && inter_skip_blocks > intra_skip_blocks) {
             distinct = 20;
             blocks = inter_skip_blocks;
         }
@@ -304,15 +311,17 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
 
             for (int i = 0; i < blocks; i++) {
                 const int y_size = FFMIN(4, height - cur_y);
+                const int x_size = FFMIN(4, width  - cur_x);
                 uint8_t value = s->color_pairs[color_table_index][1];
                 uint16_t flags = 0;
                 int shift = 15;
 
                 for (int y = 0; y < y_size; y++) {
-                    for (int x = 0; x < 4; x++) {
+                    for (int x = 0; x < x_size; x++) {
                         flags |= (value == pixel_ptr[x + y * stride]) << shift;
                         shift--;
                     }
+                    shift -= 4 - x_size;
                 }
 
                 bytestream2_put_be16(pb, flags);
@@ -353,6 +362,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
 
             for (int i = 0; i < blocks; i++) {
                 const int y_size = FFMIN(4, height - cur_y);
+                const int x_size = FFMIN(4, width  - cur_x);
                 uint32_t flags = 0;
                 uint8_t quad[4];
                 int shift = 30;
@@ -361,7 +371,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
                     quad[k] = s->color_quads[color_table_index][k];
 
                 for (int y = 0; y < y_size; y++) {
-                    for (int x = 0; x < 4; x++) {
+                    for (int x = 0; x < x_size; x++) {
                         int pixel = pixel_ptr[x + y * stride];
                         uint32_t idx = 0;
 
@@ -375,6 +385,8 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
                         flags |= idx << shift;
                         shift -= 2;
                     }
+
+                    shift -= 2 * (4 - x_size);
                 }
 
                 bytestream2_put_be32(pb, flags);
@@ -421,6 +433,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
 
             for (int i = 0; i < blocks; i++) {
                 const int y_size = FFMIN(4, height - cur_y);
+                const int x_size = FFMIN(4, width  - cur_x);
                 uint64_t flags = 0;
                 uint8_t octet[8];
                 int shift = 45;
@@ -429,7 +442,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
                     octet[k] = s->color_octets[color_table_index][k];
 
                 for (int y = 0; y < y_size; y++) {
-                    for (int x = 0; x < 4; x++) {
+                    for (int x = 0; x < x_size; x++) {
                         int pixel = pixel_ptr[x + y * stride];
                         uint64_t idx = 0;
 
@@ -443,6 +456,8 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
                         flags |= idx << shift;
                         shift -= 3;
                     }
+
+                    shift -= 3 * (4 - x_size);
                 }
 
                 bytestream2_put_be16(pb, ((flags >> 32) & 0xFFF0) | ((flags >> 8) & 0xF));
@@ -456,9 +471,12 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
             bytestream2_put_byte(pb, 0xE0 | (blocks - 1));
             for (int i = 0; i < blocks; i++) {
                 const int y_size = FFMIN(4, height - cur_y);
+                const int x_size = FFMIN(4, width  - cur_x);
                 for (int y = 0; y < y_size; y++) {
-                    for (int x = 0; x < 4; x++)
+                    for (int x = 0; x < x_size; x++)
                         bytestream2_put_byte(pb, pixel_ptr[x + y * stride]);
+                    for (int x = x_size; x < 4; x++)
+                        bytestream2_put_byte(pb, 0);
                 }
 
                 for (int y = y_size; y < 4; y++) {