Fix DPX decoder

Rewrite the DPX decoder to work with provided sample DPXs at
http://samples.ffmpeg.org/image-samples/dpx_samples.zip

The decoder could only decode 8 and 10 bit without alpha correctly,
failing or even crashing at other flavors. This patch aims to fix
these issues, properly decoding all variants of DPX provided in the
referenced DPX sample zip. For 10 and 12 bit, the alpha channel
is ignored, but decoding is still possible.

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 90 insertions, 74 deletions

diff --git a/libavcodec/dpx.c b/libavcodec/dpx.c
index d96e0ec19e..9b96ed8c64 100644
--- a/libavcodec/dpx.c
+++ b/libavcodec/dpx.c
@@ -41,13 +41,27 @@ static unsigned int read32(const uint8_t **ptr, int is_big)
     return temp;
 }
 
+static uint16_t read10in32(const uint8_t **ptr, uint32_t * lbuf,
+                                  int * n_datum, int is_big)
+{
+    if (*n_datum)
+        (*n_datum)--;
+    else {
+        *lbuf = read32(ptr, is_big);
+        *n_datum = 2;
+    }
+
+    *lbuf = (*lbuf << 10) | (*lbuf >> 22);
+
+    return *lbuf & 0x3FF;
+}
+
 static int decode_frame(AVCodecContext *avctx,
                         void *data,
                         int *data_size,
                         AVPacket *avpkt)
 {
     const uint8_t *buf = avpkt->data;
-    const uint8_t *buf_end = avpkt->data + avpkt->size;
     int buf_size       = avpkt->size;
     DPXContext *const s = avctx->priv_data;
     AVFrame *picture  = data;
@@ -57,10 +71,10 @@ static int decode_frame(AVCodecContext *avctx,
     unsigned int offset;
     int magic_num, endian;
     int x, y, i;
-    int w, h, stride, bits_per_color, descriptor, elements, target_packet_size, source_packet_size;
-    int planar;
+    int w, h, bits_per_color, descriptor, elements, packing, total_size;
 
-    unsigned int rgbBuffer;
+    unsigned int rgbBuffer = 0;
+    int n_datum = 0;
 
     if (avpkt->size <= 1634) {
         av_log(avctx, AV_LOG_ERROR, "Packet too small for DPX header\n");
@@ -99,8 +113,10 @@ static int decode_frame(AVCodecContext *avctx,
     buf += 3;
     avctx->bits_per_raw_sample =
     bits_per_color = buf[0];
+    buf++;
+    packing = *((uint16_t*)buf);
 
-    buf += 825;
+    buf += 824;
     avctx->sample_aspect_ratio.num = read32(&buf, endian);
     avctx->sample_aspect_ratio.den = read32(&buf, endian);
     if (avctx->sample_aspect_ratio.num > 0 && avctx->sample_aspect_ratio.den > 0)
@@ -129,25 +145,27 @@ static int decode_frame(AVCodecContext *avctx,
             } else {
                 avctx->pix_fmt = AV_PIX_FMT_RGB24;
             }
-            source_packet_size = elements;
-            target_packet_size = elements;
-            planar = 0;
+            total_size = avctx->width * avctx->height * elements;
             break;
         case 10:
+            if (!packing) {
+                av_log(avctx, AV_LOG_ERROR, "Packing to 32bit required\n");
+                return -1;
+            }
             avctx->pix_fmt = AV_PIX_FMT_GBRP10;
-            target_packet_size = 6;
-            source_packet_size = 4;
-            planar = 1;
+            total_size = (4 * avctx->width * avctx->height * elements) / 3;
             break;
         case 12:
+            if (!packing) {
+                av_log(avctx, AV_LOG_ERROR, "Packing to 16bit required\n");
+                return -1;
+            }
             if (endian) {
-                avctx->pix_fmt = elements == 4 ? AV_PIX_FMT_GBRP12BE : AV_PIX_FMT_GBRP12BE;
+                avctx->pix_fmt = AV_PIX_FMT_GBRP12BE;
             } else {
-                avctx->pix_fmt = elements == 4 ? AV_PIX_FMT_GBRP12LE : AV_PIX_FMT_GBRP12LE;
+                avctx->pix_fmt = AV_PIX_FMT_GBRP12LE;
             }
-            target_packet_size = 6;
-            source_packet_size = 6;
-            planar = 1;
+            total_size = 2 * avctx->width * avctx->height * elements;
             break;
         case 16:
             if (endian) {
@@ -155,9 +173,7 @@ static int decode_frame(AVCodecContext *avctx,
             } else {
                 avctx->pix_fmt = elements == 4 ? AV_PIX_FMT_RGBA64LE : AV_PIX_FMT_RGB48LE;
             }
-            target_packet_size =
-            source_packet_size = elements * 2;
-            planar = 0;
+            total_size = 2 * avctx->width * avctx->height * elements;
             break;
         default:
             av_log(avctx, AV_LOG_ERROR, "Unsupported color depth : %d\n", bits_per_color);
@@ -180,68 +196,68 @@ static int decode_frame(AVCodecContext *avctx,
 
     for (i=0; i<AV_NUM_DATA_POINTERS; i++)
         ptr[i] = p->data[i];
-    stride = p->linesize[0];
 
-    if (source_packet_size*avctx->width*avctx->height > buf_end - buf) {
+    if (total_size > avpkt->size) {
         av_log(avctx, AV_LOG_ERROR, "Overread buffer. Invalid header?\n");
         return -1;
     }
     switch (bits_per_color) {
-        case 10:
-            for (x = 0; x < avctx->height; x++) {
-                uint16_t *dst[3] = {(uint16_t*)ptr[0],
-                                    (uint16_t*)ptr[1],
-                                    (uint16_t*)ptr[2]};
-               for (y = 0; y < avctx->width; y++) {
-                   rgbBuffer = read32(&buf, endian);
-                   *dst[0]++ = (rgbBuffer >> 12) & 0x3FF;
-                   *dst[1]++ = (rgbBuffer >> 2)  & 0x3FF;
-                   *dst[2]++ = (rgbBuffer >> 22) & 0x3FF;
-               }
-               for (i=0; i<3; i++)
-                   ptr[i] += p->linesize[i];
+    case 10:
+        for (x = 0; x < avctx->height; x++) {
+            uint16_t *dst[3] = {(uint16_t*)ptr[0],
+                                (uint16_t*)ptr[1],
+                                (uint16_t*)ptr[2]};
+            for (y = 0; y < avctx->width; y++) {
+                *dst[2]++ = read10in32(&buf, &rgbBuffer,
+                                       &n_datum, endian);
+                *dst[0]++ = read10in32(&buf, &rgbBuffer,
+                                       &n_datum, endian);
+                *dst[1]++ = read10in32(&buf, &rgbBuffer,
+                                       &n_datum, endian);
+                // For 10 bit, ignore alpha
+                if (elements == 4)
+                    read10in32(&buf, &rgbBuffer,
+                               &n_datum, endian);
             }
-            break;
-        case 8:
-        case 12:
-        case 16:
-            if (planar) {
-                int source_bpc = target_packet_size / elements;
-                int target_bpc = target_packet_size / elements;
-                for (x = 0; x < avctx->height; x++) {
-                    uint8_t *dst[AV_NUM_DATA_POINTERS];
-                    for (i=0; i<elements; i++)
-                        dst[i] = ptr[i];
-                    for (y = 0; y < avctx->width; y++) {
-                        for (i=0; i<3; i++) {
-                            memcpy(dst[i], buf, FFMIN(source_bpc, target_bpc));
-                            dst[i] += target_bpc;
-                            buf += source_bpc;
-                        }
-                    }
-                    for (i=0; i<elements; i++)
-                        ptr[i] += p->linesize[i];
-                }
-            } else {
-                if (source_packet_size == target_packet_size) {
-                    for (x = 0; x < avctx->height; x++) {
-                        memcpy(ptr[0], buf, target_packet_size*avctx->width);
-                        ptr[0] += stride;
-                        buf += source_packet_size*avctx->width;
-                    }
-                } else {
-                    for (x = 0; x < avctx->height; x++) {
-                        uint8_t *dst = ptr[0];
-                        for (y = 0; y < avctx->width; y++) {
-                            memcpy(dst, buf, target_packet_size);
-                            dst += target_packet_size;
-                            buf += source_packet_size;
-                        }
-                        ptr[0] += stride;
-                    }
-                }
+            for (i = 0; i < 3; i++)
+                ptr[i] += p->linesize[i];
+        }
+        break;
+    case 12:
+        for (x = 0; x < avctx->height; x++) {
+            uint16_t *dst[3] = {(uint16_t*)ptr[0],
+                                (uint16_t*)ptr[1],
+                                (uint16_t*)ptr[2]};
+            for (y = 0; y < avctx->width; y++) {
+                *dst[2] = *((uint16_t*)buf);
+                *dst[2] = (*dst[2] >> 4) | (*dst[2] << 12);
+                dst[2]++;
+                buf += 2;
+                *dst[0] = *((uint16_t*)buf);
+                *dst[0] = (*dst[0] >> 4) | (*dst[0] << 12);
+                dst[0]++;
+                buf += 2;
+                *dst[1] = *((uint16_t*)buf);
+                *dst[1] = (*dst[1] >> 4) | (*dst[1] << 12);
+                dst[1]++;
+                buf += 2;
+                // For 12 bit, ignore alpha
+                if (elements == 4)
+                    buf += 2;
             }
-            break;
+            for (i = 0; i < 3; i++)
+                ptr[i] += p->linesize[i];
+        }
+        break;
+    case 16:
+        elements *= 2;
+    case 8:
+        for (x = 0; x < avctx->height; x++) {
+            memcpy(ptr[0], buf, elements*avctx->width);
+            ptr[0] += p->linesize[0];
+            buf += elements*avctx->width;
+        }
+        break;
     }
 
     *picture   = s->picture;