pixdesc: mark pseudopaletted formats with a special flag.

This makes it possible to dintinguish them from PAL8.

Fixes an invalid write in avpicture_layout().

4 files changed, 17 insertions, 8 deletions

diff --git a/libavutil/avutil.h b/libavutil/avutil.h
index 0e62b4a13f..05e9248375 100644
--- a/libavutil/avutil.h
+++ b/libavutil/avutil.h
@@ -155,7 +155,7 @@
 
 #define LIBAVUTIL_VERSION_MAJOR 51
 #define LIBAVUTIL_VERSION_MINOR 22
-#define LIBAVUTIL_VERSION_MICRO  0
+#define LIBAVUTIL_VERSION_MICRO  1
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \
                                                LIBAVUTIL_VERSION_MINOR, \
diff --git a/libavutil/imgutils.c b/libavutil/imgutils.c
index b734db7bb1..8c8251bddb 100644
--- a/libavutil/imgutils.c
+++ b/libavutil/imgutils.c
@@ -108,7 +108,8 @@ int av_image_fill_pointers(uint8_t *data[4], enum PixelFormat pix_fmt, int heigh
         return AVERROR(EINVAL);
     size[0] = linesizes[0] * height;
 
-    if (desc->flags & PIX_FMT_PAL) {
+    if (desc->flags & PIX_FMT_PAL ||
+        desc->flags & PIX_FMT_PSEUDOPAL) {
         size[0] = (size[0] + 3) & ~3;
         data[1] = ptr + size[0]; /* palette is stored here as 256 32 bits words */
         return size[0] + 256 * 4;
@@ -196,7 +197,8 @@ int av_image_alloc(uint8_t *pointers[4], int linesizes[4],
         av_free(buf);
         return ret;
     }
-    if (av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PAL)
+    if (av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PAL ||
+        av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PSEUDOPAL)
         ff_set_systematic_pal2((uint32_t*)pointers[1], pix_fmt);
 
     return ret;
@@ -243,7 +245,8 @@ void av_image_copy(uint8_t *dst_data[4], int dst_linesizes[4],
     if (desc->flags & PIX_FMT_HWACCEL)
         return;
 
-    if (desc->flags & PIX_FMT_PAL) {
+    if (desc->flags & PIX_FMT_PAL ||
+        desc->flags & PIX_FMT_PSEUDOPAL) {
         av_image_copy_plane(dst_data[0], dst_linesizes[0],
                             src_data[0], src_linesizes[0],
                             width, height);
diff --git a/libavutil/pixdesc.c b/libavutil/pixdesc.c
index 6e1f23b97f..8e08b5a989 100644
--- a/libavutil/pixdesc.c
+++ b/libavutil/pixdesc.c
@@ -327,7 +327,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = {
             { 0, 0, 1, 3, 2 },        /* G */
             { 0, 0, 1, 0, 2 },        /* R */
         },
-        .flags = PIX_FMT_PAL | PIX_FMT_RGB,
+        .flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL,
     },
     [PIX_FMT_BGR4] = {
         .name = "bgr4",
@@ -351,7 +351,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = {
             { 0, 0, 1, 1, 1 },        /* G */
             { 0, 0, 1, 0, 0 },        /* R */
         },
-        .flags = PIX_FMT_PAL | PIX_FMT_RGB,
+        .flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL,
     },
     [PIX_FMT_RGB8] = {
         .name = "rgb8",
@@ -363,7 +363,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = {
             { 0, 0, 1, 3, 2 },        /* G */
             { 0, 0, 1, 0, 2 },        /* B */
         },
-        .flags = PIX_FMT_PAL | PIX_FMT_RGB,
+        .flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL,
     },
     [PIX_FMT_RGB4] = {
         .name = "rgb4",
@@ -387,7 +387,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = {
             { 0, 0, 1, 1, 1 },        /* G */
             { 0, 0, 1, 0, 0 },        /* B */
         },
-        .flags = PIX_FMT_PAL | PIX_FMT_RGB,
+        .flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL,
     },
     [PIX_FMT_NV12] = {
         .name = "nv12",
diff --git a/libavutil/pixdesc.h b/libavutil/pixdesc.h
index b5972c78ff..23a97ee231 100644
--- a/libavutil/pixdesc.h
+++ b/libavutil/pixdesc.h
@@ -89,6 +89,12 @@ typedef struct AVPixFmtDescriptor{
 #define PIX_FMT_HWACCEL   8 ///< Pixel format is an HW accelerated format.
 #define PIX_FMT_PLANAR   16 ///< At least one pixel component is not in the first data plane
 #define PIX_FMT_RGB      32 ///< The pixel format contains RGB-like data (as opposed to YUV/grayscale)
+/**
+ * The pixel format is "pseudo-paletted". This means that Libav treats it as
+ * paletted internally, but the palette is generated by the decoder and is not
+ * stored in the file.
+ */
+#define PIX_FMT_PSEUDOPAL 64
 
 /**
  * The array of all the pixel format descriptors.