Move av_find_best_pix_fmt_of_2() from avcodec to avutil

This avoids a dependancy of libavfilter on libavcodec

See Ticket 3592
Fixes Ticket2784

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 223 insertions, 0 deletions

diff --git a/libavutil/pixdesc.c b/libavutil/pixdesc.c
index 036481aa7a..6b1b05f3fa 100644
--- a/libavutil/pixdesc.c
+++ b/libavutil/pixdesc.c
@@ -2082,3 +2082,226 @@ enum AVPixelFormat av_pix_fmt_swap_endianness(enum AVPixelFormat pix_fmt)
 
     return get_pix_fmt_internal(name);
 }
+
+#define FF_COLOR_NA      -1
+#define FF_COLOR_RGB      0 /**< RGB color space */
+#define FF_COLOR_GRAY     1 /**< gray color space */
+#define FF_COLOR_YUV      2 /**< YUV color space. 16 <= Y <= 235, 16 <= U, V <= 240 */
+#define FF_COLOR_YUV_JPEG 3 /**< YUV color space. 0 <= Y <= 255, 0 <= U, V <= 255 */
+
+#define pixdesc_has_alpha(pixdesc) \
+    ((pixdesc)->nb_components == 2 || (pixdesc)->nb_components == 4 || (pixdesc)->flags & AV_PIX_FMT_FLAG_PAL)
+
+
+static int get_color_type(const AVPixFmtDescriptor *desc) {
+    if (desc->flags & AV_PIX_FMT_FLAG_PAL)
+        return FF_COLOR_RGB;
+
+    if(desc->nb_components == 1 || desc->nb_components == 2)
+        return FF_COLOR_GRAY;
+
+    if(desc->name && !strncmp(desc->name, "yuvj", 4))
+        return FF_COLOR_YUV_JPEG;
+
+    if(desc->flags & AV_PIX_FMT_FLAG_RGB)
+        return  FF_COLOR_RGB;
+
+    if(desc->nb_components == 0)
+        return FF_COLOR_NA;
+
+    return FF_COLOR_YUV;
+}
+
+static int get_pix_fmt_depth(int *min, int *max, enum AVPixelFormat pix_fmt)
+{
+    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
+    int i;
+
+    if (!desc || !desc->nb_components) {
+        *min = *max = 0;
+        return AVERROR(EINVAL);
+    }
+
+    *min = INT_MAX, *max = -INT_MAX;
+    for (i = 0; i < desc->nb_components; i++) {
+        *min = FFMIN(desc->comp[i].depth_minus1+1, *min);
+        *max = FFMAX(desc->comp[i].depth_minus1+1, *max);
+    }
+    return 0;
+}
+
+static int get_pix_fmt_score(enum AVPixelFormat dst_pix_fmt,
+                              enum AVPixelFormat src_pix_fmt,
+                              unsigned *lossp, unsigned consider)
+{
+    const AVPixFmtDescriptor *src_desc = av_pix_fmt_desc_get(src_pix_fmt);
+    const AVPixFmtDescriptor *dst_desc = av_pix_fmt_desc_get(dst_pix_fmt);
+    int src_color, dst_color;
+    int src_min_depth, src_max_depth, dst_min_depth, dst_max_depth;
+    int ret, loss, i, nb_components;
+    int score = INT_MAX - 1;
+
+    if (dst_pix_fmt >= AV_PIX_FMT_NB || dst_pix_fmt <= AV_PIX_FMT_NONE)
+        return ~0;
+
+    /* compute loss */
+    *lossp = loss = 0;
+
+    if (dst_pix_fmt == src_pix_fmt)
+        return INT_MAX;
+
+    if ((ret = get_pix_fmt_depth(&src_min_depth, &src_max_depth, src_pix_fmt)) < 0)
+        return ret;
+    if ((ret = get_pix_fmt_depth(&dst_min_depth, &dst_max_depth, dst_pix_fmt)) < 0)
+        return ret;
+
+    src_color = get_color_type(src_desc);
+    dst_color = get_color_type(dst_desc);
+    if (dst_pix_fmt == AV_PIX_FMT_PAL8)
+        nb_components = FFMIN(src_desc->nb_components, 4);
+    else
+        nb_components = FFMIN(src_desc->nb_components, dst_desc->nb_components);
+
+    for (i = 0; i < nb_components; i++) {
+        int depth_minus1 = (dst_pix_fmt == AV_PIX_FMT_PAL8) ? 7/nb_components : dst_desc->comp[i].depth_minus1;
+        if (src_desc->comp[i].depth_minus1 > depth_minus1 && (consider & FF_LOSS_DEPTH)) {
+            loss |= FF_LOSS_DEPTH;
+            score -= 65536 >> depth_minus1;
+        }
+    }
+
+    if (consider & FF_LOSS_RESOLUTION) {
+        if (dst_desc->log2_chroma_w > src_desc->log2_chroma_w) {
+            loss |= FF_LOSS_RESOLUTION;
+            score -= 256 << dst_desc->log2_chroma_w;
+        }
+        if (dst_desc->log2_chroma_h > src_desc->log2_chroma_h) {
+            loss |= FF_LOSS_RESOLUTION;
+            score -= 256 << dst_desc->log2_chroma_h;
+        }
+        // don't favor 422 over 420 if downsampling is needed, because 420 has much better support on the decoder side
+        if (dst_desc->log2_chroma_w == 1 && src_desc->log2_chroma_w == 0 &&
+            dst_desc->log2_chroma_h == 1 && src_desc->log2_chroma_h == 0 ) {
+            score += 512;
+        }
+    }
+
+    if(consider & FF_LOSS_COLORSPACE)
+    switch(dst_color) {
+    case FF_COLOR_RGB:
+        if (src_color != FF_COLOR_RGB &&
+            src_color != FF_COLOR_GRAY)
+            loss |= FF_LOSS_COLORSPACE;
+        break;
+    case FF_COLOR_GRAY:
+        if (src_color != FF_COLOR_GRAY)
+            loss |= FF_LOSS_COLORSPACE;
+        break;
+    case FF_COLOR_YUV:
+        if (src_color != FF_COLOR_YUV)
+            loss |= FF_LOSS_COLORSPACE;
+        break;
+    case FF_COLOR_YUV_JPEG:
+        if (src_color != FF_COLOR_YUV_JPEG &&
+            src_color != FF_COLOR_YUV &&
+            src_color != FF_COLOR_GRAY)
+            loss |= FF_LOSS_COLORSPACE;
+        break;
+    default:
+        /* fail safe test */
+        if (src_color != dst_color)
+            loss |= FF_LOSS_COLORSPACE;
+        break;
+    }
+    if(loss & FF_LOSS_COLORSPACE)
+        score -= (nb_components * 65536) >> FFMIN(dst_desc->comp[0].depth_minus1, src_desc->comp[0].depth_minus1);
+
+    if (dst_color == FF_COLOR_GRAY &&
+        src_color != FF_COLOR_GRAY && (consider & FF_LOSS_CHROMA)) {
+        loss |= FF_LOSS_CHROMA;
+        score -= 2 * 65536;
+    }
+    if (!pixdesc_has_alpha(dst_desc) && (pixdesc_has_alpha(src_desc) && (consider & FF_LOSS_ALPHA))) {
+        loss |= FF_LOSS_ALPHA;
+        score -= 65536;
+    }
+    if (dst_pix_fmt == AV_PIX_FMT_PAL8 && (consider & FF_LOSS_COLORQUANT) &&
+        (src_pix_fmt != AV_PIX_FMT_PAL8 && (src_color != FF_COLOR_GRAY || (pixdesc_has_alpha(src_desc) && (consider & FF_LOSS_ALPHA))))) {
+        loss |= FF_LOSS_COLORQUANT;
+        score -= 65536;
+    }
+
+    *lossp = loss;
+    return score;
+}
+
+int av_get_pix_fmt_loss(enum AVPixelFormat dst_pix_fmt,
+                            enum AVPixelFormat src_pix_fmt,
+                            int has_alpha)
+{
+    int loss;
+    int ret = get_pix_fmt_score(dst_pix_fmt, src_pix_fmt, &loss, has_alpha ? ~0 : ~FF_LOSS_ALPHA);
+    if (ret < 0)
+        return ret;
+    return loss;
+}
+
+enum AVPixelFormat av_find_best_pix_fmt_of_2(enum AVPixelFormat dst_pix_fmt1, enum AVPixelFormat dst_pix_fmt2,
+                                             enum AVPixelFormat src_pix_fmt, int has_alpha, int *loss_ptr)
+{
+    enum AVPixelFormat dst_pix_fmt;
+    int loss1, loss2, loss_mask;
+    const AVPixFmtDescriptor *desc1 = av_pix_fmt_desc_get(dst_pix_fmt1);
+    const AVPixFmtDescriptor *desc2 = av_pix_fmt_desc_get(dst_pix_fmt2);
+    int score1, score2;
+
+    loss_mask= loss_ptr?~*loss_ptr:~0; /* use loss mask if provided */
+    if(!has_alpha)
+        loss_mask &= ~FF_LOSS_ALPHA;
+
+    dst_pix_fmt = AV_PIX_FMT_NONE;
+    score1 = get_pix_fmt_score(dst_pix_fmt1, src_pix_fmt, &loss1, loss_mask);
+    score2 = get_pix_fmt_score(dst_pix_fmt2, src_pix_fmt, &loss2, loss_mask);
+
+    if (score1 == score2) {
+        if(av_get_padded_bits_per_pixel(desc2) != av_get_padded_bits_per_pixel(desc1)) {
+            dst_pix_fmt = av_get_padded_bits_per_pixel(desc2) < av_get_padded_bits_per_pixel(desc1) ? dst_pix_fmt2 : dst_pix_fmt1;
+        } else {
+            dst_pix_fmt = desc2->nb_components < desc1->nb_components ? dst_pix_fmt2 : dst_pix_fmt1;
+        }
+    } else {
+        dst_pix_fmt = score1 < score2 ? dst_pix_fmt2 : dst_pix_fmt1;
+    }
+
+    if (loss_ptr)
+        *loss_ptr = av_get_pix_fmt_loss(dst_pix_fmt, src_pix_fmt, has_alpha);
+    return dst_pix_fmt;
+}
+
+#ifdef TEST
+
+int main(void){
+    int i;
+    int err=0;
+    int skip = 0;
+
+    for (i=0; i<AV_PIX_FMT_NB*2; i++) {
+        const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(i);
+        if(!desc || !desc->name) {
+            skip ++;
+            continue;
+        }
+        if (skip) {
+            av_log(NULL, AV_LOG_INFO, "%3d unused pixel format values\n", skip);
+            skip = 0;
+        }
+        av_log(NULL, AV_LOG_INFO, "pix fmt %s avg_bpp:%d colortype:%d\n", desc->name, av_get_padded_bits_per_pixel(desc), get_color_type(desc));
+        if ((!(desc->flags & AV_PIX_FMT_FLAG_ALPHA)) != (desc->nb_components != 2 && desc->nb_components != 4)) {
+            av_log(NULL, AV_LOG_ERROR, "Alpha flag mismatch\n");
+            err = 1;
+        }
+    }
+    return err;
+}
+
+#endif