aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMark Thompson <sw@jkqxz.net>2016-05-18 10:58:56 +0100
committerMark Thompson <sw@jkqxz.net>2016-05-27 10:56:45 +0100
commit2bfa067d0b636e7b2004fb0ad5a53d0d48c6de32 (patch)
tree67bf6b90a4d302ae890a9adda95f97b793b4a83e
parent6641819feedb086ebba3d2be89b8d33980f367e1 (diff)
downloadffmpeg-2bfa067d0b636e7b2004fb0ad5a53d0d48c6de32.tar.gz
vaapi_encode: Check config attributes before creating config
This prevents attempts to use unsupported modes, such as low-power H.264 mode on non-Skylake targets. Also fixes a crash on invalid configuration, when trying to destroy an invalid VA config/context.
-rw-r--r--libavcodec/vaapi_encode.c132
1 files changed, 130 insertions, 2 deletions
diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c
index 075c848f60..9da55c97fd 100644
--- a/libavcodec/vaapi_encode.c
+++ b/libavcodec/vaapi_encode.c
@@ -20,6 +20,7 @@
#include <string.h>
#include "libavutil/avassert.h"
+#include "libavutil/common.h"
#include "libavutil/log.h"
#include "libavutil/pixdesc.h"
@@ -887,6 +888,122 @@ fail:
return err;
}
+static av_cold int vaapi_encode_check_config(AVCodecContext *avctx)
+{
+ VAAPIEncodeContext *ctx = avctx->priv_data;
+ VAStatus vas;
+ int i, n, err;
+ VAProfile *profiles = NULL;
+ VAEntrypoint *entrypoints = NULL;
+ VAConfigAttrib attr[] = {
+ { VAConfigAttribRateControl },
+ { VAConfigAttribEncMaxRefFrames },
+ };
+
+ n = vaMaxNumProfiles(ctx->hwctx->display);
+ profiles = av_malloc_array(n, sizeof(VAProfile));
+ if (!profiles) {
+ err = AVERROR(ENOMEM);
+ goto fail;
+ }
+ vas = vaQueryConfigProfiles(ctx->hwctx->display, profiles, &n);
+ if (vas != VA_STATUS_SUCCESS) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to query profiles: %d (%s).\n",
+ vas, vaErrorStr(vas));
+ err = AVERROR(ENOSYS);
+ goto fail;
+ }
+ for (i = 0; i < n; i++) {
+ if (profiles[i] == ctx->va_profile)
+ break;
+ }
+ if (i >= n) {
+ av_log(ctx, AV_LOG_ERROR, "Encoding profile not found (%d).\n",
+ ctx->va_profile);
+ err = AVERROR(ENOSYS);
+ goto fail;
+ }
+
+ n = vaMaxNumEntrypoints(ctx->hwctx->display);
+ entrypoints = av_malloc_array(n, sizeof(VAEntrypoint));
+ if (!entrypoints) {
+ err = AVERROR(ENOMEM);
+ goto fail;
+ }
+ vas = vaQueryConfigEntrypoints(ctx->hwctx->display, ctx->va_profile,
+ entrypoints, &n);
+ if (vas != VA_STATUS_SUCCESS) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to query entrypoints for "
+ "profile %u: %d (%s).\n", ctx->va_profile,
+ vas, vaErrorStr(vas));
+ err = AVERROR(ENOSYS);
+ goto fail;
+ }
+ for (i = 0; i < n; i++) {
+ if (entrypoints[i] == ctx->va_entrypoint)
+ break;
+ }
+ if (i >= n) {
+ av_log(ctx, AV_LOG_ERROR, "Encoding entrypoint not found "
+ "(%d / %d).\n", ctx->va_profile, ctx->va_entrypoint);
+ err = AVERROR(ENOSYS);
+ goto fail;
+ }
+
+ vas = vaGetConfigAttributes(ctx->hwctx->display,
+ ctx->va_profile, ctx->va_entrypoint,
+ attr, FF_ARRAY_ELEMS(attr));
+ if (vas != VA_STATUS_SUCCESS) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to fetch config "
+ "attributes: %d (%s).\n", vas, vaErrorStr(vas));
+ return AVERROR(EINVAL);
+ }
+
+ for (i = 0; i < FF_ARRAY_ELEMS(attr); i++) {
+ if (attr[i].value == VA_ATTRIB_NOT_SUPPORTED) {
+ // Unfortunately we have to treat this as "don't know" and hope
+ // for the best, because the Intel MJPEG encoder returns this
+ // for all the interesting attributes.
+ continue;
+ }
+ switch (attr[i].type) {
+ case VAConfigAttribRateControl:
+ if (!(ctx->va_rc_mode & attr[i].value)) {
+ av_log(avctx, AV_LOG_ERROR, "Rate control mode is not "
+ "supported: %x\n", attr[i].value);
+ err = AVERROR(EINVAL);
+ goto fail;
+ }
+ break;
+ case VAConfigAttribEncMaxRefFrames:
+ {
+ unsigned int ref_l0 = attr[i].value & 0xffff;
+ unsigned int ref_l1 = (attr[i].value >> 16) & 0xffff;
+
+ if (avctx->gop_size > 1 && ref_l0 < 1) {
+ av_log(avctx, AV_LOG_ERROR, "P frames are not "
+ "supported (%x).\n", attr[i].value);
+ err = AVERROR(EINVAL);
+ goto fail;
+ }
+ if (avctx->max_b_frames > 0 && ref_l1 < 1) {
+ av_log(avctx, AV_LOG_ERROR, "B frames are not "
+ "supported (%x).\n", attr[i].value);
+ err = AVERROR(EINVAL);
+ goto fail;
+ }
+ }
+ break;
+ }
+ }
+
+ err = 0;
+fail:
+ av_freep(&profiles);
+ av_freep(&entrypoints);
+ return err;
+}
+
av_cold int ff_vaapi_encode_init(AVCodecContext *avctx,
const VAAPIEncodeType *type)
{
@@ -907,6 +1024,9 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx,
ctx->codec = type;
ctx->codec_options = ctx->codec_options_data;
+ ctx->va_config = VA_INVALID_ID;
+ ctx->va_context = VA_INVALID_ID;
+
ctx->priv_data = av_mallocz(type->priv_data_size);
if (!ctx->priv_data) {
err = AVERROR(ENOMEM);
@@ -932,6 +1052,10 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx,
if (err < 0)
goto fail;
+ err = vaapi_encode_check_config(avctx);
+ if (err < 0)
+ goto fail;
+
vas = vaCreateConfig(ctx->hwctx->display,
ctx->va_profile, ctx->va_entrypoint,
ctx->config_attributes, ctx->nb_config_attributes,
@@ -1088,11 +1212,15 @@ av_cold int ff_vaapi_encode_close(AVCodecContext *avctx)
vaapi_encode_free(avctx, pic);
}
- if (ctx->va_context != VA_INVALID_ID)
+ if (ctx->va_context != VA_INVALID_ID) {
vaDestroyContext(ctx->hwctx->display, ctx->va_context);
+ ctx->va_context = VA_INVALID_ID;
+ }
- if (ctx->va_config != VA_INVALID_ID)
+ if (ctx->va_config != VA_INVALID_ID) {
vaDestroyConfig(ctx->hwctx->display, ctx->va_config);
+ ctx->va_config = VA_INVALID_ID;
+ }
if (ctx->codec->close)
ctx->codec->close(avctx);