diff options
author | James Almer <jamrial@gmail.com> | 2023-11-25 10:56:36 -0300 |
---|---|---|
committer | James Almer <jamrial@gmail.com> | 2023-12-18 15:07:11 -0300 |
commit | d2af93bbefcce7aae1f891023699bf976b8d0c18 (patch) | |
tree | 71f25c8206fac7f1e862739fb5383473830ca786 | |
parent | be8a4f80b97222d99b4262c9230ca8a1db28973a (diff) | |
download | ffmpeg-d2af93bbefcce7aae1f891023699bf976b8d0c18.tar.gz |
avutil: introduce an Immersive Audio Model and Formats API
Signed-off-by: James Almer <jamrial@gmail.com>
-rw-r--r-- | doc/APIchanges | 3 | ||||
-rw-r--r-- | libavutil/Makefile | 2 | ||||
-rw-r--r-- | libavutil/iamf.c | 563 | ||||
-rw-r--r-- | libavutil/iamf.h | 625 | ||||
-rw-r--r-- | libavutil/version.h | 2 |
5 files changed, 1194 insertions, 1 deletions
diff --git a/doc/APIchanges b/doc/APIchanges index 2b0326165e..563f67e488 100644 --- a/doc/APIchanges +++ b/doc/APIchanges @@ -2,6 +2,9 @@ The last version increases of all libraries were on 2023-02-09 API changes, most recent first: +2023-12-18 - xxxxxxxxxxx - lavu 58.35.100 - iamf.h + Add a new API to support Immersive Audio Model and Formats. + 2023-12-xx - xxxxxxxxxxx - lavu 58.33.100 - imgutils.h Add av_image_fill_color(). diff --git a/libavutil/Makefile b/libavutil/Makefile index 4711f8cde8..62cc1a1831 100644 --- a/libavutil/Makefile +++ b/libavutil/Makefile @@ -51,6 +51,7 @@ HEADERS = adler32.h \ hwcontext_videotoolbox.h \ hwcontext_vdpau.h \ hwcontext_vulkan.h \ + iamf.h \ imgutils.h \ intfloat.h \ intreadwrite.h \ @@ -140,6 +141,7 @@ OBJS = adler32.o \ hdr_dynamic_vivid_metadata.o \ hmac.o \ hwcontext.o \ + iamf.o \ imgutils.o \ integer.o \ intmath.o \ diff --git a/libavutil/iamf.c b/libavutil/iamf.c new file mode 100644 index 0000000000..62b6051049 --- /dev/null +++ b/libavutil/iamf.c @@ -0,0 +1,563 @@ +/* + * Immersive Audio Model and Formats helper functions and defines + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <limits.h> +#include <stddef.h> +#include <stdint.h> + +#include "avassert.h" +#include "error.h" +#include "iamf.h" +#include "log.h" +#include "mem.h" +#include "opt.h" + +#define IAMF_ADD_FUNC_TEMPLATE(parent_type, parent_name, child_type, child_name, suffix) \ +child_type *av_iamf_ ## parent_name ## _add_ ## child_name(parent_type *parent_name) \ +{ \ + child_type **child_name ## suffix, *child_name; \ + \ + if (parent_name->nb_## child_name ## suffix == UINT_MAX) \ + return NULL; \ + \ + child_name ## suffix = av_realloc_array(parent_name->child_name ## suffix, \ + parent_name->nb_## child_name ## suffix + 1, \ + sizeof(*parent_name->child_name ## suffix)); \ + if (!child_name ## suffix) \ + return NULL; \ + \ + parent_name->child_name ## suffix = child_name ## suffix; \ + \ + child_name = parent_name->child_name ## suffix[parent_name->nb_## child_name ## suffix] \ + = av_mallocz(sizeof(*child_name)); \ + if (!child_name) \ + return NULL; \ + \ + child_name->av_class = &child_name ## _class; \ + av_opt_set_defaults(child_name); \ + parent_name->nb_## child_name ## suffix++; \ + \ + return child_name; \ +} + +#define FLAGS AV_OPT_FLAG_ENCODING_PARAM + +// +// Param Definition +// +#define OFFSET(x) offsetof(AVIAMFMixGain, x) +static const AVOption mix_gain_options[] = { + { "subblock_duration", "set subblock_duration", OFFSET(subblock_duration), AV_OPT_TYPE_INT64, {.i64 = 1 }, 1, UINT_MAX, FLAGS }, + { "animation_type", "set animation_type", OFFSET(animation_type), AV_OPT_TYPE_INT, {.i64 = 0 }, 0, 2, FLAGS }, + { "start_point_value", "set start_point_value", OFFSET(animation_type), AV_OPT_TYPE_RATIONAL, {.dbl = 0 }, -128.0, 128.0, FLAGS }, + { "end_point_value", "set end_point_value", OFFSET(animation_type), AV_OPT_TYPE_RATIONAL, {.dbl = 0 }, -128.0, 128.0, FLAGS }, + { "control_point_value", "set control_point_value", OFFSET(animation_type), AV_OPT_TYPE_RATIONAL, {.dbl = 0 }, -128.0, 128.0, FLAGS }, + { "control_point_relative_time", "set control_point_relative_time", OFFSET(animation_type), AV_OPT_TYPE_RATIONAL, {.dbl = 0 }, 0.0, 1.0, FLAGS }, + { NULL }, +}; + +static const AVClass mix_gain_class = { + .class_name = "AVIAMFSubmixElement", + .item_name = av_default_item_name, + .version = LIBAVUTIL_VERSION_INT, + .option = mix_gain_options, +}; + +#undef OFFSET +#define OFFSET(x) offsetof(AVIAMFDemixingInfo, x) +static const AVOption demixing_info_options[] = { + { "subblock_duration", "set subblock_duration", OFFSET(subblock_duration), AV_OPT_TYPE_INT64, {.i64 = 1 }, 1, UINT_MAX, FLAGS }, + { "dmixp_mode", "set dmixp_mode", OFFSET(dmixp_mode), AV_OPT_TYPE_INT, {.i64 = 0 }, 0, 6, FLAGS }, + { NULL }, +}; + +static const AVClass demixing_info_class = { + .class_name = "AVIAMFDemixingInfo", + .item_name = av_default_item_name, + .version = LIBAVUTIL_VERSION_INT, + .option = demixing_info_options, +}; + +#undef OFFSET +#define OFFSET(x) offsetof(AVIAMFReconGain, x) +static const AVOption recon_gain_options[] = { + { "subblock_duration", "set subblock_duration", OFFSET(subblock_duration), AV_OPT_TYPE_INT64, {.i64 = 1 }, 1, UINT_MAX, FLAGS }, + { NULL }, +}; + +static const AVClass recon_gain_class = { + .class_name = "AVIAMFReconGain", + .item_name = av_default_item_name, + .version = LIBAVUTIL_VERSION_INT, + .option = recon_gain_options, +}; + +#undef OFFSET +#define OFFSET(x) offsetof(AVIAMFParamDefinition, x) +static const AVOption param_definition_options[] = { + { "parameter_id", "set parameter_id", OFFSET(parameter_id), AV_OPT_TYPE_INT64, {.i64 = 0 }, 0, UINT_MAX, FLAGS }, + { "parameter_rate", "set parameter_rate", OFFSET(parameter_rate), AV_OPT_TYPE_INT64, {.i64 = 0 }, 0, UINT_MAX, FLAGS }, + { "duration", "set duration", OFFSET(duration), AV_OPT_TYPE_INT64, {.i64 = 0 }, 0, UINT_MAX, FLAGS }, + { "constant_subblock_duration", "set constant_subblock_duration", OFFSET(constant_subblock_duration), AV_OPT_TYPE_INT64, {.i64 = 0 }, 0, UINT_MAX, FLAGS }, + { NULL }, +}; + +static const AVClass *param_definition_child_iterate(void **opaque) +{ + uintptr_t i = (uintptr_t)*opaque; + const AVClass *ret = NULL; + + switch(i) { + case AV_IAMF_PARAMETER_DEFINITION_MIX_GAIN: + ret = &mix_gain_class; + break; + case AV_IAMF_PARAMETER_DEFINITION_DEMIXING: + ret = &demixing_info_class; + break; + case AV_IAMF_PARAMETER_DEFINITION_RECON_GAIN: + ret = &recon_gain_class; + break; + default: + break; + } + + if (ret) + *opaque = (void*)(i + 1); + return ret; +} + +static const AVClass param_definition_class = { + .class_name = "AVIAMFParamDefinition", + .item_name = av_default_item_name, + .version = LIBAVUTIL_VERSION_INT, + .option = param_definition_options, + .child_class_iterate = param_definition_child_iterate, +}; + +const AVClass *av_iamf_param_definition_get_class(void) +{ + return ¶m_definition_class; +} + +AVIAMFParamDefinition *av_iamf_param_definition_alloc(enum AVIAMFParamDefinitionType type, + unsigned int nb_subblocks, size_t *out_size) +{ + + struct MixGainStruct { + AVIAMFParamDefinition p; + AVIAMFMixGain m; + }; + struct DemixStruct { + AVIAMFParamDefinition p; + AVIAMFDemixingInfo d; + }; + struct ReconGainStruct { + AVIAMFParamDefinition p; + AVIAMFReconGain r; + }; + size_t subblocks_offset, subblock_size; + size_t size; + AVIAMFParamDefinition *par; + + switch (type) { + case AV_IAMF_PARAMETER_DEFINITION_MIX_GAIN: + subblocks_offset = offsetof(struct MixGainStruct, m); + subblock_size = sizeof(AVIAMFMixGain); + break; + case AV_IAMF_PARAMETER_DEFINITION_DEMIXING: + subblocks_offset = offsetof(struct DemixStruct, d); + subblock_size = sizeof(AVIAMFDemixingInfo); + break; + case AV_IAMF_PARAMETER_DEFINITION_RECON_GAIN: + subblocks_offset = offsetof(struct ReconGainStruct, r); + subblock_size = sizeof(AVIAMFReconGain); + break; + default: + return NULL; + } + + size = subblocks_offset; + if (nb_subblocks > (SIZE_MAX - size) / subblock_size) + return NULL; + size += subblock_size * nb_subblocks; + + par = av_mallocz(size); + if (!par) + return NULL; + + par->av_class = ¶m_definition_class; + av_opt_set_defaults(par); + + par->type = type; + par->nb_subblocks = nb_subblocks; + par->subblock_size = subblock_size; + par->subblocks_offset = subblocks_offset; + + for (int i = 0; i < nb_subblocks; i++) { + void *subblock = av_iamf_param_definition_get_subblock(par, i); + + switch (type) { + case AV_IAMF_PARAMETER_DEFINITION_MIX_GAIN: + ((AVIAMFMixGain *)subblock)->av_class = &mix_gain_class; + break; + case AV_IAMF_PARAMETER_DEFINITION_DEMIXING: + ((AVIAMFDemixingInfo *)subblock)->av_class = &demixing_info_class; + break; + case AV_IAMF_PARAMETER_DEFINITION_RECON_GAIN: + ((AVIAMFReconGain *)subblock)->av_class = &recon_gain_class; + break; + default: + av_assert0(0); + } + + av_opt_set_defaults(subblock); + } + + if (out_size) + *out_size = size; + + return par; +} + +// +// Audio Element +// +#undef OFFSET +#define OFFSET(x) offsetof(AVIAMFLayer, x) +static const AVOption layer_options[] = { + { "ch_layout", "set ch_layout", OFFSET(ch_layout), AV_OPT_TYPE_CHLAYOUT, {.str = NULL }, 0, 0, FLAGS }, + { "flags", "set flags", OFFSET(flags), AV_OPT_TYPE_FLAGS, + {.i64 = 0 }, 0, AV_IAMF_LAYER_FLAG_RECON_GAIN, FLAGS, "flags" }, + {"recon_gain", "Recon gain is present", 0, AV_OPT_TYPE_CONST, + {.i64 = AV_IAMF_LAYER_FLAG_RECON_GAIN }, INT_MIN, INT_MAX, FLAGS, "flags"}, + { "output_gain_flags", "set output_gain_flags", OFFSET(output_gain_flags), AV_OPT_TYPE_FLAGS, + {.i64 = 0 }, 0, (1 << 6) - 1, FLAGS, "output_gain_flags" }, + {"FL", "Left channel", 0, AV_OPT_TYPE_CONST, + {.i64 = 1 << 5 }, INT_MIN, INT_MAX, FLAGS, "output_gain_flags"}, + {"FR", "Right channel", 0, AV_OPT_TYPE_CONST, + {.i64 = 1 << 4 }, INT_MIN, INT_MAX, FLAGS, "output_gain_flags"}, + {"BL", "Left surround channel", 0, AV_OPT_TYPE_CONST, + {.i64 = 1 << 3 }, INT_MIN, INT_MAX, FLAGS, "output_gain_flags"}, + {"BR", "Right surround channel", 0, AV_OPT_TYPE_CONST, + {.i64 = 1 << 2 }, INT_MIN, INT_MAX, FLAGS, "output_gain_flags"}, + {"TFL", "Left top front channel", 0, AV_OPT_TYPE_CONST, + {.i64 = 1 << 1 }, INT_MIN, INT_MAX, FLAGS, "output_gain_flags"}, + {"TFR", "Right top front channel", 0, AV_OPT_TYPE_CONST, + {.i64 = 1 << 0 }, INT_MIN, INT_MAX, FLAGS, "output_gain_flags"}, + { "output_gain", "set output_gain", OFFSET(output_gain), AV_OPT_TYPE_RATIONAL, { .dbl = 0 }, -128.0, 128.0, FLAGS }, + { "ambisonics_mode", "set ambisonics_mode", OFFSET(ambisonics_mode), AV_OPT_TYPE_INT, + { .i64 = AV_IAMF_AMBISONICS_MODE_MONO }, + AV_IAMF_AMBISONICS_MODE_MONO, AV_IAMF_AMBISONICS_MODE_PROJECTION, FLAGS, "ambisonics_mode" }, + { "mono", NULL, 0, AV_OPT_TYPE_CONST, + { .i64 = AV_IAMF_AMBISONICS_MODE_MONO }, .unit = "ambisonics_mode" }, + { "projection", NULL, 0, AV_OPT_TYPE_CONST, + { .i64 = AV_IAMF_AMBISONICS_MODE_PROJECTION }, .unit = "ambisonics_mode" }, + { NULL }, +}; + +static const AVClass layer_class = { + .class_name = "AVIAMFLayer", + .item_name = av_default_item_name, + .version = LIBAVUTIL_VERSION_INT, + .option = layer_options, +}; + +#undef OFFSET +#define OFFSET(x) offsetof(AVIAMFAudioElement, x) +static const AVOption audio_element_options[] = { + { "audio_element_type", "set audio_element_type", OFFSET(audio_element_type), AV_OPT_TYPE_INT, + {.i64 = AV_IAMF_AUDIO_ELEMENT_TYPE_CHANNEL }, + AV_IAMF_AUDIO_ELEMENT_TYPE_CHANNEL, AV_IAMF_AUDIO_ELEMENT_TYPE_SCENE, FLAGS, "audio_element_type" }, + { "channel", NULL, 0, AV_OPT_TYPE_CONST, + { .i64 = AV_IAMF_AUDIO_ELEMENT_TYPE_CHANNEL }, .unit = "audio_element_type" }, + { "scene", NULL, 0, AV_OPT_TYPE_CONST, + { .i64 = AV_IAMF_AUDIO_ELEMENT_TYPE_SCENE }, .unit = "audio_element_type" }, + { "default_w", "set default_w", OFFSET(default_w), AV_OPT_TYPE_INT, {.i64 = 0 }, 0, 10, FLAGS }, + { NULL }, +}; + +static const AVClass *audio_element_child_iterate(void **opaque) +{ + uintptr_t i = (uintptr_t)*opaque; + const AVClass *ret = NULL; + + if (i) + ret = &layer_class; + + if (ret) + *opaque = (void*)(i + 1); + return ret; +} + +static const AVClass audio_element_class = { + .class_name = "AVIAMFAudioElement", + .item_name = av_default_item_name, + .version = LIBAVUTIL_VERSION_INT, + .option = audio_element_options, + .child_class_iterate = audio_element_child_iterate, +}; + +const AVClass *av_iamf_audio_element_get_class(void) +{ + return &audio_element_class; +} + +AVIAMFAudioElement *av_iamf_audio_element_alloc(void) +{ + AVIAMFAudioElement *audio_element = av_mallocz(sizeof(*audio_element)); + + if (audio_element) { + audio_element->av_class = &audio_element_class; + av_opt_set_defaults(audio_element); + } + + return audio_element; +} + +IAMF_ADD_FUNC_TEMPLATE(AVIAMFAudioElement, audio_element, AVIAMFLayer, layer, s) + +void av_iamf_audio_element_free(AVIAMFAudioElement **paudio_element) +{ + AVIAMFAudioElement *audio_element = *paudio_element; + + if (!audio_element) + return; + + for (int i = 0; i < audio_element->nb_layers; i++) { + AVIAMFLayer *layer = audio_element->layers[i]; + av_opt_free(layer); + av_free(layer->demixing_matrix); + av_free(layer); + } + av_free(audio_element->layers); + + av_free(audio_element->demixing_info); + av_free(audio_element->recon_gain_info); + av_freep(paudio_element); +} + +// +// Mix Presentation +// +#undef OFFSET +#define OFFSET(x) offsetof(AVIAMFSubmixElement, x) +static const AVOption submix_element_options[] = { + { "headphones_rendering_mode", "Headphones rendering mode", OFFSET(headphones_rendering_mode), AV_OPT_TYPE_INT, + { .i64 = AV_IAMF_HEADPHONES_MODE_STEREO }, + AV_IAMF_HEADPHONES_MODE_STEREO, AV_IAMF_HEADPHONES_MODE_BINAURAL, FLAGS, "headphones_rendering_mode" }, + { "stereo", NULL, 0, AV_OPT_TYPE_CONST, + { .i64 = AV_IAMF_HEADPHONES_MODE_STEREO }, .unit = "headphones_rendering_mode" }, + { "binaural", NULL, 0, AV_OPT_TYPE_CONST, + { .i64 = AV_IAMF_HEADPHONES_MODE_BINAURAL }, .unit = "headphones_rendering_mode" }, + { "default_mix_gain", "Default mix gain", OFFSET(default_mix_gain), AV_OPT_TYPE_RATIONAL, { .dbl = 0 }, -128.0, 128.0, FLAGS }, + { "annotations", "Annotations", OFFSET(annotations), AV_OPT_TYPE_DICT, { .str = NULL }, 0, 0, FLAGS }, + { NULL }, +}; + +static void *submix_element_child_next(void *obj, void *prev) +{ + AVIAMFSubmixElement *submix_element = obj; + if (!prev) + return submix_element->element_mix_config; + + return NULL; +} + +static const AVClass *submix_element_child_iterate(void **opaque) +{ + uintptr_t i = (uintptr_t)*opaque; + const AVClass *ret = NULL; + + if (i) + ret = ¶m_definition_class; + + if (ret) + *opaque = (void*)(i + 1); + return ret; +} + +static const AVClass element_class = { + .class_name = "AVIAMFSubmixElement", + .item_name = av_default_item_name, + .version = LIBAVUTIL_VERSION_INT, + .option = submix_element_options, + .child_next = submix_element_child_next, + .child_class_iterate = submix_element_child_iterate, +}; + +IAMF_ADD_FUNC_TEMPLATE(AVIAMFSubmix, submix, AVIAMFSubmixElement, element, s) + +#undef OFFSET +#define OFFSET(x) offsetof(AVIAMFSubmixLayout, x) +static const AVOption submix_layout_options[] = { + { "layout_type", "Layout type", OFFSET(layout_type), AV_OPT_TYPE_INT, + { .i64 = AV_IAMF_SUBMIX_LAYOUT_TYPE_LOUDSPEAKERS }, + AV_IAMF_SUBMIX_LAYOUT_TYPE_LOUDSPEAKERS, AV_IAMF_SUBMIX_LAYOUT_TYPE_BINAURAL, FLAGS, "layout_type" }, + { "loudspeakers", NULL, 0, AV_OPT_TYPE_CONST, + { .i64 = AV_IAMF_SUBMIX_LAYOUT_TYPE_LOUDSPEAKERS }, .unit = "layout_type" }, + { "binaural", NULL, 0, AV_OPT_TYPE_CONST, + { .i64 = AV_IAMF_SUBMIX_LAYOUT_TYPE_BINAURAL }, .unit = "layout_type" }, + { "sound_system", "Sound System", OFFSET(sound_system), AV_OPT_TYPE_CHLAYOUT, { .str = NULL }, 0, 0, FLAGS }, + { "integrated_loudness", "Integrated loudness", OFFSET(integrated_loudness), AV_OPT_TYPE_RATIONAL, { .dbl = 0 }, -128.0, 128.0, FLAGS }, + { "digital_peak", "Digital peak", OFFSET(digital_peak), AV_OPT_TYPE_RATIONAL, { .dbl = 0 }, -128.0, 128.0, FLAGS }, + { "true_peak", "True peak", OFFSET(true_peak), AV_OPT_TYPE_RATIONAL, { .dbl = 0 }, -128.0, 128.0, FLAGS }, + { "dialog_anchored_loudness", "Anchored loudness (Dialog)", OFFSET(dialogue_anchored_loudness), AV_OPT_TYPE_RATIONAL, { .dbl = 0 }, -128.0, 128.0, FLAGS }, + { "album_anchored_loudness", "Anchored loudness (Album)", OFFSET(album_anchored_loudness), AV_OPT_TYPE_RATIONAL, { .dbl = 0 }, -128.0, 128.0, FLAGS }, + { NULL }, +}; + +static const AVClass layout_class = { + .class_name = "AVIAMFSubmixLayout", + .item_name = av_default_item_name, + .version = LIBAVUTIL_VERSION_INT, + .option = submix_layout_options, +}; + +IAMF_ADD_FUNC_TEMPLATE(AVIAMFSubmix, submix, AVIAMFSubmixLayout, layout, s) + +#undef OFFSET +#define OFFSET(x) offsetof(AVIAMFSubmix, x) +static const AVOption submix_presentation_options[] = { + { "default_mix_gain", "Default mix gain", OFFSET(default_mix_gain), AV_OPT_TYPE_RATIONAL, { .dbl = 0 }, -128.0, 128.0, FLAGS }, + { NULL }, +}; + +static void *submix_presentation_child_next(void *obj, void *prev) +{ + AVIAMFSubmix *sub_mix = obj; + if (!prev) + return sub_mix->output_mix_config; + + return NULL; +} + +static const AVClass *submix_presentation_child_iterate(void **opaque) +{ + uintptr_t i = (uintptr_t)*opaque; + const AVClass *ret = NULL; + + switch(i) { + case 0: + ret = &element_class; + break; + case 1: + ret = &layout_class; + break; + case 2: + ret = ¶m_definition_class; + break; + default: + break; + } + + if (ret) + *opaque = (void*)(i + 1); + return ret; +} + +static const AVClass submix_class = { + .class_name = "AVIAMFSubmix", + .item_name = av_default_item_name, + .version = LIBAVUTIL_VERSION_INT, + .option = submix_presentation_options, + .child_next = submix_presentation_child_next, + .child_class_iterate = submix_presentation_child_iterate, +}; + +#undef OFFSET +#define OFFSET(x) offsetof(AVIAMFMixPresentation, x) +static const AVOption mix_presentation_options[] = { + { "annotations", "set annotations", OFFSET(annotations), AV_OPT_TYPE_DICT, {.str = NULL }, 0, 0, FLAGS }, + { NULL }, +}; + +#undef OFFSET +#undef FLAGS + +static const AVClass *mix_presentation_child_iterate(void **opaque) +{ + uintptr_t i = (uintptr_t)*opaque; + const AVClass *ret = NULL; + + if (i) + ret = &submix_class; + + if (ret) + *opaque = (void*)(i + 1); + return ret; +} + +static const AVClass mix_presentation_class = { + .class_name = "AVIAMFMixPresentation", + .item_name = av_default_item_name, + .version = LIBAVUTIL_VERSION_INT, + .option = mix_presentation_options, + .child_class_iterate = mix_presentation_child_iterate, +}; + +const AVClass *av_iamf_mix_presentation_get_class(void) +{ + return &mix_presentation_class; +} + +AVIAMFMixPresentation *av_iamf_mix_presentation_alloc(void) +{ + AVIAMFMixPresentation *mix_presentation = av_mallocz(sizeof(*mix_presentation)); + + if (mix_presentation) { + mix_presentation->av_class = &mix_presentation_class; + av_opt_set_defaults(mix_presentation); + } + + return mix_presentation; +} + +IAMF_ADD_FUNC_TEMPLATE(AVIAMFMixPresentation, mix_presentation, AVIAMFSubmix, submix, es) + +void av_iamf_mix_presentation_free(AVIAMFMixPresentation **pmix_presentation) +{ + AVIAMFMixPresentation *mix_presentation = *pmix_presentation; + + if (!mix_presentation) + return; + + for (int i = 0; i < mix_presentation->nb_submixes; i++) { + AVIAMFSubmix *sub_mix = mix_presentation->submixes[i]; + for (int j = 0; j < sub_mix->nb_elements; j++) { + AVIAMFSubmixElement *submix_element = sub_mix->elements[j]; + av_opt_free(submix_element); + av_free(submix_element->element_mix_config); + av_free(submix_element); + } + av_free(sub_mix->elements); + for (int j = 0; j < sub_mix->nb_layouts; j++) { + AVIAMFSubmixLayout *submix_layout = sub_mix->layouts[j]; + av_opt_free(submix_layout); + av_free(submix_layout); + } + av_free(sub_mix->layouts); + av_free(sub_mix->output_mix_config); + av_free(sub_mix); + } + av_opt_free(mix_presentation); + av_free(mix_presentation->submixes); + + av_freep(pmix_presentation); +} diff --git a/libavutil/iamf.h b/libavutil/iamf.h new file mode 100644 index 0000000000..e03c018169 --- /dev/null +++ b/libavutil/iamf.h @@ -0,0 +1,625 @@ +/* + * Immersive Audio Model and Formats helper functions and defines + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVUTIL_IAMF_H +#define AVUTIL_IAMF_H + +/** + * @file + * Immersive Audio Model and Formats API header + * @see <a href="https://aomediacodec.github.io/iamf/">Immersive Audio Model and Formats</a> + */ + +#include <stdint.h> +#include <stddef.h> + +#include "attributes.h" +#include "avassert.h" +#include "channel_layout.h" +#include "dict.h" +#include "rational.h" + +/** + * @defgroup lavf_iamf_params Parameter Definition + * @{ + * Parameters as defined in section 3.6.1 and 3.8 of IAMF. + * @} + * @defgroup lavf_iamf_audio Audio Element + * @{ + * Audio Elements as defined in section 3.6 of IAMF. + * @} + * @defgroup lavf_iamf_mix Mix Presentation + * @{ + * Mix Presentations as defined in section 3.7 of IAMF. + * @} + * + * @} + * @addtogroup lavf_iamf_params + * @{ + */ +enum AVIAMFAnimationType { + AV_IAMF_ANIMATION_TYPE_STEP, + AV_IAMF_ANIMATION_TYPE_LINEAR, + AV_IAMF_ANIMATION_TYPE_BEZIER, +}; + +/** + * Mix Gain Parameter Data as defined in section 3.8.1 of IAMF. + */ +typedef struct AVIAMFMixGain { + const AVClass *av_class; + + /** + * Duration for the given subblock, in units of + * 1 / @ref AVIAMFParamDefinition.parameter_rate "parameter_rate". + * It must not be 0. + */ + unsigned int subblock_duration; + /** + * The type of animation applied to the parameter values. + */ + enum AVIAMFAnimationType animation_type; + /** + * Parameter value that is applied at the start of the subblock. + * Applies to all defined Animation Types. + * + * Valid range of values is -128.0 to 128.0 + */ + AVRational start_point_value; + /** + * Parameter value that is applied at the end of the subblock. + * Applies only to AV_IAMF_ANIMATION_TYPE_LINEAR and + * AV_IAMF_ANIMATION_TYPE_BEZIER Animation Types. + * + * Valid range of values is -128.0 to 128.0 + */ + AVRational end_point_value; + /** + * Parameter value of the middle control point of a quadratic Bezier + * curve, i.e., its y-axis value. + * Applies only to AV_IAMF_ANIMATION_TYPE_BEZIER Animation Type. + * + * Valid range of values is -128.0 to 128.0 + */ + AVRational control_point_value; + /** + * Parameter value of the time of the middle control point of a + * quadratic Bezier curve, i.e., its x-axis value. + * Applies only to AV_IAMF_ANIMATION_TYPE_BEZIER Animation Type. + * + * Valid range of values is 0.0 to 1.0 + */ + AVRational control_point_relative_time; +} AVIAMFMixGain; + +/** + * Demixing Info Parameter Data as defined in section 3.8.2 of IAMF. + */ +typedef struct AVIAMFDemixingInfo { + const AVClass *av_class; + + /** + * Duration for the given subblock, in units of + * 1 / @ref AVIAMFParamDefinition.parameter_rate "parameter_rate". + * It must not be 0. + */ + unsigned int subblock_duration; + /** + * Pre-defined combination of demixing parameters. + */ + unsigned int dmixp_mode; +} AVIAMFDemixingInfo; + +/** + * Recon Gain Info Parameter Data as defined in section 3.8.3 of IAMF. + */ +typedef struct AVIAMFReconGain { + const AVClass *av_class; + + /** + * Duration for the given subblock, in units of + * 1 / @ref AVIAMFParamDefinition.parameter_rate "parameter_rate". + * It must not be 0. + */ + unsigned int subblock_duration; + + /** + * Array of gain values to be applied to each channel for each layer + * defined in the Audio Element referencing the parent Parameter Definition. + * Values for layers where the AV_IAMF_LAYER_FLAG_RECON_GAIN flag is not set + * are undefined. + * + * Channel order is: FL, C, FR, SL, SR, TFL, TFR, BL, BR, TBL, TBR, LFE + */ + uint8_t recon_gain[6][12]; +} AVIAMFReconGain; + +enum AVIAMFParamDefinitionType { + /** + * Subblocks are of struct type AVIAMFMixGain + */ + AV_IAMF_PARAMETER_DEFINITION_MIX_GAIN, + /** + * Subblocks are of struct type AVIAMFDemixingInfo + */ + AV_IAMF_PARAMETER_DEFINITION_DEMIXING, + /** + * Subblocks are of struct type AVIAMFReconGain + */ + AV_IAMF_PARAMETER_DEFINITION_RECON_GAIN, +}; + +/** + * Parameters as defined in section 3.6.1 of IAMF. + * + * The struct is allocated by av_iamf_param_definition_alloc() along with an + * array of subblocks, its type depending on the value of type. + * This array is placed subblocks_offset bytes after the start of this struct. + */ +typedef struct AVIAMFParamDefinition { + const AVClass *av_class; + + /** + * Offset in bytes from the start of this struct, at which the subblocks + * array is located. + */ + size_t subblocks_offset; + /** + * Size in bytes of each element in the subblocks array. + */ + size_t subblock_size; + /** + * Number of subblocks in the array. + */ + unsigned int nb_subblocks; + + /** + * Parameters type. Determines the type of the subblock elements. + */ + enum AVIAMFParamDefinitionType type; + + /** + * Identifier for the paremeter substream. + */ + unsigned int parameter_id; + /** + * Sample rate for the paremeter substream. It must not be 0. + */ + unsigned int parameter_rate; + + /** + * The accumulated duration of all blocks in this parameter definition, + * in units of 1 / @ref parameter_rate. + * + * May be 0, in which case all duration values should be specified in + * another parameter definition referencing the same parameter_id. + */ + unsigned int duration; + /** + * The duration of every subblock in the case where all subblocks, with + * the optional exception of the last subblock, have equal durations. + * + * Must be 0 if subblocks have different durations. + */ + unsigned int constant_subblock_duration; +} AVIAMFParamDefinition; + +const AVClass *av_iamf_param_definition_get_class(void); + +/** + * Allocates memory for AVIAMFParamDefinition, plus an array of {@code nb_subblocks} + * amount of subblocks of the given type and initializes the variables. Can be + * freed with a normal av_free() call. + * + * @param size if non-NULL, the size in bytes of the resulting data array is written here. + */ +AVIAMFParamDefinition *av_iamf_param_definition_alloc(enum AVIAMFParamDefinitionType type, + unsigned int nb_subblocks, size_t *size); + +/** + * Get the subblock at the specified {@code idx}. Must be between 0 and nb_subblocks - 1. + * + * The @ref AVIAMFParamDefinition.type "param definition type" defines + * the struct type of the returned pointer. + */ +static av_always_inline void* +av_iamf_param_definition_get_subblock(const AVIAMFParamDefinition *par, unsigned int idx) +{ + av_assert0(idx < par->nb_subblocks); + return (void *)((uint8_t *)par + par->subblocks_offset + idx * par->subblock_size); +} + +/** + * @} + * @addtogroup lavf_iamf_audio + * @{ + */ + +enum AVIAMFAmbisonicsMode { + AV_IAMF_AMBISONICS_MODE_MONO, + AV_IAMF_AMBISONICS_MODE_PROJECTION, +}; + +/** + * Recon gain information for the layer is present in AVIAMFReconGain + */ +#define AV_IAMF_LAYER_FLAG_RECON_GAIN (1 << 0) + +/** + * A layer defining a Channel Layout in the Audio Element. + * + * When @ref AVIAMFAudioElement.audio_element_type "the parent's Audio Element type" + * is AV_IAMF_AUDIO_ELEMENT_TYPE_CHANNEL, this corresponds to an Scalable Channel + * Layout layer as defined in section 3.6.2 of IAMF. + * For AV_IAMF_AUDIO_ELEMENT_TYPE_SCENE, it is an Ambisonics channel + * layout as defined in section 3.6.3 of IAMF. + */ +typedef struct AVIAMFLayer { + const AVClass *av_class; + + AVChannelLayout ch_layout; + + /** + * A bitmask which may contain a combination of AV_IAMF_LAYER_FLAG_* flags. + */ + unsigned int flags; + /** + * Output gain channel flags as defined in section 3.6.2 of IAMF. + * + * This field is defined only if @ref AVIAMFAudioElement.audio_element_type + * "the parent's Audio Element type" is AV_IAMF_AUDIO_ELEMENT_TYPE_CHANNEL, + * must be 0 otherwise. + */ + unsigned int output_gain_flags; + /** + * Output gain as defined in section 3.6.2 of IAMF. + * + * Must be 0 if @ref output_gain_flags is 0. + */ + AVRational output_gain; + /** + * Ambisonics mode as defined in section 3.6.3 of IAMF. + * + * This field is defined only if @ref AVIAMFAudioElement.audio_element_type + * "the parent's Audio Element type" is AV_IAMF_AUDIO_ELEMENT_TYPE_SCENE. + * + * If AV_IAMF_AMBISONICS_MODE_MONO, channel_mapping is defined implicitly + * (Ambisonic Order) or explicitly (Custom Order with ambi channels) in + * @ref ch_layout. + * If AV_IAMF_AMBISONICS_MODE_PROJECTION, @ref demixing_matrix must be set. + */ + enum AVIAMFAmbisonicsMode ambisonics_mode; + + /** + * Demixing matrix as defined in section 3.6.3 of IAMF. + * + * The length of the array is ch_layout.nb_channels multiplied by the sum of + * the amount of streams in the group plus the amount of streams in the group + * that are stereo. + * + * May be set only if @ref ambisonics_mode == AV_IAMF_AMBISONICS_MODE_PROJECTION, + * must be NULL otherwise. + */ + AVRational *demixing_matrix; +} AVIAMFLayer; + + +enum AVIAMFAudioElementType { + AV_IAMF_AUDIO_ELEMENT_TYPE_CHANNEL, + AV_IAMF_AUDIO_ELEMENT_TYPE_SCENE, +}; + +typedef struct AVIAMFAudioElement { + const AVClass *av_class; + + AVIAMFLayer **layers; + /** + * Number of layers, or channel groups, in the Audio Element. + * There may be 6 layers at most, and for @ref audio_element_type + * AV_IAMF_AUDIO_ELEMENT_TYPE_SCENE, there may be exactly 1. + * + * Set by av_iamf_audio_element_add_layer(), must not be + * modified by any other code. + */ + unsigned int nb_layers; + + /** + * Demixing information used to reconstruct a scalable channel audio + * representation. + * The @ref AVIAMFParamDefinition.type "type" must be + * AV_IAMF_PARAMETER_DEFINITION_DEMIXING. + */ + AVIAMFParamDefinition *demixing_info; + /** + * Recon gain information used to reconstruct a scalable channel audio + * representation. + * The @ref AVIAMFParamDefinition.type "type" must be + * AV_IAMF_PARAMETER_DEFINITION_RECON_GAIN. + */ + AVIAMFParamDefinition *recon_gain_info; + + /** + * Audio element type as defined in section 3.6 of IAMF. + */ + enum AVIAMFAudioElementType audio_element_type; + + /** + * Default weight value as defined in section 3.6 of IAMF. + */ + unsigned int default_w; +} AVIAMFAudioElement; + +const AVClass *av_iamf_audio_element_get_class(void); + +/** + * Allocates a AVIAMFAudioElement, and initializes its fields with default values. + * No layers are allocated. Must be freed with av_iamf_audio_element_free(). + * + * @see av_iamf_audio_element_add_layer() + */ +AVIAMFAudioElement *av_iamf_audio_element_alloc(void); + +/** + * Allocate a layer and add it to a given AVIAMFAudioElement. + * It is freed by av_iamf_audio_element_free() alongside the rest of the parent + * AVIAMFAudioElement. + * + * @return a pointer to the allocated layer. + */ +AVIAMFLayer *av_iamf_audio_element_add_layer(AVIAMFAudioElement *audio_element); + +void av_iamf_audio_element_free(AVIAMFAudioElement **audio_element); + +/** + * @} + * @addtogroup lavf_iamf_mix + * @{ + */ + +enum AVIAMFHeadphonesMode { + /** + * The referenced Audio Element shall be rendered to stereo loudspeakers. + */ + AV_IAMF_HEADPHONES_MODE_STEREO, + /** + * The referenced Audio Element shall be rendered with a binaural renderer. + */ + AV_IAMF_HEADPHONES_MODE_BINAURAL, +}; + +typedef struct AVIAMFSubmixElement { + const AVClass *av_class; + + /** + * The id of the Audio Element this submix element references. + */ + unsigned int audio_element_id; + + /** + * Information required required for applying any processing to the + * referenced and rendered Audio Element before being summed with other + * processed Audio Elements. + * The @ref AVIAMFParamDefinition.type "type" must be + * AV_IAMF_PARAMETER_DEFINITION_MIX_GAIN. + */ + AVIAMFParamDefinition *element_mix_config; + + /** + * Default mix gain value to apply when there are no AVIAMFParamDefinition + * with @ref element_mix_config "element_mix_config's" + * @ref AVIAMFParamDefinition.parameter_id "parameter_id" available for a + * given audio frame. + */ + AVRational default_mix_gain; + + /** + * A value that indicates whether the referenced channel-based Audio Element + * shall be rendered to stereo loudspeakers or spatialized with a binaural + * renderer when played back on headphones. + * If the Audio Element is not of @ref AVIAMFAudioElement.audio_element_type + * "type" AV_IAMF_AUDIO_ELEMENT_TYPE_CHANNEL, then this field is undefined. + */ + enum AVIAMFHeadphonesMode headphones_rendering_mode; + + /** + * A dictionary of strings describing the submix in different languages. + * Must have the same amount of entries as + * @ref AVIAMFMixPresentation.annotations "the mix's annotations", stored + * in the same order, and with the same key strings. + * + * @ref AVDictionaryEntry.key "key" is a string conforming to BCP-47 that + * specifies the language for the string stored in + * @ref AVDictionaryEntry.value "value". + */ + AVDictionary *annotations; +} AVIAMFSubmixElement; + +enum AVIAMFSubmixLayoutType { + /** + * The layout follows the loudspeaker sound system convention of ITU-2051-3. + */ + AV_IAMF_SUBMIX_LAYOUT_TYPE_LOUDSPEAKERS = 2, + /** + * The layout is binaural. + */ + AV_IAMF_SUBMIX_LAYOUT_TYPE_BINAURAL = 3, +}; + +typedef struct AVIAMFSubmixLayout { + const AVClass *av_class; + + enum AVIAMFSubmixLayoutType layout_type; + + /** + * Channel layout matching one of Sound Systems A to J of ITU-2051-3, plus + * 7.1.2ch and 3.1.2ch + * If layout_type is not AV_IAMF_SUBMIX_LAYOUT_TYPE_LOUDSPEAKERS, this field + * is undefined. + */ + AVChannelLayout sound_system; + /** + * The program integrated loudness information, as defined in + * ITU-1770-4. + */ + AVRational integrated_loudness; + /** + * The digital (sampled) peak value of the audio signal, as defined + * in ITU-1770-4. + */ + AVRational digital_peak; + /** + * The true peak of the audio signal, as defined in ITU-1770-4. + */ + AVRational true_peak; + /** + * The Dialogue loudness information, as defined in ITU-1770-4. + */ + AVRational dialogue_anchored_loudness; + /** + * The Album loudness information, as defined in ITU-1770-4. + */ + AVRational album_anchored_loudness; +} AVIAMFSubmixLayout; + +typedef struct AVIAMFSubmix { + const AVClass *av_class; + + /** + * Array of submix elements. + * + * Set by av_iamf_submix_add_element(), must not be modified by any + * other code. + */ + AVIAMFSubmixElement **elements; + /** + * Number of elements in the submix. + * + * Set by av_iamf_submix_add_element(), must not be modified by any + * other code. + */ + unsigned int nb_elements; + + /** + * Array of submix layouts. + * + * Set by av_iamf_submix_add_layout(), must not be modified by any + * other code. + */ + AVIAMFSubmixLayout **layouts; + /** + * Number of layouts in the submix. + * + * Set by av_iamf_submix_add_layout(), must not be modified by any + * other code. + */ + unsigned int nb_layouts; + + /** + * Information required for post-processing the mixed audio signal to + * generate the audio signal for playback. + * The @ref AVIAMFParamDefinition.type "type" must be + * AV_IAMF_PARAMETER_DEFINITION_MIX_GAIN. + */ + AVIAMFParamDefinition *output_mix_config; + + /** + * Default mix gain value to apply when there are no AVIAMFParamDefinition + * with @ref output_mix_config "output_mix_config's" + * @ref AVIAMFParamDefinition.parameter_id "parameter_id" available for a + * given audio frame. + */ + AVRational default_mix_gain; +} AVIAMFSubmix; + +typedef struct AVIAMFMixPresentation { + const AVClass *av_class; + + /** + * Array of submixes. + * + * Set by av_iamf_mix_presentation_add_submix(), must not be modified + * by any other code. + */ + AVIAMFSubmix **submixes; + /** + * Number of submixes in the presentation. + * + * Set by av_iamf_mix_presentation_add_submix(), must not be modified + * by any other code. + */ + unsigned int nb_submixes; + + /** + * A dictionary of strings describing the mix in different languages. + * Must have the same amount of entries as every + * @ref AVIAMFSubmixElement.annotations "Submix element annotations", + * stored in the same order, and with the same key strings. + * + * @ref AVDictionaryEntry.key "key" is a string conforming to BCP-47 + * that specifies the language for the string stored in + * @ref AVDictionaryEntry.value "value". + */ + AVDictionary *annotations; +} AVIAMFMixPresentation; + +const AVClass *av_iamf_mix_presentation_get_class(void); + +/** + * Allocates a AVIAMFMixPresentation, and initializes its fields with default + * values. No submixes are allocated. + * Must be freed with av_iamf_mix_presentation_free(). + * + * @see av_iamf_mix_presentation_add_submix() + */ +AVIAMFMixPresentation *av_iamf_mix_presentation_alloc(void); + +/** + * Allocate a submix and add it to a given AVIAMFMixPresentation. + * It is freed by av_iamf_mix_presentation_free() alongside the rest of the + * parent AVIAMFMixPresentation. + * + * @return a pointer to the allocated submix. + */ +AVIAMFSubmix *av_iamf_mix_presentation_add_submix(AVIAMFMixPresentation *mix_presentation); + +/** + * Allocate a submix element and add it to a given AVIAMFSubmix. + * It is freed by av_iamf_mix_presentation_free() alongside the rest of the + * parent AVIAMFSubmix. + * + * @return a pointer to the allocated submix. + */ +AVIAMFSubmixElement *av_iamf_submix_add_element(AVIAMFSubmix *submix); + +/** + * Allocate a submix layout and add it to a given AVIAMFSubmix. + * It is freed by av_iamf_mix_presentation_free() alongside the rest of the + * parent AVIAMFSubmix. + * + * @return a pointer to the allocated submix. + */ +AVIAMFSubmixLayout *av_iamf_submix_add_layout(AVIAMFSubmix *submix); + +void av_iamf_mix_presentation_free(AVIAMFMixPresentation **mix_presentation); +/** + * @} + */ + +#endif /* AVUTIL_IAMF_H */ diff --git a/libavutil/version.h b/libavutil/version.h index dcb39764a5..7664cd97bd 100644 --- a/libavutil/version.h +++ b/libavutil/version.h @@ -79,7 +79,7 @@ */ #define LIBAVUTIL_VERSION_MAJOR 58 -#define LIBAVUTIL_VERSION_MINOR 34 +#define LIBAVUTIL_VERSION_MINOR 35 #define LIBAVUTIL_VERSION_MICRO 100 #define LIBAVUTIL_VERSION_INT AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \ |