aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMartin Storsjö <martin@martin.st>2023-12-08 14:08:05 +0200
committerMartin Storsjö <martin@martin.st>2023-12-19 13:34:25 +0200
commit76cd71bf0cfc0f7958d691e5d4577b8f4cca96b2 (patch)
tree8c4ad0d1205025020dfad94cfed3179efce51090
parentd0e215fffcb4cf0fb9876bf5722dacdee71580be (diff)
downloadffmpeg-76cd71bf0cfc0f7958d691e5d4577b8f4cca96b2.tar.gz
fate: Allow running multiple rounds of tests with differing settings
This can be used to run tests multple times, with e.g. differing QEMU settings, by adding something like this to the FATE configuration file: target_exec="qemu-aarch64-static" fate_targets="fate-checkasm fate-cpu" fate_environments="sve128 sve256 sve512" sve128_env="QEMU_CPU=max,sve128=on" sve256_env="QEMU_CPU=max,sve256=on" sve512_env="QEMU_CPU=max,sve512=on" It's also possible to customize the target_exec command further by injecting a sufficiently quoted variable into it, which then can be updated for each run, e.g. target_exec="\$(CUR_EXEC_CMD)". For each of the environment names in fate_environments, the tests that are run get the name suffixed on the fate tests in the test log and fate report, e.g. "fate-checkasm-h264dsp_sve128". Signed-off-by: Martin Storsjö <martin@martin.st>
-rw-r--r--doc/fate_config.sh.template20
-rw-r--r--tests/Makefile4
-rwxr-xr-xtests/fate.sh14
3 files changed, 35 insertions, 3 deletions
diff --git a/doc/fate_config.sh.template b/doc/fate_config.sh.template
index 06bb79a832..8681805d65 100644
--- a/doc/fate_config.sh.template
+++ b/doc/fate_config.sh.template
@@ -33,3 +33,23 @@ makeopts= # extra options passed to 'make'
# defaults to 'tar c'
#fate_targets= # targets to make when running fate; defaults to "fate",
# can be set to run a subset of tests, e.g. "fate-checkasm".
+
+#fate_environments= # a list of names of configurations to run tests for;
+ # each round is run with variables from ${${name}_env} set.
+
+# One example of using fate_environments:
+
+# target_exec="qemu-aarch64-static"
+# fate_targets="fate-checkasm fate-cpu"
+# fate_environments="sve128 sve256"
+# sve128_env="QEMU_CPU=max,sve128=on"
+# sve256_env="QEMU_CPU=max,sve256=on"
+
+# The variables set by fate_environments can also be used explicitly
+# by target_exec, e.g. like this:
+
+# target_exec="qemu-aarch64-static -cpu \$(MY_CPU)"
+# fate_targets="fate-checkasm fate-cpu"
+# fate_environments="sve128 sve256"
+# sve128_env="MY_CPU=max,sve128=on"
+# sve256_env="MY_CPU=max,sve256=on"
diff --git a/tests/Makefile b/tests/Makefile
index 444c09b3de..744dbcdfb3 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -305,8 +305,8 @@ $(FATE): export PROGSUF = $(PROGSSUF)
$(FATE): export EXECSUF = $(EXESUF)
$(FATE): export HOSTEXECSUF = $(HOSTEXESUF)
$(FATE): $(FATE_UTILS:%=tests/%$(HOSTEXESUF)) | $(FATE_OUTDIRS)
- @echo "TEST $(@:fate-%=%)"
- $(Q)$(SRC_PATH)/tests/fate-run.sh $@ "$(TARGET_SAMPLES)" "$(TARGET_EXEC)" "$(TARGET_PATH)" '$(CMD)' '$(CMP)' '$(REF)' '$(FUZZ)' '$(THREADS)' '$(THREAD_TYPE)' '$(CPUFLAGS)' '$(CMP_SHIFT)' '$(CMP_TARGET)' '$(SIZE_TOLERANCE)' '$(CMP_UNIT)' '$(GEN)' '$(HWACCEL)' '$(REPORT)' '$(KEEP_FILES)'
+ @echo "TEST $(@:fate-%=%)$(FATE_SUFFIX)"
+ $(Q)$(SRC_PATH)/tests/fate-run.sh $@$(FATE_SUFFIX) "$(TARGET_SAMPLES)" "$(TARGET_EXEC)" "$(TARGET_PATH)" '$(CMD)' '$(CMP)' '$(REF)' '$(FUZZ)' '$(THREADS)' '$(THREAD_TYPE)' '$(CPUFLAGS)' '$(CMP_SHIFT)' '$(CMP_TARGET)' '$(SIZE_TOLERANCE)' '$(CMP_UNIT)' '$(GEN)' '$(HWACCEL)' '$(REPORT)' '$(KEEP_FILES)'
fate-list:
@printf '%s\n' $(sort $(FATE))
diff --git a/tests/fate.sh b/tests/fate.sh
index 1a40322ae9..07908be3a5 100755
--- a/tests/fate.sh
+++ b/tests/fate.sh
@@ -75,7 +75,19 @@ compile()(
fate()(
test "$build_only" = "yes" && return
cd ${build} || return
- ${make} ${makeopts_fate-${makeopts}} -k ${fate_targets}
+ if [ -n "${fate_environments}" ]; then
+ ret=0
+ for e in ${fate_environments}; do
+ eval "curenv=\${${e}_env}"
+ echo Testing environment ${e}: ${curenv}
+ ${make} ${makeopts_fate-${makeopts}} -k ${fate_targets} FATE_SUFFIX=_${e} ${curenv}
+ cur_ret=$?
+ test $cur_ret != 0 && ret=$cur_ret
+ done
+ return $ret
+ else
+ ${make} ${makeopts_fate-${makeopts}} -k ${fate_targets}
+ fi
)
clean(){