aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authormaxim-yurchuk <maxim-yurchuk@yandex-team.com>2025-02-11 20:36:58 +0300
committermaxim-yurchuk <maxim-yurchuk@yandex-team.com>2025-02-11 21:11:39 +0300
commit68f130528d23665ea395f02063b572479fec7765 (patch)
treec2e9d2369f274e64ea4a7bbced6992589eb89f13
parent9c9d92fa09369c85d2acc077d598f79a841b7153 (diff)
downloadydb-68f130528d23665ea395f02063b572479fec7765.tar.gz
Change "devtools/contrib/piglet/projects/ydblib/config.yaml"
В scipy лежит какой то код на фортране, гитхаб [показывает](https://github.com/ydb-platform/ydb) что ydb состоит на 1.2% из фортрана ![image.png](https://arcanum.s3.mds.yandex.net/files/maxim-yurchuk/2LEi0P30SMM3gMiOXC0ch) Во дворе дети показывают пальцем, обзывают фортранщиком commit_hash:6e17d7e2b13d628e318ba8deb5ab29e159fdf712
-rwxr-xr-xcontrib/restricted/google/benchmark/tools/compare/compare.py523
-rw-r--r--contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test1_run1.json127
-rw-r--r--contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test1_run2.json127
-rw-r--r--contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test2_run.json81
-rw-r--r--contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test3_run0.json65
-rw-r--r--contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test3_run1.json65
-rw-r--r--contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test4_run.json96
-rw-r--r--contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test4_run0.json21
-rw-r--r--contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test4_run1.json21
-rw-r--r--contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test5_run0.json18
-rw-r--r--contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test5_run1.json18
-rw-r--r--contrib/restricted/google/benchmark/tools/compare/gbench/__init__.py8
-rw-r--r--contrib/restricted/google/benchmark/tools/compare/gbench/report.py1619
-rw-r--r--contrib/restricted/google/benchmark/tools/compare/gbench/util.py231
-rw-r--r--contrib/restricted/google/benchmark/tools/compare/ya.make30
15 files changed, 0 insertions, 3050 deletions
diff --git a/contrib/restricted/google/benchmark/tools/compare/compare.py b/contrib/restricted/google/benchmark/tools/compare/compare.py
deleted file mode 100755
index 7572520cc0..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/compare.py
+++ /dev/null
@@ -1,523 +0,0 @@
-#!/usr/bin/env python3
-
-# type: ignore
-
-"""
-compare.py - versatile benchmark output compare tool
-"""
-
-import argparse
-import json
-import os
-import sys
-import unittest
-from argparse import ArgumentParser
-
-import gbench
-from gbench import report, util
-
-
-def check_inputs(in1, in2, flags):
- """
- Perform checking on the user provided inputs and diagnose any abnormalities
- """
- in1_kind, in1_err = util.classify_input_file(in1)
- in2_kind, in2_err = util.classify_input_file(in2)
- output_file = util.find_benchmark_flag("--benchmark_out=", flags)
- output_type = util.find_benchmark_flag("--benchmark_out_format=", flags)
- if (
- in1_kind == util.IT_Executable
- and in2_kind == util.IT_Executable
- and output_file
- ):
- print(
- (
- "WARNING: '--benchmark_out=%s' will be passed to both "
- "benchmarks causing it to be overwritten"
- )
- % output_file
- )
- if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON:
- # When both sides are JSON the only supported flag is
- # --benchmark_filter=
- for flag in util.remove_benchmark_flags("--benchmark_filter=", flags):
- print(
- "WARNING: passing %s has no effect since both "
- "inputs are JSON" % flag
- )
- if output_type is not None and output_type != "json":
- print(
- (
- "ERROR: passing '--benchmark_out_format=%s' to 'compare.py`"
- " is not supported."
- )
- % output_type
- )
- sys.exit(1)
-
-
-def create_parser():
- parser = ArgumentParser(
- description="versatile benchmark output compare tool"
- )
-
- parser.add_argument(
- "-a",
- "--display_aggregates_only",
- dest="display_aggregates_only",
- action="store_true",
- help="If there are repetitions, by default, we display everything - the"
- " actual runs, and the aggregates computed. Sometimes, it is "
- "desirable to only view the aggregates. E.g. when there are a lot "
- "of repetitions. Do note that only the display is affected. "
- "Internally, all the actual runs are still used, e.g. for U test.",
- )
-
- parser.add_argument(
- "--no-color",
- dest="color",
- default=True,
- action="store_false",
- help="Do not use colors in the terminal output",
- )
-
- parser.add_argument(
- "-d",
- "--dump_to_json",
- dest="dump_to_json",
- help="Additionally, dump benchmark comparison output to this file in JSON format.",
- )
-
- utest = parser.add_argument_group()
- utest.add_argument(
- "--no-utest",
- dest="utest",
- default=True,
- action="store_false",
- help="The tool can do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than {}) number of repetitions to be meaningful!\nThe test is being done by default, if at least {} repetitions were done.\nThis option can disable the U Test.".format(
- report.UTEST_OPTIMAL_REPETITIONS, report.UTEST_MIN_REPETITIONS
- ),
- )
- alpha_default = 0.05
- utest.add_argument(
- "--alpha",
- dest="utest_alpha",
- default=alpha_default,
- type=float,
- help=(
- "significance level alpha. if the calculated p-value is below this value, then the result is said to be statistically significant and the null hypothesis is rejected.\n(default: %0.4f)"
- )
- % alpha_default,
- )
-
- subparsers = parser.add_subparsers(
- help="This tool has multiple modes of operation:", dest="mode"
- )
-
- parser_a = subparsers.add_parser(
- "benchmarks",
- help="The most simple use-case, compare all the output of these two benchmarks",
- )
- baseline = parser_a.add_argument_group("baseline", "The benchmark baseline")
- baseline.add_argument(
- "test_baseline",
- metavar="test_baseline",
- type=argparse.FileType("r"),
- nargs=1,
- help="A benchmark executable or JSON output file",
- )
- contender = parser_a.add_argument_group(
- "contender", "The benchmark that will be compared against the baseline"
- )
- contender.add_argument(
- "test_contender",
- metavar="test_contender",
- type=argparse.FileType("r"),
- nargs=1,
- help="A benchmark executable or JSON output file",
- )
- parser_a.add_argument(
- "benchmark_options",
- metavar="benchmark_options",
- nargs=argparse.REMAINDER,
- help="Arguments to pass when running benchmark executables",
- )
-
- parser_b = subparsers.add_parser(
- "filters", help="Compare filter one with the filter two of benchmark"
- )
- baseline = parser_b.add_argument_group("baseline", "The benchmark baseline")
- baseline.add_argument(
- "test",
- metavar="test",
- type=argparse.FileType("r"),
- nargs=1,
- help="A benchmark executable or JSON output file",
- )
- baseline.add_argument(
- "filter_baseline",
- metavar="filter_baseline",
- type=str,
- nargs=1,
- help="The first filter, that will be used as baseline",
- )
- contender = parser_b.add_argument_group(
- "contender", "The benchmark that will be compared against the baseline"
- )
- contender.add_argument(
- "filter_contender",
- metavar="filter_contender",
- type=str,
- nargs=1,
- help="The second filter, that will be compared against the baseline",
- )
- parser_b.add_argument(
- "benchmark_options",
- metavar="benchmark_options",
- nargs=argparse.REMAINDER,
- help="Arguments to pass when running benchmark executables",
- )
-
- parser_c = subparsers.add_parser(
- "benchmarksfiltered",
- help="Compare filter one of first benchmark with filter two of the second benchmark",
- )
- baseline = parser_c.add_argument_group("baseline", "The benchmark baseline")
- baseline.add_argument(
- "test_baseline",
- metavar="test_baseline",
- type=argparse.FileType("r"),
- nargs=1,
- help="A benchmark executable or JSON output file",
- )
- baseline.add_argument(
- "filter_baseline",
- metavar="filter_baseline",
- type=str,
- nargs=1,
- help="The first filter, that will be used as baseline",
- )
- contender = parser_c.add_argument_group(
- "contender", "The benchmark that will be compared against the baseline"
- )
- contender.add_argument(
- "test_contender",
- metavar="test_contender",
- type=argparse.FileType("r"),
- nargs=1,
- help="The second benchmark executable or JSON output file, that will be compared against the baseline",
- )
- contender.add_argument(
- "filter_contender",
- metavar="filter_contender",
- type=str,
- nargs=1,
- help="The second filter, that will be compared against the baseline",
- )
- parser_c.add_argument(
- "benchmark_options",
- metavar="benchmark_options",
- nargs=argparse.REMAINDER,
- help="Arguments to pass when running benchmark executables",
- )
-
- return parser
-
-
-def main():
- # Parse the command line flags
- parser = create_parser()
- args, unknown_args = parser.parse_known_args()
- if args.mode is None:
- parser.print_help()
- exit(1)
- assert not unknown_args
- benchmark_options = args.benchmark_options
-
- if args.mode == "benchmarks":
- test_baseline = args.test_baseline[0].name
- test_contender = args.test_contender[0].name
- filter_baseline = ""
- filter_contender = ""
-
- # NOTE: if test_baseline == test_contender, you are analyzing the stdev
-
- description = "Comparing %s to %s" % (test_baseline, test_contender)
- elif args.mode == "filters":
- test_baseline = args.test[0].name
- test_contender = args.test[0].name
- filter_baseline = args.filter_baseline[0]
- filter_contender = args.filter_contender[0]
-
- # NOTE: if filter_baseline == filter_contender, you are analyzing the
- # stdev
-
- description = "Comparing %s to %s (from %s)" % (
- filter_baseline,
- filter_contender,
- args.test[0].name,
- )
- elif args.mode == "benchmarksfiltered":
- test_baseline = args.test_baseline[0].name
- test_contender = args.test_contender[0].name
- filter_baseline = args.filter_baseline[0]
- filter_contender = args.filter_contender[0]
-
- # NOTE: if test_baseline == test_contender and
- # filter_baseline == filter_contender, you are analyzing the stdev
-
- description = "Comparing %s (from %s) to %s (from %s)" % (
- filter_baseline,
- test_baseline,
- filter_contender,
- test_contender,
- )
- else:
- # should never happen
- print("Unrecognized mode of operation: '%s'" % args.mode)
- parser.print_help()
- exit(1)
-
- check_inputs(test_baseline, test_contender, benchmark_options)
-
- if args.display_aggregates_only:
- benchmark_options += ["--benchmark_display_aggregates_only=true"]
-
- options_baseline = []
- options_contender = []
-
- if filter_baseline and filter_contender:
- options_baseline = ["--benchmark_filter=%s" % filter_baseline]
- options_contender = ["--benchmark_filter=%s" % filter_contender]
-
- # Run the benchmarks and report the results
- json1 = json1_orig = gbench.util.sort_benchmark_results(
- gbench.util.run_or_load_benchmark(
- test_baseline, benchmark_options + options_baseline
- )
- )
- json2 = json2_orig = gbench.util.sort_benchmark_results(
- gbench.util.run_or_load_benchmark(
- test_contender, benchmark_options + options_contender
- )
- )
-
- # Now, filter the benchmarks so that the difference report can work
- if filter_baseline and filter_contender:
- replacement = "[%s vs. %s]" % (filter_baseline, filter_contender)
- json1 = gbench.report.filter_benchmark(
- json1_orig, filter_baseline, replacement
- )
- json2 = gbench.report.filter_benchmark(
- json2_orig, filter_contender, replacement
- )
-
- diff_report = gbench.report.get_difference_report(json1, json2, args.utest)
- output_lines = gbench.report.print_difference_report(
- diff_report,
- args.display_aggregates_only,
- args.utest,
- args.utest_alpha,
- args.color,
- )
- print(description)
- for ln in output_lines:
- print(ln)
-
- # Optionally, diff and output to JSON
- if args.dump_to_json is not None:
- with open(args.dump_to_json, "w") as f_json:
- json.dump(diff_report, f_json, indent=1)
-
-
-class TestParser(unittest.TestCase):
- def setUp(self):
- self.parser = create_parser()
- testInputs = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), "gbench", "Inputs"
- )
- self.testInput0 = os.path.join(testInputs, "test1_run1.json")
- self.testInput1 = os.path.join(testInputs, "test1_run2.json")
-
- def test_benchmarks_basic(self):
- parsed = self.parser.parse_args(
- ["benchmarks", self.testInput0, self.testInput1]
- )
- self.assertFalse(parsed.display_aggregates_only)
- self.assertTrue(parsed.utest)
- self.assertEqual(parsed.mode, "benchmarks")
- self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
- self.assertEqual(parsed.test_contender[0].name, self.testInput1)
- self.assertFalse(parsed.benchmark_options)
-
- def test_benchmarks_basic_without_utest(self):
- parsed = self.parser.parse_args(
- ["--no-utest", "benchmarks", self.testInput0, self.testInput1]
- )
- self.assertFalse(parsed.display_aggregates_only)
- self.assertFalse(parsed.utest)
- self.assertEqual(parsed.utest_alpha, 0.05)
- self.assertEqual(parsed.mode, "benchmarks")
- self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
- self.assertEqual(parsed.test_contender[0].name, self.testInput1)
- self.assertFalse(parsed.benchmark_options)
-
- def test_benchmarks_basic_display_aggregates_only(self):
- parsed = self.parser.parse_args(
- ["-a", "benchmarks", self.testInput0, self.testInput1]
- )
- self.assertTrue(parsed.display_aggregates_only)
- self.assertTrue(parsed.utest)
- self.assertEqual(parsed.mode, "benchmarks")
- self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
- self.assertEqual(parsed.test_contender[0].name, self.testInput1)
- self.assertFalse(parsed.benchmark_options)
-
- def test_benchmarks_basic_with_utest_alpha(self):
- parsed = self.parser.parse_args(
- ["--alpha=0.314", "benchmarks", self.testInput0, self.testInput1]
- )
- self.assertFalse(parsed.display_aggregates_only)
- self.assertTrue(parsed.utest)
- self.assertEqual(parsed.utest_alpha, 0.314)
- self.assertEqual(parsed.mode, "benchmarks")
- self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
- self.assertEqual(parsed.test_contender[0].name, self.testInput1)
- self.assertFalse(parsed.benchmark_options)
-
- def test_benchmarks_basic_without_utest_with_utest_alpha(self):
- parsed = self.parser.parse_args(
- [
- "--no-utest",
- "--alpha=0.314",
- "benchmarks",
- self.testInput0,
- self.testInput1,
- ]
- )
- self.assertFalse(parsed.display_aggregates_only)
- self.assertFalse(parsed.utest)
- self.assertEqual(parsed.utest_alpha, 0.314)
- self.assertEqual(parsed.mode, "benchmarks")
- self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
- self.assertEqual(parsed.test_contender[0].name, self.testInput1)
- self.assertFalse(parsed.benchmark_options)
-
- def test_benchmarks_with_remainder(self):
- parsed = self.parser.parse_args(
- ["benchmarks", self.testInput0, self.testInput1, "d"]
- )
- self.assertFalse(parsed.display_aggregates_only)
- self.assertTrue(parsed.utest)
- self.assertEqual(parsed.mode, "benchmarks")
- self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
- self.assertEqual(parsed.test_contender[0].name, self.testInput1)
- self.assertEqual(parsed.benchmark_options, ["d"])
-
- def test_benchmarks_with_remainder_after_doubleminus(self):
- parsed = self.parser.parse_args(
- ["benchmarks", self.testInput0, self.testInput1, "--", "e"]
- )
- self.assertFalse(parsed.display_aggregates_only)
- self.assertTrue(parsed.utest)
- self.assertEqual(parsed.mode, "benchmarks")
- self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
- self.assertEqual(parsed.test_contender[0].name, self.testInput1)
- self.assertEqual(parsed.benchmark_options, ["e"])
-
- def test_filters_basic(self):
- parsed = self.parser.parse_args(["filters", self.testInput0, "c", "d"])
- self.assertFalse(parsed.display_aggregates_only)
- self.assertTrue(parsed.utest)
- self.assertEqual(parsed.mode, "filters")
- self.assertEqual(parsed.test[0].name, self.testInput0)
- self.assertEqual(parsed.filter_baseline[0], "c")
- self.assertEqual(parsed.filter_contender[0], "d")
- self.assertFalse(parsed.benchmark_options)
-
- def test_filters_with_remainder(self):
- parsed = self.parser.parse_args(
- ["filters", self.testInput0, "c", "d", "e"]
- )
- self.assertFalse(parsed.display_aggregates_only)
- self.assertTrue(parsed.utest)
- self.assertEqual(parsed.mode, "filters")
- self.assertEqual(parsed.test[0].name, self.testInput0)
- self.assertEqual(parsed.filter_baseline[0], "c")
- self.assertEqual(parsed.filter_contender[0], "d")
- self.assertEqual(parsed.benchmark_options, ["e"])
-
- def test_filters_with_remainder_after_doubleminus(self):
- parsed = self.parser.parse_args(
- ["filters", self.testInput0, "c", "d", "--", "f"]
- )
- self.assertFalse(parsed.display_aggregates_only)
- self.assertTrue(parsed.utest)
- self.assertEqual(parsed.mode, "filters")
- self.assertEqual(parsed.test[0].name, self.testInput0)
- self.assertEqual(parsed.filter_baseline[0], "c")
- self.assertEqual(parsed.filter_contender[0], "d")
- self.assertEqual(parsed.benchmark_options, ["f"])
-
- def test_benchmarksfiltered_basic(self):
- parsed = self.parser.parse_args(
- ["benchmarksfiltered", self.testInput0, "c", self.testInput1, "e"]
- )
- self.assertFalse(parsed.display_aggregates_only)
- self.assertTrue(parsed.utest)
- self.assertEqual(parsed.mode, "benchmarksfiltered")
- self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
- self.assertEqual(parsed.filter_baseline[0], "c")
- self.assertEqual(parsed.test_contender[0].name, self.testInput1)
- self.assertEqual(parsed.filter_contender[0], "e")
- self.assertFalse(parsed.benchmark_options)
-
- def test_benchmarksfiltered_with_remainder(self):
- parsed = self.parser.parse_args(
- [
- "benchmarksfiltered",
- self.testInput0,
- "c",
- self.testInput1,
- "e",
- "f",
- ]
- )
- self.assertFalse(parsed.display_aggregates_only)
- self.assertTrue(parsed.utest)
- self.assertEqual(parsed.mode, "benchmarksfiltered")
- self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
- self.assertEqual(parsed.filter_baseline[0], "c")
- self.assertEqual(parsed.test_contender[0].name, self.testInput1)
- self.assertEqual(parsed.filter_contender[0], "e")
- self.assertEqual(parsed.benchmark_options[0], "f")
-
- def test_benchmarksfiltered_with_remainder_after_doubleminus(self):
- parsed = self.parser.parse_args(
- [
- "benchmarksfiltered",
- self.testInput0,
- "c",
- self.testInput1,
- "e",
- "--",
- "g",
- ]
- )
- self.assertFalse(parsed.display_aggregates_only)
- self.assertTrue(parsed.utest)
- self.assertEqual(parsed.mode, "benchmarksfiltered")
- self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
- self.assertEqual(parsed.filter_baseline[0], "c")
- self.assertEqual(parsed.test_contender[0].name, self.testInput1)
- self.assertEqual(parsed.filter_contender[0], "e")
- self.assertEqual(parsed.benchmark_options[0], "g")
-
-
-if __name__ == "__main__":
- # unittest.main()
- main()
-
-# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
-# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;
-# kate: indent-mode python; remove-trailing-spaces modified;
diff --git a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test1_run1.json b/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test1_run1.json
deleted file mode 100644
index 9daed0bcc6..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test1_run1.json
+++ /dev/null
@@ -1,127 +0,0 @@
-{
- "context": {
- "date": "2016-08-02 17:44:46",
- "num_cpus": 4,
- "mhz_per_cpu": 4228,
- "cpu_scaling_enabled": false,
- "library_build_type": "release"
- },
- "benchmarks": [
- {
- "name": "BM_SameTimes",
- "iterations": 1000,
- "real_time": 10,
- "cpu_time": 10,
- "time_unit": "ns"
- },
- {
- "name": "BM_2xFaster",
- "iterations": 1000,
- "real_time": 50,
- "cpu_time": 50,
- "time_unit": "ns"
- },
- {
- "name": "BM_2xSlower",
- "iterations": 1000,
- "real_time": 50,
- "cpu_time": 50,
- "time_unit": "ns"
- },
- {
- "name": "BM_1PercentFaster",
- "iterations": 1000,
- "real_time": 100,
- "cpu_time": 100,
- "time_unit": "ns"
- },
- {
- "name": "BM_1PercentSlower",
- "iterations": 1000,
- "real_time": 100,
- "cpu_time": 100,
- "time_unit": "ns"
- },
- {
- "name": "BM_10PercentFaster",
- "iterations": 1000,
- "real_time": 100,
- "cpu_time": 100,
- "time_unit": "ns"
- },
- {
- "name": "BM_10PercentSlower",
- "iterations": 1000,
- "real_time": 100,
- "cpu_time": 100,
- "time_unit": "ns"
- },
- {
- "name": "BM_100xSlower",
- "iterations": 1000,
- "real_time": 100,
- "cpu_time": 100,
- "time_unit": "ns"
- },
- {
- "name": "BM_100xFaster",
- "iterations": 1000,
- "real_time": 10000,
- "cpu_time": 10000,
- "time_unit": "ns"
- },
- {
- "name": "BM_10PercentCPUToTime",
- "iterations": 1000,
- "real_time": 100,
- "cpu_time": 100,
- "time_unit": "ns"
- },
- {
- "name": "BM_ThirdFaster",
- "iterations": 1000,
- "real_time": 100,
- "cpu_time": 100,
- "time_unit": "ns"
- },
- {
- "name": "MyComplexityTest_BigO",
- "run_name": "MyComplexityTest",
- "run_type": "aggregate",
- "aggregate_name": "BigO",
- "cpu_coefficient": 4.2749856294592886e+00,
- "real_coefficient": 6.4789275289789780e+00,
- "big_o": "N",
- "time_unit": "ns"
- },
- {
- "name": "MyComplexityTest_RMS",
- "run_name": "MyComplexityTest",
- "run_type": "aggregate",
- "aggregate_name": "RMS",
- "rms": 4.5097802512472874e-03
- },
- {
- "name": "BM_NotBadTimeUnit",
- "iterations": 1000,
- "real_time": 0.4,
- "cpu_time": 0.5,
- "time_unit": "s"
- },
- {
- "name": "BM_DifferentTimeUnit",
- "iterations": 1,
- "real_time": 1,
- "cpu_time": 1,
- "time_unit": "s"
- },
- {
- "name": "BM_hasLabel",
- "label": "a label",
- "iterations": 1,
- "real_time": 1,
- "cpu_time": 1,
- "time_unit": "s"
- }
- ]
-}
diff --git a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test1_run2.json b/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test1_run2.json
deleted file mode 100644
index dc52970abf..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test1_run2.json
+++ /dev/null
@@ -1,127 +0,0 @@
-{
- "context": {
- "date": "2016-08-02 17:44:46",
- "num_cpus": 4,
- "mhz_per_cpu": 4228,
- "cpu_scaling_enabled": false,
- "library_build_type": "release"
- },
- "benchmarks": [
- {
- "name": "BM_SameTimes",
- "iterations": 1000,
- "real_time": 10,
- "cpu_time": 10,
- "time_unit": "ns"
- },
- {
- "name": "BM_2xFaster",
- "iterations": 1000,
- "real_time": 25,
- "cpu_time": 25,
- "time_unit": "ns"
- },
- {
- "name": "BM_2xSlower",
- "iterations": 20833333,
- "real_time": 100,
- "cpu_time": 100,
- "time_unit": "ns"
- },
- {
- "name": "BM_1PercentFaster",
- "iterations": 1000,
- "real_time": 98.9999999,
- "cpu_time": 98.9999999,
- "time_unit": "ns"
- },
- {
- "name": "BM_1PercentSlower",
- "iterations": 1000,
- "real_time": 100.9999999,
- "cpu_time": 100.9999999,
- "time_unit": "ns"
- },
- {
- "name": "BM_10PercentFaster",
- "iterations": 1000,
- "real_time": 90,
- "cpu_time": 90,
- "time_unit": "ns"
- },
- {
- "name": "BM_10PercentSlower",
- "iterations": 1000,
- "real_time": 110,
- "cpu_time": 110,
- "time_unit": "ns"
- },
- {
- "name": "BM_100xSlower",
- "iterations": 1000,
- "real_time": 1.0000e+04,
- "cpu_time": 1.0000e+04,
- "time_unit": "ns"
- },
- {
- "name": "BM_100xFaster",
- "iterations": 1000,
- "real_time": 100,
- "cpu_time": 100,
- "time_unit": "ns"
- },
- {
- "name": "BM_10PercentCPUToTime",
- "iterations": 1000,
- "real_time": 110,
- "cpu_time": 90,
- "time_unit": "ns"
- },
- {
- "name": "BM_ThirdFaster",
- "iterations": 1000,
- "real_time": 66.665,
- "cpu_time": 66.664,
- "time_unit": "ns"
- },
- {
- "name": "MyComplexityTest_BigO",
- "run_name": "MyComplexityTest",
- "run_type": "aggregate",
- "aggregate_name": "BigO",
- "cpu_coefficient": 5.6215779594361486e+00,
- "real_coefficient": 5.6288314793554610e+00,
- "big_o": "N",
- "time_unit": "ns"
- },
- {
- "name": "MyComplexityTest_RMS",
- "run_name": "MyComplexityTest",
- "run_type": "aggregate",
- "aggregate_name": "RMS",
- "rms": 3.3128901852342174e-03
- },
- {
- "name": "BM_NotBadTimeUnit",
- "iterations": 1000,
- "real_time": 0.04,
- "cpu_time": 0.6,
- "time_unit": "s"
- },
- {
- "name": "BM_DifferentTimeUnit",
- "iterations": 1,
- "real_time": 1,
- "cpu_time": 1,
- "time_unit": "ns"
- },
- {
- "name": "BM_hasLabel",
- "label": "a label",
- "iterations": 1,
- "real_time": 1,
- "cpu_time": 1,
- "time_unit": "s"
- }
- ]
-}
diff --git a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test2_run.json b/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test2_run.json
deleted file mode 100644
index 15bc698030..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test2_run.json
+++ /dev/null
@@ -1,81 +0,0 @@
-{
- "context": {
- "date": "2016-08-02 17:44:46",
- "num_cpus": 4,
- "mhz_per_cpu": 4228,
- "cpu_scaling_enabled": false,
- "library_build_type": "release"
- },
- "benchmarks": [
- {
- "name": "BM_Hi",
- "iterations": 1234,
- "real_time": 42,
- "cpu_time": 24,
- "time_unit": "ms"
- },
- {
- "name": "BM_Zero",
- "iterations": 1000,
- "real_time": 10,
- "cpu_time": 10,
- "time_unit": "ns"
- },
- {
- "name": "BM_Zero/4",
- "iterations": 4000,
- "real_time": 40,
- "cpu_time": 40,
- "time_unit": "ns"
- },
- {
- "name": "Prefix/BM_Zero",
- "iterations": 2000,
- "real_time": 20,
- "cpu_time": 20,
- "time_unit": "ns"
- },
- {
- "name": "Prefix/BM_Zero/3",
- "iterations": 3000,
- "real_time": 30,
- "cpu_time": 30,
- "time_unit": "ns"
- },
- {
- "name": "BM_One",
- "iterations": 5000,
- "real_time": 5,
- "cpu_time": 5,
- "time_unit": "ns"
- },
- {
- "name": "BM_One/4",
- "iterations": 2000,
- "real_time": 20,
- "cpu_time": 20,
- "time_unit": "ns"
- },
- {
- "name": "Prefix/BM_One",
- "iterations": 1000,
- "real_time": 10,
- "cpu_time": 10,
- "time_unit": "ns"
- },
- {
- "name": "Prefix/BM_One/3",
- "iterations": 1500,
- "real_time": 15,
- "cpu_time": 15,
- "time_unit": "ns"
- },
- {
- "name": "BM_Bye",
- "iterations": 5321,
- "real_time": 11,
- "cpu_time": 63,
- "time_unit": "ns"
- }
- ]
-}
diff --git a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test3_run0.json b/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test3_run0.json
deleted file mode 100644
index 49f8b06143..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test3_run0.json
+++ /dev/null
@@ -1,65 +0,0 @@
-{
- "context": {
- "date": "2016-08-02 17:44:46",
- "num_cpus": 4,
- "mhz_per_cpu": 4228,
- "cpu_scaling_enabled": false,
- "library_build_type": "release"
- },
- "benchmarks": [
- {
- "name": "BM_One",
- "run_type": "aggregate",
- "iterations": 1000,
- "real_time": 10,
- "cpu_time": 100,
- "time_unit": "ns"
- },
- {
- "name": "BM_Two",
- "iterations": 1000,
- "real_time": 9,
- "cpu_time": 90,
- "time_unit": "ns"
- },
- {
- "name": "BM_Two",
- "iterations": 1000,
- "real_time": 8,
- "cpu_time": 86,
- "time_unit": "ns"
- },
- {
- "name": "short",
- "run_type": "aggregate",
- "iterations": 1000,
- "real_time": 8,
- "cpu_time": 80,
- "time_unit": "ns"
- },
- {
- "name": "short",
- "run_type": "aggregate",
- "iterations": 1000,
- "real_time": 8,
- "cpu_time": 77,
- "time_unit": "ns"
- },
- {
- "name": "medium",
- "run_type": "iteration",
- "iterations": 1000,
- "real_time": 8,
- "cpu_time": 80,
- "time_unit": "ns"
- },
- {
- "name": "medium",
- "run_type": "iteration",
- "iterations": 1000,
- "real_time": 9,
- "cpu_time": 82,
- "time_unit": "ns"
- }
- ]
-}
diff --git a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test3_run1.json b/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test3_run1.json
deleted file mode 100644
index acc5ba17ae..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test3_run1.json
+++ /dev/null
@@ -1,65 +0,0 @@
-{
- "context": {
- "date": "2016-08-02 17:44:46",
- "num_cpus": 4,
- "mhz_per_cpu": 4228,
- "cpu_scaling_enabled": false,
- "library_build_type": "release"
- },
- "benchmarks": [
- {
- "name": "BM_One",
- "iterations": 1000,
- "real_time": 9,
- "cpu_time": 110,
- "time_unit": "ns"
- },
- {
- "name": "BM_Two",
- "run_type": "aggregate",
- "iterations": 1000,
- "real_time": 10,
- "cpu_time": 89,
- "time_unit": "ns"
- },
- {
- "name": "BM_Two",
- "iterations": 1000,
- "real_time": 7,
- "cpu_time": 72,
- "time_unit": "ns"
- },
- {
- "name": "short",
- "run_type": "aggregate",
- "iterations": 1000,
- "real_time": 7,
- "cpu_time": 75,
- "time_unit": "ns"
- },
- {
- "name": "short",
- "run_type": "aggregate",
- "iterations": 762,
- "real_time": 4.54,
- "cpu_time": 66.6,
- "time_unit": "ns"
- },
- {
- "name": "short",
- "run_type": "iteration",
- "iterations": 1000,
- "real_time": 800,
- "cpu_time": 1,
- "time_unit": "ns"
- },
- {
- "name": "medium",
- "run_type": "iteration",
- "iterations": 1200,
- "real_time": 5,
- "cpu_time": 53,
- "time_unit": "ns"
- }
- ]
-}
diff --git a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test4_run.json b/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test4_run.json
deleted file mode 100644
index eaa005f3a9..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test4_run.json
+++ /dev/null
@@ -1,96 +0,0 @@
-{
- "benchmarks": [
- {
- "name": "99 family 0 instance 0 repetition 0",
- "run_type": "iteration",
- "family_index": 0,
- "per_family_instance_index": 0,
- "repetition_index": 0
- },
- {
- "name": "98 family 0 instance 0 repetition 1",
- "run_type": "iteration",
- "family_index": 0,
- "per_family_instance_index": 0,
- "repetition_index": 1
- },
- {
- "name": "97 family 0 instance 0 aggregate",
- "run_type": "aggregate",
- "family_index": 0,
- "per_family_instance_index": 0,
- "aggregate_name": "9 aggregate"
- },
-
-
- {
- "name": "96 family 0 instance 1 repetition 0",
- "run_type": "iteration",
- "family_index": 0,
- "per_family_instance_index": 1,
- "repetition_index": 0
- },
- {
- "name": "95 family 0 instance 1 repetition 1",
- "run_type": "iteration",
- "family_index": 0,
- "per_family_instance_index": 1,
- "repetition_index": 1
- },
- {
- "name": "94 family 0 instance 1 aggregate",
- "run_type": "aggregate",
- "family_index": 0,
- "per_family_instance_index": 1,
- "aggregate_name": "9 aggregate"
- },
-
-
-
-
- {
- "name": "93 family 1 instance 0 repetition 0",
- "run_type": "iteration",
- "family_index": 1,
- "per_family_instance_index": 0,
- "repetition_index": 0
- },
- {
- "name": "92 family 1 instance 0 repetition 1",
- "run_type": "iteration",
- "family_index": 1,
- "per_family_instance_index": 0,
- "repetition_index": 1
- },
- {
- "name": "91 family 1 instance 0 aggregate",
- "run_type": "aggregate",
- "family_index": 1,
- "per_family_instance_index": 0,
- "aggregate_name": "9 aggregate"
- },
-
-
- {
- "name": "90 family 1 instance 1 repetition 0",
- "run_type": "iteration",
- "family_index": 1,
- "per_family_instance_index": 1,
- "repetition_index": 0
- },
- {
- "name": "89 family 1 instance 1 repetition 1",
- "run_type": "iteration",
- "family_index": 1,
- "per_family_instance_index": 1,
- "repetition_index": 1
- },
- {
- "name": "88 family 1 instance 1 aggregate",
- "run_type": "aggregate",
- "family_index": 1,
- "per_family_instance_index": 1,
- "aggregate_name": "9 aggregate"
- }
- ]
-}
diff --git a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test4_run0.json b/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test4_run0.json
deleted file mode 100644
index 54cf127585..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test4_run0.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "context": {
- "date": "2016-08-02 17:44:46",
- "num_cpus": 4,
- "mhz_per_cpu": 4228,
- "cpu_scaling_enabled": false,
- "library_build_type": "release"
- },
- "benchmarks": [
- {
- "name": "whocares",
- "run_type": "aggregate",
- "aggregate_name": "zz",
- "aggregate_unit": "percentage",
- "iterations": 1000,
- "real_time": 0.01,
- "cpu_time": 0.10,
- "time_unit": "ns"
- }
- ]
-}
diff --git a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test4_run1.json b/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test4_run1.json
deleted file mode 100644
index 25d56050c9..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test4_run1.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "context": {
- "date": "2016-08-02 17:44:46",
- "num_cpus": 4,
- "mhz_per_cpu": 4228,
- "cpu_scaling_enabled": false,
- "library_build_type": "release"
- },
- "benchmarks": [
- {
- "name": "whocares",
- "run_type": "aggregate",
- "aggregate_name": "zz",
- "aggregate_unit": "percentage",
- "iterations": 1000,
- "real_time": 0.005,
- "cpu_time": 0.15,
- "time_unit": "ns"
- }
- ]
-}
diff --git a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test5_run0.json b/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test5_run0.json
deleted file mode 100644
index 074103b11d..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test5_run0.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "context": {
- "date": "2016-08-02 17:44:46",
- "num_cpus": 4,
- "mhz_per_cpu": 4228,
- "cpu_scaling_enabled": false,
- "library_build_type": "release"
- },
- "benchmarks": [
- {
- "name": "BM_ManyRepetitions",
- "iterations": 1000,
- "real_time": 1,
- "cpu_time": 1000,
- "time_unit": "s"
- }
- ]
-}
diff --git a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test5_run1.json b/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test5_run1.json
deleted file mode 100644
index 430df9f0da..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/gbench/Inputs/test5_run1.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "context": {
- "date": "2016-08-02 17:44:46",
- "num_cpus": 4,
- "mhz_per_cpu": 4228,
- "cpu_scaling_enabled": false,
- "library_build_type": "release"
- },
- "benchmarks": [
- {
- "name": "BM_ManyRepetitions",
- "iterations": 1000,
- "real_time": 1000,
- "cpu_time": 1,
- "time_unit": "s"
- }
- ]
-}
diff --git a/contrib/restricted/google/benchmark/tools/compare/gbench/__init__.py b/contrib/restricted/google/benchmark/tools/compare/gbench/__init__.py
deleted file mode 100644
index 9212568814..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/gbench/__init__.py
+++ /dev/null
@@ -1,8 +0,0 @@
-"""Google Benchmark tooling"""
-
-__author__ = "Eric Fiselier"
-__email__ = "eric@efcs.ca"
-__versioninfo__ = (0, 5, 0)
-__version__ = ".".join(str(v) for v in __versioninfo__) + "dev"
-
-__all__ = [] # type: ignore
diff --git a/contrib/restricted/google/benchmark/tools/compare/gbench/report.py b/contrib/restricted/google/benchmark/tools/compare/gbench/report.py
deleted file mode 100644
index 7158fd1654..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/gbench/report.py
+++ /dev/null
@@ -1,1619 +0,0 @@
-# type: ignore
-
-"""
-report.py - Utilities for reporting statistics about benchmark results
-"""
-
-import copy
-import os
-import random
-import re
-import unittest
-
-from numpy import array
-from scipy.stats import gmean, mannwhitneyu
-
-
-class BenchmarkColor(object):
- def __init__(self, name, code):
- self.name = name
- self.code = code
-
- def __repr__(self):
- return "%s%r" % (self.__class__.__name__, (self.name, self.code))
-
- def __format__(self, format):
- return self.code
-
-
-# Benchmark Colors Enumeration
-BC_NONE = BenchmarkColor("NONE", "")
-BC_MAGENTA = BenchmarkColor("MAGENTA", "\033[95m")
-BC_CYAN = BenchmarkColor("CYAN", "\033[96m")
-BC_OKBLUE = BenchmarkColor("OKBLUE", "\033[94m")
-BC_OKGREEN = BenchmarkColor("OKGREEN", "\033[32m")
-BC_HEADER = BenchmarkColor("HEADER", "\033[92m")
-BC_WARNING = BenchmarkColor("WARNING", "\033[93m")
-BC_WHITE = BenchmarkColor("WHITE", "\033[97m")
-BC_FAIL = BenchmarkColor("FAIL", "\033[91m")
-BC_ENDC = BenchmarkColor("ENDC", "\033[0m")
-BC_BOLD = BenchmarkColor("BOLD", "\033[1m")
-BC_UNDERLINE = BenchmarkColor("UNDERLINE", "\033[4m")
-
-UTEST_MIN_REPETITIONS = 2
-UTEST_OPTIMAL_REPETITIONS = 9 # Lowest reasonable number, More is better.
-UTEST_COL_NAME = "_pvalue"
-
-_TIME_UNIT_TO_SECONDS_MULTIPLIER = {
- "s": 1.0,
- "ms": 1e-3,
- "us": 1e-6,
- "ns": 1e-9,
-}
-
-
-def color_format(use_color, fmt_str, *args, **kwargs):
- """
- Return the result of 'fmt_str.format(*args, **kwargs)' after transforming
- 'args' and 'kwargs' according to the value of 'use_color'. If 'use_color'
- is False then all color codes in 'args' and 'kwargs' are replaced with
- the empty string.
- """
- assert use_color is True or use_color is False
- if not use_color:
- args = [
- arg if not isinstance(arg, BenchmarkColor) else BC_NONE
- for arg in args
- ]
- kwargs = {
- key: arg if not isinstance(arg, BenchmarkColor) else BC_NONE
- for key, arg in kwargs.items()
- }
- return fmt_str.format(*args, **kwargs)
-
-
-def find_longest_name(benchmark_list):
- """
- Return the length of the longest benchmark name in a given list of
- benchmark JSON objects
- """
- longest_name = 1
- for bc in benchmark_list:
- if len(bc["name"]) > longest_name:
- longest_name = len(bc["name"])
- return longest_name
-
-
-def calculate_change(old_val, new_val):
- """
- Return a float representing the decimal change between old_val and new_val.
- """
- if old_val == 0 and new_val == 0:
- return 0.0
- if old_val == 0:
- return float(new_val - old_val) / (float(old_val + new_val) / 2)
- return float(new_val - old_val) / abs(old_val)
-
-
-def filter_benchmark(json_orig, family, replacement=""):
- """
- Apply a filter to the json, and only leave the 'family' of benchmarks.
- """
- regex = re.compile(family)
- filtered = {}
- filtered["benchmarks"] = []
- for be in json_orig["benchmarks"]:
- if not regex.search(be["name"]):
- continue
- filteredbench = copy.deepcopy(be) # Do NOT modify the old name!
- filteredbench["name"] = regex.sub(replacement, filteredbench["name"])
- filtered["benchmarks"].append(filteredbench)
- return filtered
-
-
-def get_unique_benchmark_names(json):
- """
- While *keeping* the order, give all the unique 'names' used for benchmarks.
- """
- seen = set()
- uniqued = [
- x["name"]
- for x in json["benchmarks"]
- if x["name"] not in seen and (seen.add(x["name"]) or True)
- ]
- return uniqued
-
-
-def intersect(list1, list2):
- """
- Given two lists, get a new list consisting of the elements only contained
- in *both of the input lists*, while preserving the ordering.
- """
- return [x for x in list1 if x in list2]
-
-
-def is_potentially_comparable_benchmark(x):
- return "time_unit" in x and "real_time" in x and "cpu_time" in x
-
-
-def partition_benchmarks(json1, json2):
- """
- While preserving the ordering, find benchmarks with the same names in
- both of the inputs, and group them.
- (i.e. partition/filter into groups with common name)
- """
- json1_unique_names = get_unique_benchmark_names(json1)
- json2_unique_names = get_unique_benchmark_names(json2)
- names = intersect(json1_unique_names, json2_unique_names)
- partitions = []
- for name in names:
- time_unit = None
- # Pick the time unit from the first entry of the lhs benchmark.
- # We should be careful not to crash with unexpected input.
- for x in json1["benchmarks"]:
- if x["name"] == name and is_potentially_comparable_benchmark(x):
- time_unit = x["time_unit"]
- break
- if time_unit is None:
- continue
- # Filter by name and time unit.
- # All the repetitions are assumed to be comparable.
- lhs = [
- x
- for x in json1["benchmarks"]
- if x["name"] == name and x["time_unit"] == time_unit
- ]
- rhs = [
- x
- for x in json2["benchmarks"]
- if x["name"] == name and x["time_unit"] == time_unit
- ]
- partitions.append([lhs, rhs])
- return partitions
-
-
-def get_timedelta_field_as_seconds(benchmark, field_name):
- """
- Get value of field_name field of benchmark, which is time with time unit
- time_unit, as time in seconds.
- """
- timedelta = benchmark[field_name]
- time_unit = benchmark.get("time_unit", "s")
- return timedelta * _TIME_UNIT_TO_SECONDS_MULTIPLIER.get(time_unit)
-
-
-def calculate_geomean(json):
- """
- Extract all real/cpu times from all the benchmarks as seconds,
- and calculate their geomean.
- """
- times = []
- for benchmark in json["benchmarks"]:
- if "run_type" in benchmark and benchmark["run_type"] == "aggregate":
- continue
- times.append(
- [
- get_timedelta_field_as_seconds(benchmark, "real_time"),
- get_timedelta_field_as_seconds(benchmark, "cpu_time"),
- ]
- )
- return gmean(times) if times else array([])
-
-
-def extract_field(partition, field_name):
- # The count of elements may be different. We want *all* of them.
- lhs = [x[field_name] for x in partition[0]]
- rhs = [x[field_name] for x in partition[1]]
- return [lhs, rhs]
-
-
-def calc_utest(timings_cpu, timings_time):
- min_rep_cnt = min(
- len(timings_time[0]),
- len(timings_time[1]),
- len(timings_cpu[0]),
- len(timings_cpu[1]),
- )
-
- # Does *everything* has at least UTEST_MIN_REPETITIONS repetitions?
- if min_rep_cnt < UTEST_MIN_REPETITIONS:
- return False, None, None
-
- time_pvalue = mannwhitneyu(
- timings_time[0], timings_time[1], alternative="two-sided"
- ).pvalue
- cpu_pvalue = mannwhitneyu(
- timings_cpu[0], timings_cpu[1], alternative="two-sided"
- ).pvalue
-
- return (min_rep_cnt >= UTEST_OPTIMAL_REPETITIONS), cpu_pvalue, time_pvalue
-
-
-def print_utest(bc_name, utest, utest_alpha, first_col_width, use_color=True):
- def get_utest_color(pval):
- return BC_FAIL if pval >= utest_alpha else BC_OKGREEN
-
- # Check if we failed miserably with minimum required repetitions for utest
- if (
- not utest["have_optimal_repetitions"]
- and utest["cpu_pvalue"] is None
- and utest["time_pvalue"] is None
- ):
- return []
-
- dsc = "U Test, Repetitions: {} vs {}".format(
- utest["nr_of_repetitions"], utest["nr_of_repetitions_other"]
- )
- dsc_color = BC_OKGREEN
-
- # We still got some results to show but issue a warning about it.
- if not utest["have_optimal_repetitions"]:
- dsc_color = BC_WARNING
- dsc += ". WARNING: Results unreliable! {}+ repetitions recommended.".format(
- UTEST_OPTIMAL_REPETITIONS
- )
-
- special_str = "{}{:<{}s}{endc}{}{:16.4f}{endc}{}{:16.4f}{endc}{} {}"
-
- return [
- color_format(
- use_color,
- special_str,
- BC_HEADER,
- "{}{}".format(bc_name, UTEST_COL_NAME),
- first_col_width,
- get_utest_color(utest["time_pvalue"]),
- utest["time_pvalue"],
- get_utest_color(utest["cpu_pvalue"]),
- utest["cpu_pvalue"],
- dsc_color,
- dsc,
- endc=BC_ENDC,
- )
- ]
-
-
-def get_difference_report(json1, json2, utest=False):
- """
- Calculate and report the difference between each test of two benchmarks
- runs specified as 'json1' and 'json2'. Output is another json containing
- relevant details for each test run.
- """
- assert utest is True or utest is False
-
- diff_report = []
- partitions = partition_benchmarks(json1, json2)
- for partition in partitions:
- benchmark_name = partition[0][0]["name"]
- label = partition[0][0]["label"] if "label" in partition[0][0] else ""
- time_unit = partition[0][0]["time_unit"]
- measurements = []
- utest_results = {}
- # Careful, we may have different repetition count.
- for i in range(min(len(partition[0]), len(partition[1]))):
- bn = partition[0][i]
- other_bench = partition[1][i]
- measurements.append(
- {
- "real_time": bn["real_time"],
- "cpu_time": bn["cpu_time"],
- "real_time_other": other_bench["real_time"],
- "cpu_time_other": other_bench["cpu_time"],
- "time": calculate_change(
- bn["real_time"], other_bench["real_time"]
- ),
- "cpu": calculate_change(
- bn["cpu_time"], other_bench["cpu_time"]
- ),
- }
- )
-
- # After processing the whole partition, if requested, do the U test.
- if utest:
- timings_cpu = extract_field(partition, "cpu_time")
- timings_time = extract_field(partition, "real_time")
- have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest(
- timings_cpu, timings_time
- )
- if cpu_pvalue is not None and time_pvalue is not None:
- utest_results = {
- "have_optimal_repetitions": have_optimal_repetitions,
- "cpu_pvalue": cpu_pvalue,
- "time_pvalue": time_pvalue,
- "nr_of_repetitions": len(timings_cpu[0]),
- "nr_of_repetitions_other": len(timings_cpu[1]),
- }
-
- # Store only if we had any measurements for given benchmark.
- # E.g. partition_benchmarks will filter out the benchmarks having
- # time units which are not compatible with other time units in the
- # benchmark suite.
- if measurements:
- run_type = (
- partition[0][0]["run_type"]
- if "run_type" in partition[0][0]
- else ""
- )
- aggregate_name = (
- partition[0][0]["aggregate_name"]
- if run_type == "aggregate"
- and "aggregate_name" in partition[0][0]
- else ""
- )
- diff_report.append(
- {
- "name": benchmark_name,
- "label": label,
- "measurements": measurements,
- "time_unit": time_unit,
- "run_type": run_type,
- "aggregate_name": aggregate_name,
- "utest": utest_results,
- }
- )
-
- lhs_gmean = calculate_geomean(json1)
- rhs_gmean = calculate_geomean(json2)
- if lhs_gmean.any() and rhs_gmean.any():
- diff_report.append(
- {
- "name": "OVERALL_GEOMEAN",
- "label": "",
- "measurements": [
- {
- "real_time": lhs_gmean[0],
- "cpu_time": lhs_gmean[1],
- "real_time_other": rhs_gmean[0],
- "cpu_time_other": rhs_gmean[1],
- "time": calculate_change(lhs_gmean[0], rhs_gmean[0]),
- "cpu": calculate_change(lhs_gmean[1], rhs_gmean[1]),
- }
- ],
- "time_unit": "s",
- "run_type": "aggregate",
- "aggregate_name": "geomean",
- "utest": {},
- }
- )
-
- return diff_report
-
-
-def print_difference_report(
- json_diff_report,
- include_aggregates_only=False,
- utest=False,
- utest_alpha=0.05,
- use_color=True,
-):
- """
- Calculate and report the difference between each test of two benchmarks
- runs specified as 'json1' and 'json2'.
- """
- assert utest is True or utest is False
-
- def get_color(res):
- if res > 0.05:
- return BC_FAIL
- elif res > -0.07:
- return BC_WHITE
- else:
- return BC_CYAN
-
- first_col_width = find_longest_name(json_diff_report)
- first_col_width = max(first_col_width, len("Benchmark"))
- first_col_width += len(UTEST_COL_NAME)
- first_line = "{:<{}s}Time CPU Time Old Time New CPU Old CPU New".format(
- "Benchmark", 12 + first_col_width
- )
- output_strs = [first_line, "-" * len(first_line)]
-
- fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}"
- for benchmark in json_diff_report:
- # *If* we were asked to only include aggregates,
- # and if it is non-aggregate, then don't print it.
- if (
- not include_aggregates_only
- or "run_type" not in benchmark
- or benchmark["run_type"] == "aggregate"
- ):
- for measurement in benchmark["measurements"]:
- output_strs += [
- color_format(
- use_color,
- fmt_str,
- BC_HEADER,
- benchmark["name"],
- first_col_width,
- get_color(measurement["time"]),
- measurement["time"],
- get_color(measurement["cpu"]),
- measurement["cpu"],
- measurement["real_time"],
- measurement["real_time_other"],
- measurement["cpu_time"],
- measurement["cpu_time_other"],
- endc=BC_ENDC,
- )
- ]
-
- # After processing the measurements, if requested and
- # if applicable (e.g. u-test exists for given benchmark),
- # print the U test.
- if utest and benchmark["utest"]:
- output_strs += print_utest(
- benchmark["name"],
- benchmark["utest"],
- utest_alpha=utest_alpha,
- first_col_width=first_col_width,
- use_color=use_color,
- )
-
- return output_strs
-
-
-###############################################################################
-# Unit tests
-
-
-class TestGetUniqueBenchmarkNames(unittest.TestCase):
- def load_results(self):
- import json
-
- testInputs = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), "Inputs"
- )
- testOutput = os.path.join(testInputs, "test3_run0.json")
- with open(testOutput, "r") as f:
- json = json.load(f)
- return json
-
- def test_basic(self):
- expect_lines = [
- "BM_One",
- "BM_Two",
- "short", # These two are not sorted
- "medium", # These two are not sorted
- ]
- json = self.load_results()
- output_lines = get_unique_benchmark_names(json)
- print("\n")
- print("\n".join(output_lines))
- self.assertEqual(len(output_lines), len(expect_lines))
- for i in range(0, len(output_lines)):
- self.assertEqual(expect_lines[i], output_lines[i])
-
-
-class TestReportDifference(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- def load_results():
- import json
-
- testInputs = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), "Inputs"
- )
- testOutput1 = os.path.join(testInputs, "test1_run1.json")
- testOutput2 = os.path.join(testInputs, "test1_run2.json")
- with open(testOutput1, "r") as f:
- json1 = json.load(f)
- with open(testOutput2, "r") as f:
- json2 = json.load(f)
- return json1, json2
-
- json1, json2 = load_results()
- cls.json_diff_report = get_difference_report(json1, json2)
-
- def test_json_diff_report_pretty_printing(self):
- expect_lines = [
- ["BM_SameTimes", "+0.0000", "+0.0000", "10", "10", "10", "10"],
- ["BM_2xFaster", "-0.5000", "-0.5000", "50", "25", "50", "25"],
- ["BM_2xSlower", "+1.0000", "+1.0000", "50", "100", "50", "100"],
- [
- "BM_1PercentFaster",
- "-0.0100",
- "-0.0100",
- "100",
- "99",
- "100",
- "99",
- ],
- [
- "BM_1PercentSlower",
- "+0.0100",
- "+0.0100",
- "100",
- "101",
- "100",
- "101",
- ],
- [
- "BM_10PercentFaster",
- "-0.1000",
- "-0.1000",
- "100",
- "90",
- "100",
- "90",
- ],
- [
- "BM_10PercentSlower",
- "+0.1000",
- "+0.1000",
- "100",
- "110",
- "100",
- "110",
- ],
- [
- "BM_100xSlower",
- "+99.0000",
- "+99.0000",
- "100",
- "10000",
- "100",
- "10000",
- ],
- [
- "BM_100xFaster",
- "-0.9900",
- "-0.9900",
- "10000",
- "100",
- "10000",
- "100",
- ],
- [
- "BM_10PercentCPUToTime",
- "+0.1000",
- "-0.1000",
- "100",
- "110",
- "100",
- "90",
- ],
- ["BM_ThirdFaster", "-0.3333", "-0.3334", "100", "67", "100", "67"],
- ["BM_NotBadTimeUnit", "-0.9000", "+0.2000", "0", "0", "0", "1"],
- ["BM_hasLabel", "+0.0000", "+0.0000", "1", "1", "1", "1"],
- ["OVERALL_GEOMEAN", "-0.8113", "-0.7779", "0", "0", "0", "0"],
- ]
- output_lines_with_header = print_difference_report(
- self.json_diff_report, use_color=False
- )
- output_lines = output_lines_with_header[2:]
- print("\n")
- print("\n".join(output_lines_with_header))
- self.assertEqual(len(output_lines), len(expect_lines))
- for i in range(0, len(output_lines)):
- parts = [x for x in output_lines[i].split(" ") if x]
- self.assertEqual(len(parts), 7)
- self.assertEqual(expect_lines[i], parts)
-
- def test_json_diff_report_output(self):
- expected_output = [
- {
- "name": "BM_SameTimes",
- "label": "",
- "measurements": [
- {
- "time": 0.0000,
- "cpu": 0.0000,
- "real_time": 10,
- "real_time_other": 10,
- "cpu_time": 10,
- "cpu_time_other": 10,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "BM_2xFaster",
- "label": "",
- "measurements": [
- {
- "time": -0.5000,
- "cpu": -0.5000,
- "real_time": 50,
- "real_time_other": 25,
- "cpu_time": 50,
- "cpu_time_other": 25,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "BM_2xSlower",
- "label": "",
- "measurements": [
- {
- "time": 1.0000,
- "cpu": 1.0000,
- "real_time": 50,
- "real_time_other": 100,
- "cpu_time": 50,
- "cpu_time_other": 100,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "BM_1PercentFaster",
- "label": "",
- "measurements": [
- {
- "time": -0.0100,
- "cpu": -0.0100,
- "real_time": 100,
- "real_time_other": 98.9999999,
- "cpu_time": 100,
- "cpu_time_other": 98.9999999,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "BM_1PercentSlower",
- "label": "",
- "measurements": [
- {
- "time": 0.0100,
- "cpu": 0.0100,
- "real_time": 100,
- "real_time_other": 101,
- "cpu_time": 100,
- "cpu_time_other": 101,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "BM_10PercentFaster",
- "label": "",
- "measurements": [
- {
- "time": -0.1000,
- "cpu": -0.1000,
- "real_time": 100,
- "real_time_other": 90,
- "cpu_time": 100,
- "cpu_time_other": 90,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "BM_10PercentSlower",
- "label": "",
- "measurements": [
- {
- "time": 0.1000,
- "cpu": 0.1000,
- "real_time": 100,
- "real_time_other": 110,
- "cpu_time": 100,
- "cpu_time_other": 110,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "BM_100xSlower",
- "label": "",
- "measurements": [
- {
- "time": 99.0000,
- "cpu": 99.0000,
- "real_time": 100,
- "real_time_other": 10000,
- "cpu_time": 100,
- "cpu_time_other": 10000,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "BM_100xFaster",
- "label": "",
- "measurements": [
- {
- "time": -0.9900,
- "cpu": -0.9900,
- "real_time": 10000,
- "real_time_other": 100,
- "cpu_time": 10000,
- "cpu_time_other": 100,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "BM_10PercentCPUToTime",
- "label": "",
- "measurements": [
- {
- "time": 0.1000,
- "cpu": -0.1000,
- "real_time": 100,
- "real_time_other": 110,
- "cpu_time": 100,
- "cpu_time_other": 90,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "BM_ThirdFaster",
- "label": "",
- "measurements": [
- {
- "time": -0.3333,
- "cpu": -0.3334,
- "real_time": 100,
- "real_time_other": 67,
- "cpu_time": 100,
- "cpu_time_other": 67,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "BM_NotBadTimeUnit",
- "label": "",
- "measurements": [
- {
- "time": -0.9000,
- "cpu": 0.2000,
- "real_time": 0.4,
- "real_time_other": 0.04,
- "cpu_time": 0.5,
- "cpu_time_other": 0.6,
- }
- ],
- "time_unit": "s",
- "utest": {},
- },
- {
- "name": "BM_hasLabel",
- "label": "a label",
- "measurements": [
- {
- "time": 0.0000,
- "cpu": 0.0000,
- "real_time": 1,
- "real_time_other": 1,
- "cpu_time": 1,
- "cpu_time_other": 1,
- }
- ],
- "time_unit": "s",
- "utest": {},
- },
- {
- "name": "OVERALL_GEOMEAN",
- "label": "",
- "measurements": [
- {
- "real_time": 3.1622776601683826e-06,
- "cpu_time": 3.2130844755623912e-06,
- "real_time_other": 1.9768988699420897e-07,
- "cpu_time_other": 2.397447755209533e-07,
- "time": -0.8112976497120911,
- "cpu": -0.7778551721181174,
- }
- ],
- "time_unit": "s",
- "run_type": "aggregate",
- "aggregate_name": "geomean",
- "utest": {},
- },
- ]
- self.assertEqual(len(self.json_diff_report), len(expected_output))
- for out, expected in zip(self.json_diff_report, expected_output):
- self.assertEqual(out["name"], expected["name"])
- self.assertEqual(out["label"], expected["label"])
- self.assertEqual(out["time_unit"], expected["time_unit"])
- assert_utest(self, out, expected)
- assert_measurements(self, out, expected)
-
-
-class TestReportDifferenceBetweenFamilies(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- def load_result():
- import json
-
- testInputs = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), "Inputs"
- )
- testOutput = os.path.join(testInputs, "test2_run.json")
- with open(testOutput, "r") as f:
- json = json.load(f)
- return json
-
- json = load_result()
- json1 = filter_benchmark(json, "BM_Z.ro", ".")
- json2 = filter_benchmark(json, "BM_O.e", ".")
- cls.json_diff_report = get_difference_report(json1, json2)
-
- def test_json_diff_report_pretty_printing(self):
- expect_lines = [
- [".", "-0.5000", "-0.5000", "10", "5", "10", "5"],
- ["./4", "-0.5000", "-0.5000", "40", "20", "40", "20"],
- ["Prefix/.", "-0.5000", "-0.5000", "20", "10", "20", "10"],
- ["Prefix/./3", "-0.5000", "-0.5000", "30", "15", "30", "15"],
- ["OVERALL_GEOMEAN", "-0.5000", "-0.5000", "0", "0", "0", "0"],
- ]
- output_lines_with_header = print_difference_report(
- self.json_diff_report, use_color=False
- )
- output_lines = output_lines_with_header[2:]
- print("\n")
- print("\n".join(output_lines_with_header))
- self.assertEqual(len(output_lines), len(expect_lines))
- for i in range(0, len(output_lines)):
- parts = [x for x in output_lines[i].split(" ") if x]
- self.assertEqual(len(parts), 7)
- self.assertEqual(expect_lines[i], parts)
-
- def test_json_diff_report(self):
- expected_output = [
- {
- "name": ".",
- "measurements": [
- {
- "time": -0.5,
- "cpu": -0.5,
- "real_time": 10,
- "real_time_other": 5,
- "cpu_time": 10,
- "cpu_time_other": 5,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "./4",
- "measurements": [
- {
- "time": -0.5,
- "cpu": -0.5,
- "real_time": 40,
- "real_time_other": 20,
- "cpu_time": 40,
- "cpu_time_other": 20,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "Prefix/.",
- "measurements": [
- {
- "time": -0.5,
- "cpu": -0.5,
- "real_time": 20,
- "real_time_other": 10,
- "cpu_time": 20,
- "cpu_time_other": 10,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "Prefix/./3",
- "measurements": [
- {
- "time": -0.5,
- "cpu": -0.5,
- "real_time": 30,
- "real_time_other": 15,
- "cpu_time": 30,
- "cpu_time_other": 15,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "OVERALL_GEOMEAN",
- "measurements": [
- {
- "real_time": 2.213363839400641e-08,
- "cpu_time": 2.213363839400641e-08,
- "real_time_other": 1.1066819197003185e-08,
- "cpu_time_other": 1.1066819197003185e-08,
- "time": -0.5000000000000009,
- "cpu": -0.5000000000000009,
- }
- ],
- "time_unit": "s",
- "run_type": "aggregate",
- "aggregate_name": "geomean",
- "utest": {},
- },
- ]
- self.assertEqual(len(self.json_diff_report), len(expected_output))
- for out, expected in zip(self.json_diff_report, expected_output):
- self.assertEqual(out["name"], expected["name"])
- self.assertEqual(out["time_unit"], expected["time_unit"])
- assert_utest(self, out, expected)
- assert_measurements(self, out, expected)
-
-
-class TestReportDifferenceWithUTest(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- def load_results():
- import json
-
- testInputs = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), "Inputs"
- )
- testOutput1 = os.path.join(testInputs, "test3_run0.json")
- testOutput2 = os.path.join(testInputs, "test3_run1.json")
- with open(testOutput1, "r") as f:
- json1 = json.load(f)
- with open(testOutput2, "r") as f:
- json2 = json.load(f)
- return json1, json2
-
- json1, json2 = load_results()
- cls.json_diff_report = get_difference_report(json1, json2, utest=True)
-
- def test_json_diff_report_pretty_printing(self):
- expect_lines = [
- ["BM_One", "-0.1000", "+0.1000", "10", "9", "100", "110"],
- ["BM_Two", "+0.1111", "-0.0111", "9", "10", "90", "89"],
- ["BM_Two", "-0.1250", "-0.1628", "8", "7", "86", "72"],
- [
- "BM_Two_pvalue",
- "1.0000",
- "0.6667",
- "U",
- "Test,",
- "Repetitions:",
- "2",
- "vs",
- "2.",
- "WARNING:",
- "Results",
- "unreliable!",
- "9+",
- "repetitions",
- "recommended.",
- ],
- ["short", "-0.1250", "-0.0625", "8", "7", "80", "75"],
- ["short", "-0.4325", "-0.1351", "8", "5", "77", "67"],
- [
- "short_pvalue",
- "0.7671",
- "0.2000",
- "U",
- "Test,",
- "Repetitions:",
- "2",
- "vs",
- "3.",
- "WARNING:",
- "Results",
- "unreliable!",
- "9+",
- "repetitions",
- "recommended.",
- ],
- ["medium", "-0.3750", "-0.3375", "8", "5", "80", "53"],
- ["OVERALL_GEOMEAN", "+1.6405", "-0.6985", "0", "0", "0", "0"],
- ]
- output_lines_with_header = print_difference_report(
- self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False
- )
- output_lines = output_lines_with_header[2:]
- print("\n")
- print("\n".join(output_lines_with_header))
- self.assertEqual(len(output_lines), len(expect_lines))
- for i in range(0, len(output_lines)):
- parts = [x for x in output_lines[i].split(" ") if x]
- self.assertEqual(expect_lines[i], parts)
-
- def test_json_diff_report_pretty_printing_aggregates_only(self):
- expect_lines = [
- ["BM_One", "-0.1000", "+0.1000", "10", "9", "100", "110"],
- [
- "BM_Two_pvalue",
- "1.0000",
- "0.6667",
- "U",
- "Test,",
- "Repetitions:",
- "2",
- "vs",
- "2.",
- "WARNING:",
- "Results",
- "unreliable!",
- "9+",
- "repetitions",
- "recommended.",
- ],
- ["short", "-0.1250", "-0.0625", "8", "7", "80", "75"],
- ["short", "-0.4325", "-0.1351", "8", "5", "77", "67"],
- [
- "short_pvalue",
- "0.7671",
- "0.2000",
- "U",
- "Test,",
- "Repetitions:",
- "2",
- "vs",
- "3.",
- "WARNING:",
- "Results",
- "unreliable!",
- "9+",
- "repetitions",
- "recommended.",
- ],
- ["OVERALL_GEOMEAN", "+1.6405", "-0.6985", "0", "0", "0", "0"],
- ]
- output_lines_with_header = print_difference_report(
- self.json_diff_report,
- include_aggregates_only=True,
- utest=True,
- utest_alpha=0.05,
- use_color=False,
- )
- output_lines = output_lines_with_header[2:]
- print("\n")
- print("\n".join(output_lines_with_header))
- self.assertEqual(len(output_lines), len(expect_lines))
- for i in range(0, len(output_lines)):
- parts = [x for x in output_lines[i].split(" ") if x]
- self.assertEqual(expect_lines[i], parts)
-
- def test_json_diff_report(self):
- expected_output = [
- {
- "name": "BM_One",
- "measurements": [
- {
- "time": -0.1,
- "cpu": 0.1,
- "real_time": 10,
- "real_time_other": 9,
- "cpu_time": 100,
- "cpu_time_other": 110,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "BM_Two",
- "measurements": [
- {
- "time": 0.1111111111111111,
- "cpu": -0.011111111111111112,
- "real_time": 9,
- "real_time_other": 10,
- "cpu_time": 90,
- "cpu_time_other": 89,
- },
- {
- "time": -0.125,
- "cpu": -0.16279069767441862,
- "real_time": 8,
- "real_time_other": 7,
- "cpu_time": 86,
- "cpu_time_other": 72,
- },
- ],
- "time_unit": "ns",
- "utest": {
- "have_optimal_repetitions": False,
- "cpu_pvalue": 0.6666666666666666,
- "time_pvalue": 1.0,
- },
- },
- {
- "name": "short",
- "measurements": [
- {
- "time": -0.125,
- "cpu": -0.0625,
- "real_time": 8,
- "real_time_other": 7,
- "cpu_time": 80,
- "cpu_time_other": 75,
- },
- {
- "time": -0.4325,
- "cpu": -0.13506493506493514,
- "real_time": 8,
- "real_time_other": 4.54,
- "cpu_time": 77,
- "cpu_time_other": 66.6,
- },
- ],
- "time_unit": "ns",
- "utest": {
- "have_optimal_repetitions": False,
- "cpu_pvalue": 0.2,
- "time_pvalue": 0.7670968684102772,
- },
- },
- {
- "name": "medium",
- "measurements": [
- {
- "time": -0.375,
- "cpu": -0.3375,
- "real_time": 8,
- "real_time_other": 5,
- "cpu_time": 80,
- "cpu_time_other": 53,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "OVERALL_GEOMEAN",
- "measurements": [
- {
- "real_time": 8.48528137423858e-09,
- "cpu_time": 8.441336246629233e-08,
- "real_time_other": 2.2405267593145244e-08,
- "cpu_time_other": 2.5453661413660466e-08,
- "time": 1.6404861082353634,
- "cpu": -0.6984640740519662,
- }
- ],
- "time_unit": "s",
- "run_type": "aggregate",
- "aggregate_name": "geomean",
- "utest": {},
- },
- ]
- self.assertEqual(len(self.json_diff_report), len(expected_output))
- for out, expected in zip(self.json_diff_report, expected_output):
- self.assertEqual(out["name"], expected["name"])
- self.assertEqual(out["time_unit"], expected["time_unit"])
- assert_utest(self, out, expected)
- assert_measurements(self, out, expected)
-
-
-class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly(
- unittest.TestCase
-):
- @classmethod
- def setUpClass(cls):
- def load_results():
- import json
-
- testInputs = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), "Inputs"
- )
- testOutput1 = os.path.join(testInputs, "test3_run0.json")
- testOutput2 = os.path.join(testInputs, "test3_run1.json")
- with open(testOutput1, "r") as f:
- json1 = json.load(f)
- with open(testOutput2, "r") as f:
- json2 = json.load(f)
- return json1, json2
-
- json1, json2 = load_results()
- cls.json_diff_report = get_difference_report(json1, json2, utest=True)
-
- def test_json_diff_report_pretty_printing(self):
- expect_lines = [
- ["BM_One", "-0.1000", "+0.1000", "10", "9", "100", "110"],
- ["BM_Two", "+0.1111", "-0.0111", "9", "10", "90", "89"],
- ["BM_Two", "-0.1250", "-0.1628", "8", "7", "86", "72"],
- [
- "BM_Two_pvalue",
- "1.0000",
- "0.6667",
- "U",
- "Test,",
- "Repetitions:",
- "2",
- "vs",
- "2.",
- "WARNING:",
- "Results",
- "unreliable!",
- "9+",
- "repetitions",
- "recommended.",
- ],
- ["short", "-0.1250", "-0.0625", "8", "7", "80", "75"],
- ["short", "-0.4325", "-0.1351", "8", "5", "77", "67"],
- [
- "short_pvalue",
- "0.7671",
- "0.2000",
- "U",
- "Test,",
- "Repetitions:",
- "2",
- "vs",
- "3.",
- "WARNING:",
- "Results",
- "unreliable!",
- "9+",
- "repetitions",
- "recommended.",
- ],
- ["medium", "-0.3750", "-0.3375", "8", "5", "80", "53"],
- ["OVERALL_GEOMEAN", "+1.6405", "-0.6985", "0", "0", "0", "0"],
- ]
- output_lines_with_header = print_difference_report(
- self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False
- )
- output_lines = output_lines_with_header[2:]
- print("\n")
- print("\n".join(output_lines_with_header))
- self.assertEqual(len(output_lines), len(expect_lines))
- for i in range(0, len(output_lines)):
- parts = [x for x in output_lines[i].split(" ") if x]
- self.assertEqual(expect_lines[i], parts)
-
- def test_json_diff_report(self):
- expected_output = [
- {
- "name": "BM_One",
- "measurements": [
- {
- "time": -0.1,
- "cpu": 0.1,
- "real_time": 10,
- "real_time_other": 9,
- "cpu_time": 100,
- "cpu_time_other": 110,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- },
- {
- "name": "BM_Two",
- "measurements": [
- {
- "time": 0.1111111111111111,
- "cpu": -0.011111111111111112,
- "real_time": 9,
- "real_time_other": 10,
- "cpu_time": 90,
- "cpu_time_other": 89,
- },
- {
- "time": -0.125,
- "cpu": -0.16279069767441862,
- "real_time": 8,
- "real_time_other": 7,
- "cpu_time": 86,
- "cpu_time_other": 72,
- },
- ],
- "time_unit": "ns",
- "utest": {
- "have_optimal_repetitions": False,
- "cpu_pvalue": 0.6666666666666666,
- "time_pvalue": 1.0,
- },
- },
- {
- "name": "short",
- "measurements": [
- {
- "time": -0.125,
- "cpu": -0.0625,
- "real_time": 8,
- "real_time_other": 7,
- "cpu_time": 80,
- "cpu_time_other": 75,
- },
- {
- "time": -0.4325,
- "cpu": -0.13506493506493514,
- "real_time": 8,
- "real_time_other": 4.54,
- "cpu_time": 77,
- "cpu_time_other": 66.6,
- },
- ],
- "time_unit": "ns",
- "utest": {
- "have_optimal_repetitions": False,
- "cpu_pvalue": 0.2,
- "time_pvalue": 0.7670968684102772,
- },
- },
- {
- "name": "medium",
- "measurements": [
- {
- "real_time_other": 5,
- "cpu_time": 80,
- "time": -0.375,
- "real_time": 8,
- "cpu_time_other": 53,
- "cpu": -0.3375,
- }
- ],
- "utest": {},
- "time_unit": "ns",
- "aggregate_name": "",
- },
- {
- "name": "OVERALL_GEOMEAN",
- "measurements": [
- {
- "real_time": 8.48528137423858e-09,
- "cpu_time": 8.441336246629233e-08,
- "real_time_other": 2.2405267593145244e-08,
- "cpu_time_other": 2.5453661413660466e-08,
- "time": 1.6404861082353634,
- "cpu": -0.6984640740519662,
- }
- ],
- "time_unit": "s",
- "run_type": "aggregate",
- "aggregate_name": "geomean",
- "utest": {},
- },
- ]
- self.assertEqual(len(self.json_diff_report), len(expected_output))
- for out, expected in zip(self.json_diff_report, expected_output):
- self.assertEqual(out["name"], expected["name"])
- self.assertEqual(out["time_unit"], expected["time_unit"])
- assert_utest(self, out, expected)
- assert_measurements(self, out, expected)
-
-
-class TestReportDifferenceForPercentageAggregates(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- def load_results():
- import json
-
- testInputs = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), "Inputs"
- )
- testOutput1 = os.path.join(testInputs, "test4_run0.json")
- testOutput2 = os.path.join(testInputs, "test4_run1.json")
- with open(testOutput1, "r") as f:
- json1 = json.load(f)
- with open(testOutput2, "r") as f:
- json2 = json.load(f)
- return json1, json2
-
- json1, json2 = load_results()
- cls.json_diff_report = get_difference_report(json1, json2, utest=True)
-
- def test_json_diff_report_pretty_printing(self):
- expect_lines = [["whocares", "-0.5000", "+0.5000", "0", "0", "0", "0"]]
- output_lines_with_header = print_difference_report(
- self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False
- )
- output_lines = output_lines_with_header[2:]
- print("\n")
- print("\n".join(output_lines_with_header))
- self.assertEqual(len(output_lines), len(expect_lines))
- for i in range(0, len(output_lines)):
- parts = [x for x in output_lines[i].split(" ") if x]
- self.assertEqual(expect_lines[i], parts)
-
- def test_json_diff_report(self):
- expected_output = [
- {
- "name": "whocares",
- "measurements": [
- {
- "time": -0.5,
- "cpu": 0.5,
- "real_time": 0.01,
- "real_time_other": 0.005,
- "cpu_time": 0.10,
- "cpu_time_other": 0.15,
- }
- ],
- "time_unit": "ns",
- "utest": {},
- }
- ]
- self.assertEqual(len(self.json_diff_report), len(expected_output))
- for out, expected in zip(self.json_diff_report, expected_output):
- self.assertEqual(out["name"], expected["name"])
- self.assertEqual(out["time_unit"], expected["time_unit"])
- assert_utest(self, out, expected)
- assert_measurements(self, out, expected)
-
-
-class TestReportSorting(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- def load_result():
- import json
-
- testInputs = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), "Inputs"
- )
- testOutput = os.path.join(testInputs, "test4_run.json")
- with open(testOutput, "r") as f:
- json = json.load(f)
- return json
-
- cls.json = load_result()
-
- def test_json_diff_report_pretty_printing(self):
- import util
-
- expected_names = [
- "99 family 0 instance 0 repetition 0",
- "98 family 0 instance 0 repetition 1",
- "97 family 0 instance 0 aggregate",
- "96 family 0 instance 1 repetition 0",
- "95 family 0 instance 1 repetition 1",
- "94 family 0 instance 1 aggregate",
- "93 family 1 instance 0 repetition 0",
- "92 family 1 instance 0 repetition 1",
- "91 family 1 instance 0 aggregate",
- "90 family 1 instance 1 repetition 0",
- "89 family 1 instance 1 repetition 1",
- "88 family 1 instance 1 aggregate",
- ]
-
- for n in range(len(self.json["benchmarks"]) ** 2):
- random.shuffle(self.json["benchmarks"])
- sorted_benchmarks = util.sort_benchmark_results(self.json)[
- "benchmarks"
- ]
- self.assertEqual(len(expected_names), len(sorted_benchmarks))
- for out, expected in zip(sorted_benchmarks, expected_names):
- self.assertEqual(out["name"], expected)
-
-
-class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly2(
- unittest.TestCase
-):
- @classmethod
- def setUpClass(cls):
- def load_results():
- import json
-
- testInputs = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), "Inputs"
- )
- testOutput1 = os.path.join(testInputs, "test5_run0.json")
- testOutput2 = os.path.join(testInputs, "test5_run1.json")
- with open(testOutput1, "r") as f:
- json1 = json.load(f)
- json1["benchmarks"] = [
- json1["benchmarks"][0] for i in range(1000)
- ]
- with open(testOutput2, "r") as f:
- json2 = json.load(f)
- json2["benchmarks"] = [
- json2["benchmarks"][0] for i in range(1000)
- ]
- return json1, json2
-
- json1, json2 = load_results()
- cls.json_diff_report = get_difference_report(json1, json2, utest=True)
-
- def test_json_diff_report_pretty_printing(self):
- expect_line = [
- "BM_ManyRepetitions_pvalue",
- "0.0000",
- "0.0000",
- "U",
- "Test,",
- "Repetitions:",
- "1000",
- "vs",
- "1000",
- ]
- output_lines_with_header = print_difference_report(
- self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False
- )
- output_lines = output_lines_with_header[2:]
- found = False
- for i in range(0, len(output_lines)):
- parts = [x for x in output_lines[i].split(" ") if x]
- found = expect_line == parts
- if found:
- break
- self.assertTrue(found)
-
- def test_json_diff_report(self):
- expected_output = [
- {
- "name": "BM_ManyRepetitions",
- "label": "",
- "time_unit": "s",
- "run_type": "",
- "aggregate_name": "",
- "utest": {
- "have_optimal_repetitions": True,
- "cpu_pvalue": 0.0,
- "time_pvalue": 0.0,
- "nr_of_repetitions": 1000,
- "nr_of_repetitions_other": 1000,
- },
- },
- {
- "name": "OVERALL_GEOMEAN",
- "label": "",
- "measurements": [
- {
- "real_time": 1.0,
- "cpu_time": 1000.000000000069,
- "real_time_other": 1000.000000000069,
- "cpu_time_other": 1.0,
- "time": 999.000000000069,
- "cpu": -0.9990000000000001,
- }
- ],
- "time_unit": "s",
- "run_type": "aggregate",
- "aggregate_name": "geomean",
- "utest": {},
- },
- ]
- self.assertEqual(len(self.json_diff_report), len(expected_output))
- for out, expected in zip(self.json_diff_report, expected_output):
- self.assertEqual(out["name"], expected["name"])
- self.assertEqual(out["time_unit"], expected["time_unit"])
- assert_utest(self, out, expected)
-
-
-def assert_utest(unittest_instance, lhs, rhs):
- if lhs["utest"]:
- unittest_instance.assertAlmostEqual(
- lhs["utest"]["cpu_pvalue"], rhs["utest"]["cpu_pvalue"]
- )
- unittest_instance.assertAlmostEqual(
- lhs["utest"]["time_pvalue"], rhs["utest"]["time_pvalue"]
- )
- unittest_instance.assertEqual(
- lhs["utest"]["have_optimal_repetitions"],
- rhs["utest"]["have_optimal_repetitions"],
- )
- else:
- # lhs is empty. assert if rhs is not.
- unittest_instance.assertEqual(lhs["utest"], rhs["utest"])
-
-
-def assert_measurements(unittest_instance, lhs, rhs):
- for m1, m2 in zip(lhs["measurements"], rhs["measurements"]):
- unittest_instance.assertEqual(m1["real_time"], m2["real_time"])
- unittest_instance.assertEqual(m1["cpu_time"], m2["cpu_time"])
- # m1['time'] and m1['cpu'] hold values which are being calculated,
- # and therefore we must use almost-equal pattern.
- unittest_instance.assertAlmostEqual(m1["time"], m2["time"], places=4)
- unittest_instance.assertAlmostEqual(m1["cpu"], m2["cpu"], places=4)
-
-
-if __name__ == "__main__":
- unittest.main()
-
-# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
-# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;
-# kate: indent-mode python; remove-trailing-spaces modified;
diff --git a/contrib/restricted/google/benchmark/tools/compare/gbench/util.py b/contrib/restricted/google/benchmark/tools/compare/gbench/util.py
deleted file mode 100644
index 1119a1a2ca..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/gbench/util.py
+++ /dev/null
@@ -1,231 +0,0 @@
-"""util.py - General utilities for running, loading, and processing benchmarks"""
-
-import json
-import os
-import re
-import subprocess
-import sys
-import tempfile
-
-# Input file type enumeration
-IT_Invalid = 0
-IT_JSON = 1
-IT_Executable = 2
-
-_num_magic_bytes = 2 if sys.platform.startswith("win") else 4
-
-
-def is_executable_file(filename):
- """
- Return 'True' if 'filename' names a valid file which is likely
- an executable. A file is considered an executable if it starts with the
- magic bytes for a EXE, Mach O, or ELF file.
- """
- if not os.path.isfile(filename):
- return False
- with open(filename, mode="rb") as f:
- magic_bytes = f.read(_num_magic_bytes)
- if sys.platform == "darwin":
- return magic_bytes in [
- b"\xfe\xed\xfa\xce", # MH_MAGIC
- b"\xce\xfa\xed\xfe", # MH_CIGAM
- b"\xfe\xed\xfa\xcf", # MH_MAGIC_64
- b"\xcf\xfa\xed\xfe", # MH_CIGAM_64
- b"\xca\xfe\xba\xbe", # FAT_MAGIC
- b"\xbe\xba\xfe\xca", # FAT_CIGAM
- ]
- elif sys.platform.startswith("win"):
- return magic_bytes == b"MZ"
- else:
- return magic_bytes == b"\x7fELF"
-
-
-def is_json_file(filename):
- """
- Returns 'True' if 'filename' names a valid JSON output file.
- 'False' otherwise.
- """
- try:
- with open(filename, "r") as f:
- json.load(f)
- return True
- except BaseException:
- pass
- return False
-
-
-def classify_input_file(filename):
- """
- Return a tuple (type, msg) where 'type' specifies the classified type
- of 'filename'. If 'type' is 'IT_Invalid' then 'msg' is a human readable
- string representing the error.
- """
- ftype = IT_Invalid
- err_msg = None
- if not os.path.exists(filename):
- err_msg = "'%s' does not exist" % filename
- elif not os.path.isfile(filename):
- err_msg = "'%s' does not name a file" % filename
- elif is_executable_file(filename):
- ftype = IT_Executable
- elif is_json_file(filename):
- ftype = IT_JSON
- else:
- err_msg = (
- "'%s' does not name a valid benchmark executable or JSON file"
- % filename
- )
- return ftype, err_msg
-
-
-def check_input_file(filename):
- """
- Classify the file named by 'filename' and return the classification.
- If the file is classified as 'IT_Invalid' print an error message and exit
- the program.
- """
- ftype, msg = classify_input_file(filename)
- if ftype == IT_Invalid:
- print("Invalid input file: %s" % msg)
- sys.exit(1)
- return ftype
-
-
-def find_benchmark_flag(prefix, benchmark_flags):
- """
- Search the specified list of flags for a flag matching `<prefix><arg>` and
- if it is found return the arg it specifies. If specified more than once the
- last value is returned. If the flag is not found None is returned.
- """
- assert prefix.startswith("--") and prefix.endswith("=")
- result = None
- for f in benchmark_flags:
- if f.startswith(prefix):
- result = f[len(prefix) :]
- return result
-
-
-def remove_benchmark_flags(prefix, benchmark_flags):
- """
- Return a new list containing the specified benchmark_flags except those
- with the specified prefix.
- """
- assert prefix.startswith("--") and prefix.endswith("=")
- return [f for f in benchmark_flags if not f.startswith(prefix)]
-
-
-def load_benchmark_results(fname, benchmark_filter):
- """
- Read benchmark output from a file and return the JSON object.
-
- Apply benchmark_filter, a regular expression, with nearly the same
- semantics of the --benchmark_filter argument. May be None.
- Note: the Python regular expression engine is used instead of the
- one used by the C++ code, which may produce different results
- in complex cases.
-
- REQUIRES: 'fname' names a file containing JSON benchmark output.
- """
-
- def benchmark_wanted(benchmark):
- if benchmark_filter is None:
- return True
- name = benchmark.get("run_name", None) or benchmark["name"]
- return re.search(benchmark_filter, name) is not None
-
- with open(fname, "r") as f:
- results = json.load(f)
- if "context" in results:
- if "json_schema_version" in results["context"]:
- json_schema_version = results["context"]["json_schema_version"]
- if json_schema_version != 1:
- print(
- "In %s, got unnsupported JSON schema version: %i, expected 1"
- % (fname, json_schema_version)
- )
- sys.exit(1)
- if "benchmarks" in results:
- results["benchmarks"] = list(
- filter(benchmark_wanted, results["benchmarks"])
- )
- return results
-
-
-def sort_benchmark_results(result):
- benchmarks = result["benchmarks"]
-
- # From inner key to the outer key!
- benchmarks = sorted(
- benchmarks,
- key=lambda benchmark: benchmark["repetition_index"]
- if "repetition_index" in benchmark
- else -1,
- )
- benchmarks = sorted(
- benchmarks,
- key=lambda benchmark: 1
- if "run_type" in benchmark and benchmark["run_type"] == "aggregate"
- else 0,
- )
- benchmarks = sorted(
- benchmarks,
- key=lambda benchmark: benchmark["per_family_instance_index"]
- if "per_family_instance_index" in benchmark
- else -1,
- )
- benchmarks = sorted(
- benchmarks,
- key=lambda benchmark: benchmark["family_index"]
- if "family_index" in benchmark
- else -1,
- )
-
- result["benchmarks"] = benchmarks
- return result
-
-
-def run_benchmark(exe_name, benchmark_flags):
- """
- Run a benchmark specified by 'exe_name' with the specified
- 'benchmark_flags'. The benchmark is run directly as a subprocess to preserve
- real time console output.
- RETURNS: A JSON object representing the benchmark output
- """
- output_name = find_benchmark_flag("--benchmark_out=", benchmark_flags)
- is_temp_output = False
- if output_name is None:
- is_temp_output = True
- thandle, output_name = tempfile.mkstemp()
- os.close(thandle)
- benchmark_flags = list(benchmark_flags) + [
- "--benchmark_out=%s" % output_name
- ]
-
- cmd = [exe_name] + benchmark_flags
- print("RUNNING: %s" % " ".join(cmd))
- exitCode = subprocess.call(cmd)
- if exitCode != 0:
- print("TEST FAILED...")
- sys.exit(exitCode)
- json_res = load_benchmark_results(output_name, None)
- if is_temp_output:
- os.unlink(output_name)
- return json_res
-
-
-def run_or_load_benchmark(filename, benchmark_flags):
- """
- Get the results for a specified benchmark. If 'filename' specifies
- an executable benchmark then the results are generated by running the
- benchmark. Otherwise 'filename' must name a valid JSON output file,
- which is loaded and the result returned.
- """
- ftype = check_input_file(filename)
- if ftype == IT_JSON:
- benchmark_filter = find_benchmark_flag(
- "--benchmark_filter=", benchmark_flags
- )
- return load_benchmark_results(filename, benchmark_filter)
- if ftype == IT_Executable:
- return run_benchmark(filename, benchmark_flags)
- raise ValueError("Unknown file type %s" % ftype)
diff --git a/contrib/restricted/google/benchmark/tools/compare/ya.make b/contrib/restricted/google/benchmark/tools/compare/ya.make
deleted file mode 100644
index 0d4aa76cfb..0000000000
--- a/contrib/restricted/google/benchmark/tools/compare/ya.make
+++ /dev/null
@@ -1,30 +0,0 @@
-# Generated by devtools/yamaker.
-
-PY3_PROGRAM()
-
-WITHOUT_LICENSE_TEXTS()
-
-VERSION(1.9.1)
-
-ORIGINAL_SOURCE(https://github.com/google/benchmark/archive/v1.9.1.tar.gz)
-
-LICENSE(Apache-2.0)
-
-PEERDIR(
- contrib/python/numpy
- contrib/python/scipy
-)
-
-NO_LINT()
-
-PY_MAIN(compare)
-
-PY_SRCS(
- TOP_LEVEL
- compare.py
- gbench/__init__.py
- gbench/report.py
- gbench/util.py
-)
-
-END()