1620 lines
55 KiB
Python
1620 lines
55 KiB
Python
# type: ignore
|
|
|
|
"""
|
|
report.py - Utilities for reporting statistics about benchmark results
|
|
"""
|
|
|
|
import copy
|
|
import os
|
|
import random
|
|
import re
|
|
import unittest
|
|
|
|
from numpy import array
|
|
from scipy.stats import gmean, mannwhitneyu
|
|
|
|
|
|
class BenchmarkColor(object):
|
|
def __init__(self, name, code):
|
|
self.name = name
|
|
self.code = code
|
|
|
|
def __repr__(self):
|
|
return "%s%r" % (self.__class__.__name__, (self.name, self.code))
|
|
|
|
def __format__(self, format):
|
|
return self.code
|
|
|
|
|
|
# Benchmark Colors Enumeration
|
|
BC_NONE = BenchmarkColor("NONE", "")
|
|
BC_MAGENTA = BenchmarkColor("MAGENTA", "\033[95m")
|
|
BC_CYAN = BenchmarkColor("CYAN", "\033[96m")
|
|
BC_OKBLUE = BenchmarkColor("OKBLUE", "\033[94m")
|
|
BC_OKGREEN = BenchmarkColor("OKGREEN", "\033[32m")
|
|
BC_HEADER = BenchmarkColor("HEADER", "\033[92m")
|
|
BC_WARNING = BenchmarkColor("WARNING", "\033[93m")
|
|
BC_WHITE = BenchmarkColor("WHITE", "\033[97m")
|
|
BC_FAIL = BenchmarkColor("FAIL", "\033[91m")
|
|
BC_ENDC = BenchmarkColor("ENDC", "\033[0m")
|
|
BC_BOLD = BenchmarkColor("BOLD", "\033[1m")
|
|
BC_UNDERLINE = BenchmarkColor("UNDERLINE", "\033[4m")
|
|
|
|
UTEST_MIN_REPETITIONS = 2
|
|
UTEST_OPTIMAL_REPETITIONS = 9 # Lowest reasonable number, More is better.
|
|
UTEST_COL_NAME = "_pvalue"
|
|
|
|
_TIME_UNIT_TO_SECONDS_MULTIPLIER = {
|
|
"s": 1.0,
|
|
"ms": 1e-3,
|
|
"us": 1e-6,
|
|
"ns": 1e-9,
|
|
}
|
|
|
|
|
|
def color_format(use_color, fmt_str, *args, **kwargs):
|
|
"""
|
|
Return the result of 'fmt_str.format(*args, **kwargs)' after transforming
|
|
'args' and 'kwargs' according to the value of 'use_color'. If 'use_color'
|
|
is False then all color codes in 'args' and 'kwargs' are replaced with
|
|
the empty string.
|
|
"""
|
|
assert use_color is True or use_color is False
|
|
if not use_color:
|
|
args = [
|
|
arg if not isinstance(arg, BenchmarkColor) else BC_NONE
|
|
for arg in args
|
|
]
|
|
kwargs = {
|
|
key: arg if not isinstance(arg, BenchmarkColor) else BC_NONE
|
|
for key, arg in kwargs.items()
|
|
}
|
|
return fmt_str.format(*args, **kwargs)
|
|
|
|
|
|
def find_longest_name(benchmark_list):
|
|
"""
|
|
Return the length of the longest benchmark name in a given list of
|
|
benchmark JSON objects
|
|
"""
|
|
longest_name = 1
|
|
for bc in benchmark_list:
|
|
if len(bc["name"]) > longest_name:
|
|
longest_name = len(bc["name"])
|
|
return longest_name
|
|
|
|
|
|
def calculate_change(old_val, new_val):
|
|
"""
|
|
Return a float representing the decimal change between old_val and new_val.
|
|
"""
|
|
if old_val == 0 and new_val == 0:
|
|
return 0.0
|
|
if old_val == 0:
|
|
return float(new_val - old_val) / (float(old_val + new_val) / 2)
|
|
return float(new_val - old_val) / abs(old_val)
|
|
|
|
|
|
def filter_benchmark(json_orig, family, replacement=""):
|
|
"""
|
|
Apply a filter to the json, and only leave the 'family' of benchmarks.
|
|
"""
|
|
regex = re.compile(family)
|
|
filtered = {}
|
|
filtered["benchmarks"] = []
|
|
for be in json_orig["benchmarks"]:
|
|
if not regex.search(be["name"]):
|
|
continue
|
|
filteredbench = copy.deepcopy(be) # Do NOT modify the old name!
|
|
filteredbench["name"] = regex.sub(replacement, filteredbench["name"])
|
|
filtered["benchmarks"].append(filteredbench)
|
|
return filtered
|
|
|
|
|
|
def get_unique_benchmark_names(json):
|
|
"""
|
|
While *keeping* the order, give all the unique 'names' used for benchmarks.
|
|
"""
|
|
seen = set()
|
|
uniqued = [
|
|
x["name"]
|
|
for x in json["benchmarks"]
|
|
if x["name"] not in seen and (seen.add(x["name"]) or True)
|
|
]
|
|
return uniqued
|
|
|
|
|
|
def intersect(list1, list2):
|
|
"""
|
|
Given two lists, get a new list consisting of the elements only contained
|
|
in *both of the input lists*, while preserving the ordering.
|
|
"""
|
|
return [x for x in list1 if x in list2]
|
|
|
|
|
|
def is_potentially_comparable_benchmark(x):
|
|
return "time_unit" in x and "real_time" in x and "cpu_time" in x
|
|
|
|
|
|
def partition_benchmarks(json1, json2):
|
|
"""
|
|
While preserving the ordering, find benchmarks with the same names in
|
|
both of the inputs, and group them.
|
|
(i.e. partition/filter into groups with common name)
|
|
"""
|
|
json1_unique_names = get_unique_benchmark_names(json1)
|
|
json2_unique_names = get_unique_benchmark_names(json2)
|
|
names = intersect(json1_unique_names, json2_unique_names)
|
|
partitions = []
|
|
for name in names:
|
|
time_unit = None
|
|
# Pick the time unit from the first entry of the lhs benchmark.
|
|
# We should be careful not to crash with unexpected input.
|
|
for x in json1["benchmarks"]:
|
|
if x["name"] == name and is_potentially_comparable_benchmark(x):
|
|
time_unit = x["time_unit"]
|
|
break
|
|
if time_unit is None:
|
|
continue
|
|
# Filter by name and time unit.
|
|
# All the repetitions are assumed to be comparable.
|
|
lhs = [
|
|
x
|
|
for x in json1["benchmarks"]
|
|
if x["name"] == name and x["time_unit"] == time_unit
|
|
]
|
|
rhs = [
|
|
x
|
|
for x in json2["benchmarks"]
|
|
if x["name"] == name and x["time_unit"] == time_unit
|
|
]
|
|
partitions.append([lhs, rhs])
|
|
return partitions
|
|
|
|
|
|
def get_timedelta_field_as_seconds(benchmark, field_name):
|
|
"""
|
|
Get value of field_name field of benchmark, which is time with time unit
|
|
time_unit, as time in seconds.
|
|
"""
|
|
timedelta = benchmark[field_name]
|
|
time_unit = benchmark.get("time_unit", "s")
|
|
return timedelta * _TIME_UNIT_TO_SECONDS_MULTIPLIER.get(time_unit)
|
|
|
|
|
|
def calculate_geomean(json):
|
|
"""
|
|
Extract all real/cpu times from all the benchmarks as seconds,
|
|
and calculate their geomean.
|
|
"""
|
|
times = []
|
|
for benchmark in json["benchmarks"]:
|
|
if "run_type" in benchmark and benchmark["run_type"] == "aggregate":
|
|
continue
|
|
times.append(
|
|
[
|
|
get_timedelta_field_as_seconds(benchmark, "real_time"),
|
|
get_timedelta_field_as_seconds(benchmark, "cpu_time"),
|
|
]
|
|
)
|
|
return gmean(times) if times else array([])
|
|
|
|
|
|
def extract_field(partition, field_name):
|
|
# The count of elements may be different. We want *all* of them.
|
|
lhs = [x[field_name] for x in partition[0]]
|
|
rhs = [x[field_name] for x in partition[1]]
|
|
return [lhs, rhs]
|
|
|
|
|
|
def calc_utest(timings_cpu, timings_time):
|
|
min_rep_cnt = min(
|
|
len(timings_time[0]),
|
|
len(timings_time[1]),
|
|
len(timings_cpu[0]),
|
|
len(timings_cpu[1]),
|
|
)
|
|
|
|
# Does *everything* has at least UTEST_MIN_REPETITIONS repetitions?
|
|
if min_rep_cnt < UTEST_MIN_REPETITIONS:
|
|
return False, None, None
|
|
|
|
time_pvalue = mannwhitneyu(
|
|
timings_time[0], timings_time[1], alternative="two-sided"
|
|
).pvalue
|
|
cpu_pvalue = mannwhitneyu(
|
|
timings_cpu[0], timings_cpu[1], alternative="two-sided"
|
|
).pvalue
|
|
|
|
return (min_rep_cnt >= UTEST_OPTIMAL_REPETITIONS), cpu_pvalue, time_pvalue
|
|
|
|
|
|
def print_utest(bc_name, utest, utest_alpha, first_col_width, use_color=True):
|
|
def get_utest_color(pval):
|
|
return BC_FAIL if pval >= utest_alpha else BC_OKGREEN
|
|
|
|
# Check if we failed miserably with minimum required repetitions for utest
|
|
if (
|
|
not utest["have_optimal_repetitions"]
|
|
and utest["cpu_pvalue"] is None
|
|
and utest["time_pvalue"] is None
|
|
):
|
|
return []
|
|
|
|
dsc = "U Test, Repetitions: {} vs {}".format(
|
|
utest["nr_of_repetitions"], utest["nr_of_repetitions_other"]
|
|
)
|
|
dsc_color = BC_OKGREEN
|
|
|
|
# We still got some results to show but issue a warning about it.
|
|
if not utest["have_optimal_repetitions"]:
|
|
dsc_color = BC_WARNING
|
|
dsc += ". WARNING: Results unreliable! {}+ repetitions recommended.".format(
|
|
UTEST_OPTIMAL_REPETITIONS
|
|
)
|
|
|
|
special_str = "{}{:<{}s}{endc}{}{:16.4f}{endc}{}{:16.4f}{endc}{} {}"
|
|
|
|
return [
|
|
color_format(
|
|
use_color,
|
|
special_str,
|
|
BC_HEADER,
|
|
"{}{}".format(bc_name, UTEST_COL_NAME),
|
|
first_col_width,
|
|
get_utest_color(utest["time_pvalue"]),
|
|
utest["time_pvalue"],
|
|
get_utest_color(utest["cpu_pvalue"]),
|
|
utest["cpu_pvalue"],
|
|
dsc_color,
|
|
dsc,
|
|
endc=BC_ENDC,
|
|
)
|
|
]
|
|
|
|
|
|
def get_difference_report(json1, json2, utest=False):
|
|
"""
|
|
Calculate and report the difference between each test of two benchmarks
|
|
runs specified as 'json1' and 'json2'. Output is another json containing
|
|
relevant details for each test run.
|
|
"""
|
|
assert utest is True or utest is False
|
|
|
|
diff_report = []
|
|
partitions = partition_benchmarks(json1, json2)
|
|
for partition in partitions:
|
|
benchmark_name = partition[0][0]["name"]
|
|
label = partition[0][0]["label"] if "label" in partition[0][0] else ""
|
|
time_unit = partition[0][0]["time_unit"]
|
|
measurements = []
|
|
utest_results = {}
|
|
# Careful, we may have different repetition count.
|
|
for i in range(min(len(partition[0]), len(partition[1]))):
|
|
bn = partition[0][i]
|
|
other_bench = partition[1][i]
|
|
measurements.append(
|
|
{
|
|
"real_time": bn["real_time"],
|
|
"cpu_time": bn["cpu_time"],
|
|
"real_time_other": other_bench["real_time"],
|
|
"cpu_time_other": other_bench["cpu_time"],
|
|
"time": calculate_change(
|
|
bn["real_time"], other_bench["real_time"]
|
|
),
|
|
"cpu": calculate_change(
|
|
bn["cpu_time"], other_bench["cpu_time"]
|
|
),
|
|
}
|
|
)
|
|
|
|
# After processing the whole partition, if requested, do the U test.
|
|
if utest:
|
|
timings_cpu = extract_field(partition, "cpu_time")
|
|
timings_time = extract_field(partition, "real_time")
|
|
have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest(
|
|
timings_cpu, timings_time
|
|
)
|
|
if cpu_pvalue is not None and time_pvalue is not None:
|
|
utest_results = {
|
|
"have_optimal_repetitions": have_optimal_repetitions,
|
|
"cpu_pvalue": cpu_pvalue,
|
|
"time_pvalue": time_pvalue,
|
|
"nr_of_repetitions": len(timings_cpu[0]),
|
|
"nr_of_repetitions_other": len(timings_cpu[1]),
|
|
}
|
|
|
|
# Store only if we had any measurements for given benchmark.
|
|
# E.g. partition_benchmarks will filter out the benchmarks having
|
|
# time units which are not compatible with other time units in the
|
|
# benchmark suite.
|
|
if measurements:
|
|
run_type = (
|
|
partition[0][0]["run_type"]
|
|
if "run_type" in partition[0][0]
|
|
else ""
|
|
)
|
|
aggregate_name = (
|
|
partition[0][0]["aggregate_name"]
|
|
if run_type == "aggregate"
|
|
and "aggregate_name" in partition[0][0]
|
|
else ""
|
|
)
|
|
diff_report.append(
|
|
{
|
|
"name": benchmark_name,
|
|
"label": label,
|
|
"measurements": measurements,
|
|
"time_unit": time_unit,
|
|
"run_type": run_type,
|
|
"aggregate_name": aggregate_name,
|
|
"utest": utest_results,
|
|
}
|
|
)
|
|
|
|
lhs_gmean = calculate_geomean(json1)
|
|
rhs_gmean = calculate_geomean(json2)
|
|
if lhs_gmean.any() and rhs_gmean.any():
|
|
diff_report.append(
|
|
{
|
|
"name": "OVERALL_GEOMEAN",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"real_time": lhs_gmean[0],
|
|
"cpu_time": lhs_gmean[1],
|
|
"real_time_other": rhs_gmean[0],
|
|
"cpu_time_other": rhs_gmean[1],
|
|
"time": calculate_change(lhs_gmean[0], rhs_gmean[0]),
|
|
"cpu": calculate_change(lhs_gmean[1], rhs_gmean[1]),
|
|
}
|
|
],
|
|
"time_unit": "s",
|
|
"run_type": "aggregate",
|
|
"aggregate_name": "geomean",
|
|
"utest": {},
|
|
}
|
|
)
|
|
|
|
return diff_report
|
|
|
|
|
|
def print_difference_report(
|
|
json_diff_report,
|
|
include_aggregates_only=False,
|
|
utest=False,
|
|
utest_alpha=0.05,
|
|
use_color=True,
|
|
):
|
|
"""
|
|
Calculate and report the difference between each test of two benchmarks
|
|
runs specified as 'json1' and 'json2'.
|
|
"""
|
|
assert utest is True or utest is False
|
|
|
|
def get_color(res):
|
|
if res > 0.05:
|
|
return BC_FAIL
|
|
elif res > -0.07:
|
|
return BC_WHITE
|
|
else:
|
|
return BC_CYAN
|
|
|
|
first_col_width = find_longest_name(json_diff_report)
|
|
first_col_width = max(first_col_width, len("Benchmark"))
|
|
first_col_width += len(UTEST_COL_NAME)
|
|
first_line = "{:<{}s}Time CPU Time Old Time New CPU Old CPU New".format(
|
|
"Benchmark", 12 + first_col_width
|
|
)
|
|
output_strs = [first_line, "-" * len(first_line)]
|
|
|
|
fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}"
|
|
for benchmark in json_diff_report:
|
|
# *If* we were asked to only include aggregates,
|
|
# and if it is non-aggregate, then don't print it.
|
|
if (
|
|
not include_aggregates_only
|
|
or "run_type" not in benchmark
|
|
or benchmark["run_type"] == "aggregate"
|
|
):
|
|
for measurement in benchmark["measurements"]:
|
|
output_strs += [
|
|
color_format(
|
|
use_color,
|
|
fmt_str,
|
|
BC_HEADER,
|
|
benchmark["name"],
|
|
first_col_width,
|
|
get_color(measurement["time"]),
|
|
measurement["time"],
|
|
get_color(measurement["cpu"]),
|
|
measurement["cpu"],
|
|
measurement["real_time"],
|
|
measurement["real_time_other"],
|
|
measurement["cpu_time"],
|
|
measurement["cpu_time_other"],
|
|
endc=BC_ENDC,
|
|
)
|
|
]
|
|
|
|
# After processing the measurements, if requested and
|
|
# if applicable (e.g. u-test exists for given benchmark),
|
|
# print the U test.
|
|
if utest and benchmark["utest"]:
|
|
output_strs += print_utest(
|
|
benchmark["name"],
|
|
benchmark["utest"],
|
|
utest_alpha=utest_alpha,
|
|
first_col_width=first_col_width,
|
|
use_color=use_color,
|
|
)
|
|
|
|
return output_strs
|
|
|
|
|
|
###############################################################################
|
|
# Unit tests
|
|
|
|
|
|
class TestGetUniqueBenchmarkNames(unittest.TestCase):
|
|
def load_results(self):
|
|
import json
|
|
|
|
testInputs = os.path.join(
|
|
os.path.dirname(os.path.realpath(__file__)), "Inputs"
|
|
)
|
|
testOutput = os.path.join(testInputs, "test3_run0.json")
|
|
with open(testOutput, "r") as f:
|
|
json = json.load(f)
|
|
return json
|
|
|
|
def test_basic(self):
|
|
expect_lines = [
|
|
"BM_One",
|
|
"BM_Two",
|
|
"short", # These two are not sorted
|
|
"medium", # These two are not sorted
|
|
]
|
|
json = self.load_results()
|
|
output_lines = get_unique_benchmark_names(json)
|
|
print("\n")
|
|
print("\n".join(output_lines))
|
|
self.assertEqual(len(output_lines), len(expect_lines))
|
|
for i in range(0, len(output_lines)):
|
|
self.assertEqual(expect_lines[i], output_lines[i])
|
|
|
|
|
|
class TestReportDifference(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
def load_results():
|
|
import json
|
|
|
|
testInputs = os.path.join(
|
|
os.path.dirname(os.path.realpath(__file__)), "Inputs"
|
|
)
|
|
testOutput1 = os.path.join(testInputs, "test1_run1.json")
|
|
testOutput2 = os.path.join(testInputs, "test1_run2.json")
|
|
with open(testOutput1, "r") as f:
|
|
json1 = json.load(f)
|
|
with open(testOutput2, "r") as f:
|
|
json2 = json.load(f)
|
|
return json1, json2
|
|
|
|
json1, json2 = load_results()
|
|
cls.json_diff_report = get_difference_report(json1, json2)
|
|
|
|
def test_json_diff_report_pretty_printing(self):
|
|
expect_lines = [
|
|
["BM_SameTimes", "+0.0000", "+0.0000", "10", "10", "10", "10"],
|
|
["BM_2xFaster", "-0.5000", "-0.5000", "50", "25", "50", "25"],
|
|
["BM_2xSlower", "+1.0000", "+1.0000", "50", "100", "50", "100"],
|
|
[
|
|
"BM_1PercentFaster",
|
|
"-0.0100",
|
|
"-0.0100",
|
|
"100",
|
|
"99",
|
|
"100",
|
|
"99",
|
|
],
|
|
[
|
|
"BM_1PercentSlower",
|
|
"+0.0100",
|
|
"+0.0100",
|
|
"100",
|
|
"101",
|
|
"100",
|
|
"101",
|
|
],
|
|
[
|
|
"BM_10PercentFaster",
|
|
"-0.1000",
|
|
"-0.1000",
|
|
"100",
|
|
"90",
|
|
"100",
|
|
"90",
|
|
],
|
|
[
|
|
"BM_10PercentSlower",
|
|
"+0.1000",
|
|
"+0.1000",
|
|
"100",
|
|
"110",
|
|
"100",
|
|
"110",
|
|
],
|
|
[
|
|
"BM_100xSlower",
|
|
"+99.0000",
|
|
"+99.0000",
|
|
"100",
|
|
"10000",
|
|
"100",
|
|
"10000",
|
|
],
|
|
[
|
|
"BM_100xFaster",
|
|
"-0.9900",
|
|
"-0.9900",
|
|
"10000",
|
|
"100",
|
|
"10000",
|
|
"100",
|
|
],
|
|
[
|
|
"BM_10PercentCPUToTime",
|
|
"+0.1000",
|
|
"-0.1000",
|
|
"100",
|
|
"110",
|
|
"100",
|
|
"90",
|
|
],
|
|
["BM_ThirdFaster", "-0.3333", "-0.3334", "100", "67", "100", "67"],
|
|
["BM_NotBadTimeUnit", "-0.9000", "+0.2000", "0", "0", "0", "1"],
|
|
["BM_hasLabel", "+0.0000", "+0.0000", "1", "1", "1", "1"],
|
|
["OVERALL_GEOMEAN", "-0.8113", "-0.7779", "0", "0", "0", "0"],
|
|
]
|
|
output_lines_with_header = print_difference_report(
|
|
self.json_diff_report, use_color=False
|
|
)
|
|
output_lines = output_lines_with_header[2:]
|
|
print("\n")
|
|
print("\n".join(output_lines_with_header))
|
|
self.assertEqual(len(output_lines), len(expect_lines))
|
|
for i in range(0, len(output_lines)):
|
|
parts = [x for x in output_lines[i].split(" ") if x]
|
|
self.assertEqual(len(parts), 7)
|
|
self.assertEqual(expect_lines[i], parts)
|
|
|
|
def test_json_diff_report_output(self):
|
|
expected_output = [
|
|
{
|
|
"name": "BM_SameTimes",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"time": 0.0000,
|
|
"cpu": 0.0000,
|
|
"real_time": 10,
|
|
"real_time_other": 10,
|
|
"cpu_time": 10,
|
|
"cpu_time_other": 10,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "BM_2xFaster",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"time": -0.5000,
|
|
"cpu": -0.5000,
|
|
"real_time": 50,
|
|
"real_time_other": 25,
|
|
"cpu_time": 50,
|
|
"cpu_time_other": 25,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "BM_2xSlower",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"time": 1.0000,
|
|
"cpu": 1.0000,
|
|
"real_time": 50,
|
|
"real_time_other": 100,
|
|
"cpu_time": 50,
|
|
"cpu_time_other": 100,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "BM_1PercentFaster",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"time": -0.0100,
|
|
"cpu": -0.0100,
|
|
"real_time": 100,
|
|
"real_time_other": 98.9999999,
|
|
"cpu_time": 100,
|
|
"cpu_time_other": 98.9999999,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "BM_1PercentSlower",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"time": 0.0100,
|
|
"cpu": 0.0100,
|
|
"real_time": 100,
|
|
"real_time_other": 101,
|
|
"cpu_time": 100,
|
|
"cpu_time_other": 101,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "BM_10PercentFaster",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"time": -0.1000,
|
|
"cpu": -0.1000,
|
|
"real_time": 100,
|
|
"real_time_other": 90,
|
|
"cpu_time": 100,
|
|
"cpu_time_other": 90,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "BM_10PercentSlower",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"time": 0.1000,
|
|
"cpu": 0.1000,
|
|
"real_time": 100,
|
|
"real_time_other": 110,
|
|
"cpu_time": 100,
|
|
"cpu_time_other": 110,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "BM_100xSlower",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"time": 99.0000,
|
|
"cpu": 99.0000,
|
|
"real_time": 100,
|
|
"real_time_other": 10000,
|
|
"cpu_time": 100,
|
|
"cpu_time_other": 10000,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "BM_100xFaster",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"time": -0.9900,
|
|
"cpu": -0.9900,
|
|
"real_time": 10000,
|
|
"real_time_other": 100,
|
|
"cpu_time": 10000,
|
|
"cpu_time_other": 100,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "BM_10PercentCPUToTime",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"time": 0.1000,
|
|
"cpu": -0.1000,
|
|
"real_time": 100,
|
|
"real_time_other": 110,
|
|
"cpu_time": 100,
|
|
"cpu_time_other": 90,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "BM_ThirdFaster",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"time": -0.3333,
|
|
"cpu": -0.3334,
|
|
"real_time": 100,
|
|
"real_time_other": 67,
|
|
"cpu_time": 100,
|
|
"cpu_time_other": 67,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "BM_NotBadTimeUnit",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"time": -0.9000,
|
|
"cpu": 0.2000,
|
|
"real_time": 0.4,
|
|
"real_time_other": 0.04,
|
|
"cpu_time": 0.5,
|
|
"cpu_time_other": 0.6,
|
|
}
|
|
],
|
|
"time_unit": "s",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "BM_hasLabel",
|
|
"label": "a label",
|
|
"measurements": [
|
|
{
|
|
"time": 0.0000,
|
|
"cpu": 0.0000,
|
|
"real_time": 1,
|
|
"real_time_other": 1,
|
|
"cpu_time": 1,
|
|
"cpu_time_other": 1,
|
|
}
|
|
],
|
|
"time_unit": "s",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "OVERALL_GEOMEAN",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"real_time": 3.1622776601683826e-06,
|
|
"cpu_time": 3.2130844755623912e-06,
|
|
"real_time_other": 1.9768988699420897e-07,
|
|
"cpu_time_other": 2.397447755209533e-07,
|
|
"time": -0.8112976497120911,
|
|
"cpu": -0.7778551721181174,
|
|
}
|
|
],
|
|
"time_unit": "s",
|
|
"run_type": "aggregate",
|
|
"aggregate_name": "geomean",
|
|
"utest": {},
|
|
},
|
|
]
|
|
self.assertEqual(len(self.json_diff_report), len(expected_output))
|
|
for out, expected in zip(self.json_diff_report, expected_output):
|
|
self.assertEqual(out["name"], expected["name"])
|
|
self.assertEqual(out["label"], expected["label"])
|
|
self.assertEqual(out["time_unit"], expected["time_unit"])
|
|
assert_utest(self, out, expected)
|
|
assert_measurements(self, out, expected)
|
|
|
|
|
|
class TestReportDifferenceBetweenFamilies(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
def load_result():
|
|
import json
|
|
|
|
testInputs = os.path.join(
|
|
os.path.dirname(os.path.realpath(__file__)), "Inputs"
|
|
)
|
|
testOutput = os.path.join(testInputs, "test2_run.json")
|
|
with open(testOutput, "r") as f:
|
|
json = json.load(f)
|
|
return json
|
|
|
|
json = load_result()
|
|
json1 = filter_benchmark(json, "BM_Z.ro", ".")
|
|
json2 = filter_benchmark(json, "BM_O.e", ".")
|
|
cls.json_diff_report = get_difference_report(json1, json2)
|
|
|
|
def test_json_diff_report_pretty_printing(self):
|
|
expect_lines = [
|
|
[".", "-0.5000", "-0.5000", "10", "5", "10", "5"],
|
|
["./4", "-0.5000", "-0.5000", "40", "20", "40", "20"],
|
|
["Prefix/.", "-0.5000", "-0.5000", "20", "10", "20", "10"],
|
|
["Prefix/./3", "-0.5000", "-0.5000", "30", "15", "30", "15"],
|
|
["OVERALL_GEOMEAN", "-0.5000", "-0.5000", "0", "0", "0", "0"],
|
|
]
|
|
output_lines_with_header = print_difference_report(
|
|
self.json_diff_report, use_color=False
|
|
)
|
|
output_lines = output_lines_with_header[2:]
|
|
print("\n")
|
|
print("\n".join(output_lines_with_header))
|
|
self.assertEqual(len(output_lines), len(expect_lines))
|
|
for i in range(0, len(output_lines)):
|
|
parts = [x for x in output_lines[i].split(" ") if x]
|
|
self.assertEqual(len(parts), 7)
|
|
self.assertEqual(expect_lines[i], parts)
|
|
|
|
def test_json_diff_report(self):
|
|
expected_output = [
|
|
{
|
|
"name": ".",
|
|
"measurements": [
|
|
{
|
|
"time": -0.5,
|
|
"cpu": -0.5,
|
|
"real_time": 10,
|
|
"real_time_other": 5,
|
|
"cpu_time": 10,
|
|
"cpu_time_other": 5,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "./4",
|
|
"measurements": [
|
|
{
|
|
"time": -0.5,
|
|
"cpu": -0.5,
|
|
"real_time": 40,
|
|
"real_time_other": 20,
|
|
"cpu_time": 40,
|
|
"cpu_time_other": 20,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "Prefix/.",
|
|
"measurements": [
|
|
{
|
|
"time": -0.5,
|
|
"cpu": -0.5,
|
|
"real_time": 20,
|
|
"real_time_other": 10,
|
|
"cpu_time": 20,
|
|
"cpu_time_other": 10,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "Prefix/./3",
|
|
"measurements": [
|
|
{
|
|
"time": -0.5,
|
|
"cpu": -0.5,
|
|
"real_time": 30,
|
|
"real_time_other": 15,
|
|
"cpu_time": 30,
|
|
"cpu_time_other": 15,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "OVERALL_GEOMEAN",
|
|
"measurements": [
|
|
{
|
|
"real_time": 2.213363839400641e-08,
|
|
"cpu_time": 2.213363839400641e-08,
|
|
"real_time_other": 1.1066819197003185e-08,
|
|
"cpu_time_other": 1.1066819197003185e-08,
|
|
"time": -0.5000000000000009,
|
|
"cpu": -0.5000000000000009,
|
|
}
|
|
],
|
|
"time_unit": "s",
|
|
"run_type": "aggregate",
|
|
"aggregate_name": "geomean",
|
|
"utest": {},
|
|
},
|
|
]
|
|
self.assertEqual(len(self.json_diff_report), len(expected_output))
|
|
for out, expected in zip(self.json_diff_report, expected_output):
|
|
self.assertEqual(out["name"], expected["name"])
|
|
self.assertEqual(out["time_unit"], expected["time_unit"])
|
|
assert_utest(self, out, expected)
|
|
assert_measurements(self, out, expected)
|
|
|
|
|
|
class TestReportDifferenceWithUTest(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
def load_results():
|
|
import json
|
|
|
|
testInputs = os.path.join(
|
|
os.path.dirname(os.path.realpath(__file__)), "Inputs"
|
|
)
|
|
testOutput1 = os.path.join(testInputs, "test3_run0.json")
|
|
testOutput2 = os.path.join(testInputs, "test3_run1.json")
|
|
with open(testOutput1, "r") as f:
|
|
json1 = json.load(f)
|
|
with open(testOutput2, "r") as f:
|
|
json2 = json.load(f)
|
|
return json1, json2
|
|
|
|
json1, json2 = load_results()
|
|
cls.json_diff_report = get_difference_report(json1, json2, utest=True)
|
|
|
|
def test_json_diff_report_pretty_printing(self):
|
|
expect_lines = [
|
|
["BM_One", "-0.1000", "+0.1000", "10", "9", "100", "110"],
|
|
["BM_Two", "+0.1111", "-0.0111", "9", "10", "90", "89"],
|
|
["BM_Two", "-0.1250", "-0.1628", "8", "7", "86", "72"],
|
|
[
|
|
"BM_Two_pvalue",
|
|
"1.0000",
|
|
"0.6667",
|
|
"U",
|
|
"Test,",
|
|
"Repetitions:",
|
|
"2",
|
|
"vs",
|
|
"2.",
|
|
"WARNING:",
|
|
"Results",
|
|
"unreliable!",
|
|
"9+",
|
|
"repetitions",
|
|
"recommended.",
|
|
],
|
|
["short", "-0.1250", "-0.0625", "8", "7", "80", "75"],
|
|
["short", "-0.4325", "-0.1351", "8", "5", "77", "67"],
|
|
[
|
|
"short_pvalue",
|
|
"0.7671",
|
|
"0.2000",
|
|
"U",
|
|
"Test,",
|
|
"Repetitions:",
|
|
"2",
|
|
"vs",
|
|
"3.",
|
|
"WARNING:",
|
|
"Results",
|
|
"unreliable!",
|
|
"9+",
|
|
"repetitions",
|
|
"recommended.",
|
|
],
|
|
["medium", "-0.3750", "-0.3375", "8", "5", "80", "53"],
|
|
["OVERALL_GEOMEAN", "+1.6405", "-0.6985", "0", "0", "0", "0"],
|
|
]
|
|
output_lines_with_header = print_difference_report(
|
|
self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False
|
|
)
|
|
output_lines = output_lines_with_header[2:]
|
|
print("\n")
|
|
print("\n".join(output_lines_with_header))
|
|
self.assertEqual(len(output_lines), len(expect_lines))
|
|
for i in range(0, len(output_lines)):
|
|
parts = [x for x in output_lines[i].split(" ") if x]
|
|
self.assertEqual(expect_lines[i], parts)
|
|
|
|
def test_json_diff_report_pretty_printing_aggregates_only(self):
|
|
expect_lines = [
|
|
["BM_One", "-0.1000", "+0.1000", "10", "9", "100", "110"],
|
|
[
|
|
"BM_Two_pvalue",
|
|
"1.0000",
|
|
"0.6667",
|
|
"U",
|
|
"Test,",
|
|
"Repetitions:",
|
|
"2",
|
|
"vs",
|
|
"2.",
|
|
"WARNING:",
|
|
"Results",
|
|
"unreliable!",
|
|
"9+",
|
|
"repetitions",
|
|
"recommended.",
|
|
],
|
|
["short", "-0.1250", "-0.0625", "8", "7", "80", "75"],
|
|
["short", "-0.4325", "-0.1351", "8", "5", "77", "67"],
|
|
[
|
|
"short_pvalue",
|
|
"0.7671",
|
|
"0.2000",
|
|
"U",
|
|
"Test,",
|
|
"Repetitions:",
|
|
"2",
|
|
"vs",
|
|
"3.",
|
|
"WARNING:",
|
|
"Results",
|
|
"unreliable!",
|
|
"9+",
|
|
"repetitions",
|
|
"recommended.",
|
|
],
|
|
["OVERALL_GEOMEAN", "+1.6405", "-0.6985", "0", "0", "0", "0"],
|
|
]
|
|
output_lines_with_header = print_difference_report(
|
|
self.json_diff_report,
|
|
include_aggregates_only=True,
|
|
utest=True,
|
|
utest_alpha=0.05,
|
|
use_color=False,
|
|
)
|
|
output_lines = output_lines_with_header[2:]
|
|
print("\n")
|
|
print("\n".join(output_lines_with_header))
|
|
self.assertEqual(len(output_lines), len(expect_lines))
|
|
for i in range(0, len(output_lines)):
|
|
parts = [x for x in output_lines[i].split(" ") if x]
|
|
self.assertEqual(expect_lines[i], parts)
|
|
|
|
def test_json_diff_report(self):
|
|
expected_output = [
|
|
{
|
|
"name": "BM_One",
|
|
"measurements": [
|
|
{
|
|
"time": -0.1,
|
|
"cpu": 0.1,
|
|
"real_time": 10,
|
|
"real_time_other": 9,
|
|
"cpu_time": 100,
|
|
"cpu_time_other": 110,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "BM_Two",
|
|
"measurements": [
|
|
{
|
|
"time": 0.1111111111111111,
|
|
"cpu": -0.011111111111111112,
|
|
"real_time": 9,
|
|
"real_time_other": 10,
|
|
"cpu_time": 90,
|
|
"cpu_time_other": 89,
|
|
},
|
|
{
|
|
"time": -0.125,
|
|
"cpu": -0.16279069767441862,
|
|
"real_time": 8,
|
|
"real_time_other": 7,
|
|
"cpu_time": 86,
|
|
"cpu_time_other": 72,
|
|
},
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {
|
|
"have_optimal_repetitions": False,
|
|
"cpu_pvalue": 0.6666666666666666,
|
|
"time_pvalue": 1.0,
|
|
},
|
|
},
|
|
{
|
|
"name": "short",
|
|
"measurements": [
|
|
{
|
|
"time": -0.125,
|
|
"cpu": -0.0625,
|
|
"real_time": 8,
|
|
"real_time_other": 7,
|
|
"cpu_time": 80,
|
|
"cpu_time_other": 75,
|
|
},
|
|
{
|
|
"time": -0.4325,
|
|
"cpu": -0.13506493506493514,
|
|
"real_time": 8,
|
|
"real_time_other": 4.54,
|
|
"cpu_time": 77,
|
|
"cpu_time_other": 66.6,
|
|
},
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {
|
|
"have_optimal_repetitions": False,
|
|
"cpu_pvalue": 0.2,
|
|
"time_pvalue": 0.7670968684102772,
|
|
},
|
|
},
|
|
{
|
|
"name": "medium",
|
|
"measurements": [
|
|
{
|
|
"time": -0.375,
|
|
"cpu": -0.3375,
|
|
"real_time": 8,
|
|
"real_time_other": 5,
|
|
"cpu_time": 80,
|
|
"cpu_time_other": 53,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "OVERALL_GEOMEAN",
|
|
"measurements": [
|
|
{
|
|
"real_time": 8.48528137423858e-09,
|
|
"cpu_time": 8.441336246629233e-08,
|
|
"real_time_other": 2.2405267593145244e-08,
|
|
"cpu_time_other": 2.5453661413660466e-08,
|
|
"time": 1.6404861082353634,
|
|
"cpu": -0.6984640740519662,
|
|
}
|
|
],
|
|
"time_unit": "s",
|
|
"run_type": "aggregate",
|
|
"aggregate_name": "geomean",
|
|
"utest": {},
|
|
},
|
|
]
|
|
self.assertEqual(len(self.json_diff_report), len(expected_output))
|
|
for out, expected in zip(self.json_diff_report, expected_output):
|
|
self.assertEqual(out["name"], expected["name"])
|
|
self.assertEqual(out["time_unit"], expected["time_unit"])
|
|
assert_utest(self, out, expected)
|
|
assert_measurements(self, out, expected)
|
|
|
|
|
|
class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly(
|
|
unittest.TestCase
|
|
):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
def load_results():
|
|
import json
|
|
|
|
testInputs = os.path.join(
|
|
os.path.dirname(os.path.realpath(__file__)), "Inputs"
|
|
)
|
|
testOutput1 = os.path.join(testInputs, "test3_run0.json")
|
|
testOutput2 = os.path.join(testInputs, "test3_run1.json")
|
|
with open(testOutput1, "r") as f:
|
|
json1 = json.load(f)
|
|
with open(testOutput2, "r") as f:
|
|
json2 = json.load(f)
|
|
return json1, json2
|
|
|
|
json1, json2 = load_results()
|
|
cls.json_diff_report = get_difference_report(json1, json2, utest=True)
|
|
|
|
def test_json_diff_report_pretty_printing(self):
|
|
expect_lines = [
|
|
["BM_One", "-0.1000", "+0.1000", "10", "9", "100", "110"],
|
|
["BM_Two", "+0.1111", "-0.0111", "9", "10", "90", "89"],
|
|
["BM_Two", "-0.1250", "-0.1628", "8", "7", "86", "72"],
|
|
[
|
|
"BM_Two_pvalue",
|
|
"1.0000",
|
|
"0.6667",
|
|
"U",
|
|
"Test,",
|
|
"Repetitions:",
|
|
"2",
|
|
"vs",
|
|
"2.",
|
|
"WARNING:",
|
|
"Results",
|
|
"unreliable!",
|
|
"9+",
|
|
"repetitions",
|
|
"recommended.",
|
|
],
|
|
["short", "-0.1250", "-0.0625", "8", "7", "80", "75"],
|
|
["short", "-0.4325", "-0.1351", "8", "5", "77", "67"],
|
|
[
|
|
"short_pvalue",
|
|
"0.7671",
|
|
"0.2000",
|
|
"U",
|
|
"Test,",
|
|
"Repetitions:",
|
|
"2",
|
|
"vs",
|
|
"3.",
|
|
"WARNING:",
|
|
"Results",
|
|
"unreliable!",
|
|
"9+",
|
|
"repetitions",
|
|
"recommended.",
|
|
],
|
|
["medium", "-0.3750", "-0.3375", "8", "5", "80", "53"],
|
|
["OVERALL_GEOMEAN", "+1.6405", "-0.6985", "0", "0", "0", "0"],
|
|
]
|
|
output_lines_with_header = print_difference_report(
|
|
self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False
|
|
)
|
|
output_lines = output_lines_with_header[2:]
|
|
print("\n")
|
|
print("\n".join(output_lines_with_header))
|
|
self.assertEqual(len(output_lines), len(expect_lines))
|
|
for i in range(0, len(output_lines)):
|
|
parts = [x for x in output_lines[i].split(" ") if x]
|
|
self.assertEqual(expect_lines[i], parts)
|
|
|
|
def test_json_diff_report(self):
|
|
expected_output = [
|
|
{
|
|
"name": "BM_One",
|
|
"measurements": [
|
|
{
|
|
"time": -0.1,
|
|
"cpu": 0.1,
|
|
"real_time": 10,
|
|
"real_time_other": 9,
|
|
"cpu_time": 100,
|
|
"cpu_time_other": 110,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
},
|
|
{
|
|
"name": "BM_Two",
|
|
"measurements": [
|
|
{
|
|
"time": 0.1111111111111111,
|
|
"cpu": -0.011111111111111112,
|
|
"real_time": 9,
|
|
"real_time_other": 10,
|
|
"cpu_time": 90,
|
|
"cpu_time_other": 89,
|
|
},
|
|
{
|
|
"time": -0.125,
|
|
"cpu": -0.16279069767441862,
|
|
"real_time": 8,
|
|
"real_time_other": 7,
|
|
"cpu_time": 86,
|
|
"cpu_time_other": 72,
|
|
},
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {
|
|
"have_optimal_repetitions": False,
|
|
"cpu_pvalue": 0.6666666666666666,
|
|
"time_pvalue": 1.0,
|
|
},
|
|
},
|
|
{
|
|
"name": "short",
|
|
"measurements": [
|
|
{
|
|
"time": -0.125,
|
|
"cpu": -0.0625,
|
|
"real_time": 8,
|
|
"real_time_other": 7,
|
|
"cpu_time": 80,
|
|
"cpu_time_other": 75,
|
|
},
|
|
{
|
|
"time": -0.4325,
|
|
"cpu": -0.13506493506493514,
|
|
"real_time": 8,
|
|
"real_time_other": 4.54,
|
|
"cpu_time": 77,
|
|
"cpu_time_other": 66.6,
|
|
},
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {
|
|
"have_optimal_repetitions": False,
|
|
"cpu_pvalue": 0.2,
|
|
"time_pvalue": 0.7670968684102772,
|
|
},
|
|
},
|
|
{
|
|
"name": "medium",
|
|
"measurements": [
|
|
{
|
|
"real_time_other": 5,
|
|
"cpu_time": 80,
|
|
"time": -0.375,
|
|
"real_time": 8,
|
|
"cpu_time_other": 53,
|
|
"cpu": -0.3375,
|
|
}
|
|
],
|
|
"utest": {},
|
|
"time_unit": "ns",
|
|
"aggregate_name": "",
|
|
},
|
|
{
|
|
"name": "OVERALL_GEOMEAN",
|
|
"measurements": [
|
|
{
|
|
"real_time": 8.48528137423858e-09,
|
|
"cpu_time": 8.441336246629233e-08,
|
|
"real_time_other": 2.2405267593145244e-08,
|
|
"cpu_time_other": 2.5453661413660466e-08,
|
|
"time": 1.6404861082353634,
|
|
"cpu": -0.6984640740519662,
|
|
}
|
|
],
|
|
"time_unit": "s",
|
|
"run_type": "aggregate",
|
|
"aggregate_name": "geomean",
|
|
"utest": {},
|
|
},
|
|
]
|
|
self.assertEqual(len(self.json_diff_report), len(expected_output))
|
|
for out, expected in zip(self.json_diff_report, expected_output):
|
|
self.assertEqual(out["name"], expected["name"])
|
|
self.assertEqual(out["time_unit"], expected["time_unit"])
|
|
assert_utest(self, out, expected)
|
|
assert_measurements(self, out, expected)
|
|
|
|
|
|
class TestReportDifferenceForPercentageAggregates(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
def load_results():
|
|
import json
|
|
|
|
testInputs = os.path.join(
|
|
os.path.dirname(os.path.realpath(__file__)), "Inputs"
|
|
)
|
|
testOutput1 = os.path.join(testInputs, "test4_run0.json")
|
|
testOutput2 = os.path.join(testInputs, "test4_run1.json")
|
|
with open(testOutput1, "r") as f:
|
|
json1 = json.load(f)
|
|
with open(testOutput2, "r") as f:
|
|
json2 = json.load(f)
|
|
return json1, json2
|
|
|
|
json1, json2 = load_results()
|
|
cls.json_diff_report = get_difference_report(json1, json2, utest=True)
|
|
|
|
def test_json_diff_report_pretty_printing(self):
|
|
expect_lines = [["whocares", "-0.5000", "+0.5000", "0", "0", "0", "0"]]
|
|
output_lines_with_header = print_difference_report(
|
|
self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False
|
|
)
|
|
output_lines = output_lines_with_header[2:]
|
|
print("\n")
|
|
print("\n".join(output_lines_with_header))
|
|
self.assertEqual(len(output_lines), len(expect_lines))
|
|
for i in range(0, len(output_lines)):
|
|
parts = [x for x in output_lines[i].split(" ") if x]
|
|
self.assertEqual(expect_lines[i], parts)
|
|
|
|
def test_json_diff_report(self):
|
|
expected_output = [
|
|
{
|
|
"name": "whocares",
|
|
"measurements": [
|
|
{
|
|
"time": -0.5,
|
|
"cpu": 0.5,
|
|
"real_time": 0.01,
|
|
"real_time_other": 0.005,
|
|
"cpu_time": 0.10,
|
|
"cpu_time_other": 0.15,
|
|
}
|
|
],
|
|
"time_unit": "ns",
|
|
"utest": {},
|
|
}
|
|
]
|
|
self.assertEqual(len(self.json_diff_report), len(expected_output))
|
|
for out, expected in zip(self.json_diff_report, expected_output):
|
|
self.assertEqual(out["name"], expected["name"])
|
|
self.assertEqual(out["time_unit"], expected["time_unit"])
|
|
assert_utest(self, out, expected)
|
|
assert_measurements(self, out, expected)
|
|
|
|
|
|
class TestReportSorting(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
def load_result():
|
|
import json
|
|
|
|
testInputs = os.path.join(
|
|
os.path.dirname(os.path.realpath(__file__)), "Inputs"
|
|
)
|
|
testOutput = os.path.join(testInputs, "test4_run.json")
|
|
with open(testOutput, "r") as f:
|
|
json = json.load(f)
|
|
return json
|
|
|
|
cls.json = load_result()
|
|
|
|
def test_json_diff_report_pretty_printing(self):
|
|
import util
|
|
|
|
expected_names = [
|
|
"99 family 0 instance 0 repetition 0",
|
|
"98 family 0 instance 0 repetition 1",
|
|
"97 family 0 instance 0 aggregate",
|
|
"96 family 0 instance 1 repetition 0",
|
|
"95 family 0 instance 1 repetition 1",
|
|
"94 family 0 instance 1 aggregate",
|
|
"93 family 1 instance 0 repetition 0",
|
|
"92 family 1 instance 0 repetition 1",
|
|
"91 family 1 instance 0 aggregate",
|
|
"90 family 1 instance 1 repetition 0",
|
|
"89 family 1 instance 1 repetition 1",
|
|
"88 family 1 instance 1 aggregate",
|
|
]
|
|
|
|
for n in range(len(self.json["benchmarks"]) ** 2):
|
|
random.shuffle(self.json["benchmarks"])
|
|
sorted_benchmarks = util.sort_benchmark_results(self.json)[
|
|
"benchmarks"
|
|
]
|
|
self.assertEqual(len(expected_names), len(sorted_benchmarks))
|
|
for out, expected in zip(sorted_benchmarks, expected_names):
|
|
self.assertEqual(out["name"], expected)
|
|
|
|
|
|
class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly2(
|
|
unittest.TestCase
|
|
):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
def load_results():
|
|
import json
|
|
|
|
testInputs = os.path.join(
|
|
os.path.dirname(os.path.realpath(__file__)), "Inputs"
|
|
)
|
|
testOutput1 = os.path.join(testInputs, "test5_run0.json")
|
|
testOutput2 = os.path.join(testInputs, "test5_run1.json")
|
|
with open(testOutput1, "r") as f:
|
|
json1 = json.load(f)
|
|
json1["benchmarks"] = [
|
|
json1["benchmarks"][0] for i in range(1000)
|
|
]
|
|
with open(testOutput2, "r") as f:
|
|
json2 = json.load(f)
|
|
json2["benchmarks"] = [
|
|
json2["benchmarks"][0] for i in range(1000)
|
|
]
|
|
return json1, json2
|
|
|
|
json1, json2 = load_results()
|
|
cls.json_diff_report = get_difference_report(json1, json2, utest=True)
|
|
|
|
def test_json_diff_report_pretty_printing(self):
|
|
expect_line = [
|
|
"BM_ManyRepetitions_pvalue",
|
|
"0.0000",
|
|
"0.0000",
|
|
"U",
|
|
"Test,",
|
|
"Repetitions:",
|
|
"1000",
|
|
"vs",
|
|
"1000",
|
|
]
|
|
output_lines_with_header = print_difference_report(
|
|
self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False
|
|
)
|
|
output_lines = output_lines_with_header[2:]
|
|
found = False
|
|
for i in range(0, len(output_lines)):
|
|
parts = [x for x in output_lines[i].split(" ") if x]
|
|
found = expect_line == parts
|
|
if found:
|
|
break
|
|
self.assertTrue(found)
|
|
|
|
def test_json_diff_report(self):
|
|
expected_output = [
|
|
{
|
|
"name": "BM_ManyRepetitions",
|
|
"label": "",
|
|
"time_unit": "s",
|
|
"run_type": "",
|
|
"aggregate_name": "",
|
|
"utest": {
|
|
"have_optimal_repetitions": True,
|
|
"cpu_pvalue": 0.0,
|
|
"time_pvalue": 0.0,
|
|
"nr_of_repetitions": 1000,
|
|
"nr_of_repetitions_other": 1000,
|
|
},
|
|
},
|
|
{
|
|
"name": "OVERALL_GEOMEAN",
|
|
"label": "",
|
|
"measurements": [
|
|
{
|
|
"real_time": 1.0,
|
|
"cpu_time": 1000.000000000069,
|
|
"real_time_other": 1000.000000000069,
|
|
"cpu_time_other": 1.0,
|
|
"time": 999.000000000069,
|
|
"cpu": -0.9990000000000001,
|
|
}
|
|
],
|
|
"time_unit": "s",
|
|
"run_type": "aggregate",
|
|
"aggregate_name": "geomean",
|
|
"utest": {},
|
|
},
|
|
]
|
|
self.assertEqual(len(self.json_diff_report), len(expected_output))
|
|
for out, expected in zip(self.json_diff_report, expected_output):
|
|
self.assertEqual(out["name"], expected["name"])
|
|
self.assertEqual(out["time_unit"], expected["time_unit"])
|
|
assert_utest(self, out, expected)
|
|
|
|
|
|
def assert_utest(unittest_instance, lhs, rhs):
|
|
if lhs["utest"]:
|
|
unittest_instance.assertAlmostEqual(
|
|
lhs["utest"]["cpu_pvalue"], rhs["utest"]["cpu_pvalue"]
|
|
)
|
|
unittest_instance.assertAlmostEqual(
|
|
lhs["utest"]["time_pvalue"], rhs["utest"]["time_pvalue"]
|
|
)
|
|
unittest_instance.assertEqual(
|
|
lhs["utest"]["have_optimal_repetitions"],
|
|
rhs["utest"]["have_optimal_repetitions"],
|
|
)
|
|
else:
|
|
# lhs is empty. assert if rhs is not.
|
|
unittest_instance.assertEqual(lhs["utest"], rhs["utest"])
|
|
|
|
|
|
def assert_measurements(unittest_instance, lhs, rhs):
|
|
for m1, m2 in zip(lhs["measurements"], rhs["measurements"]):
|
|
unittest_instance.assertEqual(m1["real_time"], m2["real_time"])
|
|
unittest_instance.assertEqual(m1["cpu_time"], m2["cpu_time"])
|
|
# m1['time'] and m1['cpu'] hold values which are being calculated,
|
|
# and therefore we must use almost-equal pattern.
|
|
unittest_instance.assertAlmostEqual(m1["time"], m2["time"], places=4)
|
|
unittest_instance.assertAlmostEqual(m1["cpu"], m2["cpu"], places=4)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|
|
|
|
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
|
|
# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;
|
|
# kate: indent-mode python; remove-trailing-spaces modified;
|