504 lines
16 KiB
Python
504 lines
16 KiB
Python
import matplotlib
|
|
import matplotlib.pyplot as plt
|
|
import matplotlib.patches as mpatches
|
|
import matplotlib.ticker as mticker
|
|
import csv
|
|
import json
|
|
import math
|
|
import random
|
|
import bisect
|
|
import os
|
|
import argparse
|
|
import fnmatch
|
|
import sys
|
|
|
|
|
|
def len_sorter(x):
|
|
return len(x)
|
|
|
|
|
|
def entry_name_sorter(b):
|
|
return b["name"]
|
|
|
|
|
|
def is_noop_category(n):
|
|
noop_names = ["noop", "no-op", "no op"]
|
|
s = n.casefold()
|
|
return s in noop_names
|
|
|
|
|
|
def aggregate_categories(all_benchmarks, data_point_names):
|
|
benchmarks = {}
|
|
# sort by mean
|
|
# so the actual ordering of graphs
|
|
# is lowest to highest
|
|
lower_is_better = data_point_names[0][1]
|
|
|
|
def mean_sorter(b):
|
|
if (b.get("error") != None):
|
|
return sys.float_info.max
|
|
mean_group = b["statistics"]["mean"]
|
|
data_point_name = data_point_names[0][0]
|
|
return mean_group[data_point_name]
|
|
|
|
# find no-op category and use it in all benchmarks
|
|
noop_benches = None
|
|
for b in all_benchmarks:
|
|
category = b["category"]
|
|
if is_noop_category(category):
|
|
noop_benches = b
|
|
break
|
|
|
|
for b in all_benchmarks:
|
|
category = b["category"]
|
|
if is_noop_category(category):
|
|
continue
|
|
|
|
if category not in benchmarks:
|
|
benchmarks[category] = {
|
|
"benchmarks": [],
|
|
"heuristics": {
|
|
"min": sys.float_info.max,
|
|
"max": sys.float_info.min
|
|
}
|
|
}
|
|
if (noop_benches):
|
|
benchmarks[category]["benchmarks"].append(noop_benches)
|
|
|
|
target_category = benchmarks[category]
|
|
target_entries = target_category["benchmarks"]
|
|
target_heuristics = target_category["heuristics"]
|
|
|
|
target_entries.append(b)
|
|
target_heuristics["min"] = min(b["heuristics"]["min"],
|
|
target_heuristics["min"])
|
|
target_heuristics["max"] = max(b["heuristics"]["max"],
|
|
target_heuristics["max"])
|
|
|
|
for category_name in benchmarks:
|
|
category_benchmarks = benchmarks[category_name]
|
|
# first, sort by name so we can assign colors to each
|
|
# benchmark appropriately (and make those
|
|
# color assignments stable)
|
|
entries = category_benchmarks["benchmarks"]
|
|
entries.sort(key=entry_name_sorter)
|
|
for bi, entry in enumerate(entries):
|
|
entry["name_index"] = bi
|
|
ci = entry["color_index"]
|
|
if (len(data_point_names) < 2):
|
|
dp = data_point_names[0]
|
|
ci[dp[0]] = bi
|
|
else:
|
|
for dpi, dp in enumerate(data_point_names):
|
|
ci[dp[0]] = dpi
|
|
|
|
# then, sort by mean
|
|
entries.sort(key=mean_sorter, reverse=lower_is_better)
|
|
|
|
return benchmarks
|
|
|
|
|
|
def parse_csv(c, data_point_names, name_removals, categories, scale,
|
|
scale_categories, time_scales):
|
|
all_benchmarks = []
|
|
|
|
return aggregate_categories(all_benchmarks, data_point_names)
|
|
|
|
|
|
def parse_json(j, data_point_names, name_removals, categories, scale,
|
|
scale_categories, time_scales):
|
|
timescale_units = [x[0] for x in time_scales]
|
|
|
|
all_benchmarks = []
|
|
|
|
j_benchmarks_array = j["benchmarks"]
|
|
for j_benchmark in j_benchmarks_array:
|
|
name = j_benchmark['name']
|
|
run_name = j_benchmark['run_name']
|
|
benchmark = None
|
|
potential_targets = [
|
|
b for b in all_benchmarks if b['run_name'] == run_name
|
|
]
|
|
potential_categories = None if categories == None else [
|
|
c for c in categories if c in run_name
|
|
]
|
|
|
|
category = ""
|
|
benchmark_name = run_name
|
|
point_scalar = 1
|
|
if (len(potential_categories) > 1):
|
|
potential_categories.sort(key=len_sorter, reverse=True)
|
|
if len(potential_categories) > 0:
|
|
category = potential_categories[0]
|
|
if category in scale_categories:
|
|
point_scalar = 1 / scale
|
|
|
|
if (len(potential_targets) < 1):
|
|
benchmark_name = run_name.replace(category, "").strip("_")
|
|
for chunk in name_removals:
|
|
benchmark_name = benchmark_name.replace(chunk, "")
|
|
all_benchmarks.append({
|
|
"category": category,
|
|
"name": benchmark_name,
|
|
"run_name": run_name,
|
|
"data": {},
|
|
"statistics": {},
|
|
"heuristics": {
|
|
"max": sys.float_info.min,
|
|
"min": sys.float_info.max,
|
|
},
|
|
"name_index": {},
|
|
"color_index": {},
|
|
"error": None
|
|
})
|
|
benchmark = all_benchmarks[-1]
|
|
else:
|
|
benchmark = potential_targets[-1]
|
|
data = benchmark["data"]
|
|
statistics = benchmark["statistics"]
|
|
heuristics = benchmark["heuristics"]
|
|
# check for errors
|
|
benchmark_error = j_benchmark.get('error_occurred')
|
|
if benchmark_error != None and benchmark_error:
|
|
benchmark["error"] = j_benchmark['error_message']
|
|
continue
|
|
# populate data
|
|
for point_name_lower in data_point_names:
|
|
point_name = point_name_lower[0]
|
|
if point_name not in data:
|
|
data[point_name] = []
|
|
time_unit = j_benchmark['time_unit']
|
|
unit_index = timescale_units.index(time_unit)
|
|
time_scale = time_scales[unit_index]
|
|
to_seconds_multiplier = time_scale[2]
|
|
if name == run_name:
|
|
# is a data point
|
|
for point_name_lower in data_point_names:
|
|
point_name = point_name_lower[0]
|
|
point_list = data[point_name]
|
|
point = j_benchmark[point_name]
|
|
point_adjusted = point * to_seconds_multiplier * point_scalar
|
|
point_list.append(point_adjusted)
|
|
heuristics["min"] = min(heuristics["min"], point_adjusted)
|
|
heuristics["max"] = max(heuristics["max"], point_adjusted)
|
|
else:
|
|
# is a statistic
|
|
statistic_name = name.replace(run_name, "").strip("_")
|
|
if statistic_name not in statistics:
|
|
statistics[statistic_name] = {}
|
|
statistic = statistics[statistic_name]
|
|
for point_name_lower in data_point_names:
|
|
point_name = point_name_lower[0]
|
|
point = j_benchmark[point_name]
|
|
point_adjusted = point * to_seconds_multiplier * point_scalar
|
|
statistic[point_name] = point_adjusted
|
|
|
|
return aggregate_categories(all_benchmarks, data_point_names)
|
|
|
|
|
|
def draw_graph(name, category, benchmarks_heuristics, data_point_names,
|
|
time_scales):
|
|
# initialize figures
|
|
figures, axes = plt.subplots()
|
|
|
|
# set name we're going to use
|
|
figure_name = name if name != None and len(
|
|
name) > 0 else category.replace("_", "")
|
|
|
|
# get the values of the time scale to perform bisecting
|
|
time_scale_values_from_seconds = [x[2] for x in time_scales]
|
|
benchmarks = benchmarks_heuristics["benchmarks"]
|
|
heuristics = benchmarks_heuristics["heuristics"]
|
|
benchmarks_max = heuristics["max"]
|
|
benchmarks_min = heuristics["min"]
|
|
absolute_range = benchmarks_max - benchmarks_min
|
|
|
|
# some pattern constants, to help us be pretty
|
|
# some color constants, to help us be pretty!
|
|
# and differentiate graphs
|
|
# yapf: disable
|
|
data_point_aesthetics = [
|
|
('#a6cee3', '/'),
|
|
('#f255bb', 'O'),
|
|
('#00c9ab', '\\'),
|
|
('#b15928', 'o'),
|
|
('#33a02c', '.'),
|
|
('#fb9a99', '*'),
|
|
('#e31a1c', '+'),
|
|
('#fdbf6f', 'x'),
|
|
('#ff7f00', '|'),
|
|
('#cab2d6', None),
|
|
('#6a3d9a', '-'),
|
|
('#ffff99', 'xx'),
|
|
('#f5f5f5', '..'),
|
|
('#1f78b4', '||'),
|
|
('#b2df8a', '**'),
|
|
('#cc33cc', '--')
|
|
]
|
|
#yapf: enable
|
|
|
|
# transpose data into forms we need
|
|
benchmark_names = [b["name"] for b in benchmarks]
|
|
bars = []
|
|
scatters = []
|
|
num_data_points = len(data_point_names)
|
|
bar_padding = 0.15
|
|
bar_height = 0.35
|
|
bar_all_sizes = bar_height * num_data_points + bar_padding
|
|
quarter_bar_height = bar_height * 0.25
|
|
bar_y_positions = []
|
|
|
|
# draw mean-based bars with error indicators
|
|
# and draw scatter-plot points
|
|
for bi, benchmark in enumerate(benchmarks):
|
|
statistics = benchmark["statistics"]
|
|
for di, data_point_name_lower in enumerate(data_point_names):
|
|
data_point_name = data_point_name_lower[0]
|
|
bar_y = (bi * bar_all_sizes) + (di * bar_height) + (
|
|
bar_padding * 0.5)
|
|
bar_y_positions.append(bar_y)
|
|
err = benchmark.get('error')
|
|
|
|
color_index = benchmark["color_index"][data_point_name]
|
|
aesthetics = data_point_aesthetics[color_index]
|
|
color = aesthetics[0]
|
|
colorhsv = matplotlib.colors.rgb_to_hsv(
|
|
matplotlib.colors.hex2color(color))
|
|
colorhsv[2] *= 0.6
|
|
edgecolor = matplotlib.colors.hsv_to_rgb(colorhsv)
|
|
|
|
if err != None:
|
|
bars.append(
|
|
axes.text(
|
|
absolute_range * 0.02,
|
|
bar_y + (quarter_bar_height * 2),
|
|
err,
|
|
color=color,
|
|
style='italic',
|
|
horizontalalignment='left',
|
|
verticalalignment='center',
|
|
fontsize='small'))
|
|
continue
|
|
|
|
mean = statistics["mean"][data_point_name]
|
|
stddev = statistics["stddev"][data_point_name]
|
|
hatch = aesthetics[1]
|
|
bar = axes.barh(
|
|
bar_y,
|
|
mean,
|
|
height=bar_height,
|
|
xerr=stddev,
|
|
linewidth=0.2,
|
|
edgecolor=edgecolor,
|
|
color=color,
|
|
hatch=hatch,
|
|
align='edge',
|
|
error_kw={
|
|
"capsize": 5.0,
|
|
"mew": 1.2,
|
|
"ecolor": 'black',
|
|
},
|
|
alpha=0.82)
|
|
bars.append(bar)
|
|
# the scatter plot should be semi-transparent in color...
|
|
xscatter = benchmark["data"][data_point_name]
|
|
xscatter_len = len(xscatter)
|
|
yscatter = [
|
|
bar_y + random.uniform(quarter_bar_height,
|
|
bar_height - quarter_bar_height)
|
|
for _ in xscatter
|
|
]
|
|
scatter_alpha = 0.20 if xscatter_len < 11 else 0.10 if xscatter_len < 101 else 0.05 if xscatter_len < 1001 else 0.002
|
|
scatter = axes.scatter(
|
|
xscatter,
|
|
yscatter,
|
|
color=color,
|
|
edgecolor='#000000',
|
|
linewidth=0.5,
|
|
alpha=scatter_alpha)
|
|
scatters.append(scatter)
|
|
|
|
xscaleindex = bisect.bisect_left(time_scale_values_from_seconds,
|
|
benchmarks_max)
|
|
xscale = time_scales[xscaleindex - 1]
|
|
|
|
def time_axis_formatting(value, pos):
|
|
if value == 0:
|
|
return '0'
|
|
if value.is_integer():
|
|
return '{0:.0f}'.format(value * xscale[3])
|
|
return '{0:.1f}'.format(value * xscale[3])
|
|
|
|
axes.set_xlim([0, benchmarks_max + (absolute_range * 0.25)])
|
|
axes.xaxis.set_major_formatter(
|
|
mticker.FuncFormatter(time_axis_formatting))
|
|
|
|
# have ticks drawn from base of bar graph
|
|
# to text labels
|
|
y_ticks = [((y + 0.5) * bar_all_sizes)
|
|
for y in range(0, int(len(bar_y_positions) / num_data_points))]
|
|
y_limits = [
|
|
bar_y_positions[0] - bar_padding,
|
|
bar_y_positions[-1] + bar_height + bar_padding
|
|
]
|
|
|
|
# set the tick spacing
|
|
axes.set_yticks(y_ticks)
|
|
# label each group (each cluster along the y axes)
|
|
# with the names of the benchmarks we ran
|
|
axes.set_yticklabels(benchmark_names)
|
|
# set the visual limits so we have good spacing
|
|
axes.set_ylim(y_limits)
|
|
|
|
# if we have 2 or more data points,
|
|
# a legend will help us label it all
|
|
if (num_data_points > 1):
|
|
# a proper legend for each name in data_point_names
|
|
legend_texts = [
|
|
(data_point_name[0] +
|
|
('- lower=good' if data_point_name[1] else 'higher=good')
|
|
for data_point_name in data_point_names)
|
|
]
|
|
# retrieve the color/shape of the bar as a reference so we can construct
|
|
bar_style_references = [bar[0] for bar in bars]
|
|
# make legend
|
|
axes.legend(bar_style_references, legend_texts)
|
|
axes.set_xlabel('measured in ' + xscale[1])
|
|
else:
|
|
# no need to put a legend, it's basically fine as-is
|
|
data_point_name = data_point_names[0]
|
|
legend_text = (data_point_name[0], 'lower is better'
|
|
if data_point_name[1] else 'higher is better')
|
|
axes.set_xlabel(legend_text[0] + ' measured in ' + xscale[1] +
|
|
' - ' + legend_text[1])
|
|
|
|
# set the benchmark name, typically derived from the file name
|
|
axes.set_title(figure_name)
|
|
# get a nice, clean layout
|
|
figures.tight_layout()
|
|
|
|
# make sure to adjust top and bottoms
|
|
figures.subplots_adjust(bottom=0.2)
|
|
|
|
return figures, axes
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description=
|
|
'Generate graphs from a Google-Benchmark compatible json/csv listing of data'
|
|
)
|
|
parser.add_argument(
|
|
'-i',
|
|
'--input',
|
|
nargs='?',
|
|
default='out_ptr_benchmarks.json',
|
|
type=argparse.FileType('r'))
|
|
parser.add_argument('-f', '--input_format', nargs='?')
|
|
parser.add_argument('-o', '--output', nargs='?')
|
|
parser.add_argument('-d', '--output_dir', nargs='?')
|
|
parser.add_argument(
|
|
'-p', '--data_point_names', nargs='+', default=['real_time'])
|
|
parser.add_argument('-l', '--lower', nargs='+', default=['real_time'])
|
|
parser.add_argument('-c', '--categories', nargs='+', default=[])
|
|
parser.add_argument('-s', '--scale', nargs='?', type=int, default=1)
|
|
parser.add_argument('-t', '--scale_categories', nargs='+', default=[])
|
|
parser.add_argument('-r', '--remove_from_names', nargs='+', default=[''])
|
|
parser.add_argument('-z', '--time_format', nargs='?', default='clock')
|
|
|
|
args = parser.parse_args()
|
|
|
|
args.input_format = args.input_format or ("csv" if fnmatch.fnmatch(
|
|
args.input.name, "*.csv") else "json")
|
|
|
|
if not args.output:
|
|
directoryname, filename = os.path.split(args.input.name)
|
|
file = os.path.splitext(filename)[0]
|
|
args.output = os.path.join(directoryname, file + ".png")
|
|
|
|
if args.categories and len(args.categories) > 0 and not args.output_dir:
|
|
directoryname, filename = os.path.split(args.output)
|
|
args.output_dir = directoryname
|
|
|
|
if len(args.data_point_names) < 1:
|
|
print(
|
|
"You must specify 1 or more valid data point names",
|
|
file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
data_point_names = [(dpn, dpn in args.lower)
|
|
for dpn in args.data_point_names]
|
|
name_removals = [n for n in args.remove_from_names]
|
|
|
|
random.seed(1782905257495843795)
|
|
|
|
name = os.path.split(args.input.name)[1]
|
|
name = os.path.splitext(name)[0]
|
|
|
|
clock_time_scales = [
|
|
("fs", "femtoseconds", 1e-15, 1e+15),
|
|
("ps", "picoseconds", 1e-12, 1e+12),
|
|
("ns", "nanoseconds", 1e-9, 1e+9),
|
|
("µs", "microseconds", .00001, 1000000),
|
|
("us", "microseconds", .00001, 1000000),
|
|
("ms", "milliseconds", .001, 1000),
|
|
("s", "seconds", 1, 1),
|
|
("m", "minutes", 60, 1 / 60),
|
|
("h", "hours", 60 * 60, (1 / 60) / 60),
|
|
]
|
|
|
|
clock_time_scales = [
|
|
("fs", "femtoseconds", 1e-15, 1e+15),
|
|
("ps", "picoseconds", 1e-12, 1e+12),
|
|
("ns", "nanoseconds", 1e-9, 1e+9),
|
|
("µs", "microseconds", .00001, 1000000),
|
|
("us", "microseconds", .00001, 1000000),
|
|
("ms", "milliseconds", .001, 1000),
|
|
("s", "seconds", 1, 1),
|
|
("m", "minutes", 60, 1 / 60),
|
|
("h", "hours", 60 * 60, (1 / 60) / 60),
|
|
]
|
|
|
|
is_csv = args.input_format == "csv"
|
|
is_json = args.input_format == "json"
|
|
if (not is_csv and not is_json):
|
|
print(
|
|
"You must specify either 'json' or 'csv' as the format.",
|
|
file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
benchmarks = None
|
|
if is_csv:
|
|
c = csv.reader(args.input)
|
|
benchmarks = parse_csv(c, data_point_names, name_removals,
|
|
args.categories, args.scale,
|
|
args.scale_categories, clock_time_scales)
|
|
elif is_json:
|
|
j = json.load(args.input)
|
|
benchmarks = parse_json(j, data_point_names, name_removals,
|
|
args.categories, args.scale,
|
|
args.scale_categories, clock_time_scales)
|
|
else:
|
|
return
|
|
|
|
# we are okay to draw
|
|
# draw for each category
|
|
for benchmarks_key in benchmarks:
|
|
b = benchmarks[benchmarks_key]
|
|
category = benchmarks_key
|
|
if category == None or len(category) < 1:
|
|
category = name
|
|
benchmark_name = category.replace("_measure",
|
|
"").replace("_", " ").strip()
|
|
figures, axes = draw_graph(benchmark_name, category, b,
|
|
data_point_names, clock_time_scales)
|
|
# save drawn figures
|
|
save_name = benchmark_name
|
|
savetarget = os.path.join(args.output_dir, save_name + '.png')
|
|
print("Saving graph: {} (to '{}')".format(save_name, savetarget))
|
|
plt.savefig(savetarget, format='png')
|
|
plt.close(figures)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |