diff --git a/benchmark/parse_results.py b/benchmark/parse_results.py index 68ba0a1..c9e756b 100755 --- a/benchmark/parse_results.py +++ b/benchmark/parse_results.py @@ -207,8 +207,6 @@ class LatestResultVisitor(ResultVisitor): data.append(None) continue scene_stats = stats[scene_name] - # data.append({"x": scene_name, - # "y": scene_stats['pipeline_render_time']}) data.append(scene_stats['pipeline_render_time']) dataset = { "label": device_name, @@ -228,6 +226,93 @@ class LatestResultVisitor(ResultVisitor): f.write(code) +class HistoryResultVisitor(ResultVisitor): + def __init__(self): + ResultVisitor.__init__(self) + self.devices_ = {} + + def copyUsableStats(self, device, results): + stats = results['stats'] + blender_version = results['blender_version'] + timestamp = util.blenderCommitUnixTimestamp( + blender_version['build_commit_date'], + blender_version['build_commit_time']) + for scene_name in stats: + stat = stats[scene_name] + # Ignore benchmark results which crashed or aborted. + if "result" not in stat or stat['result'] != 'OK': + continue + device_scene_stat = dict(stat) + device_scene_stat.pop("result") + device_scene_stat['timestamp'] = timestamp + if scene_name not in device: + device[scene_name] = [] + device[scene_name].append(device_scene_stat) + + def handleResults(self, results): + if 'device_info' not in results or \ + 'stats' not in results: + return + device_info = results['device_info'] + device_name = util.deviceInfoAsString(device_info) + # If there were no stats for the device, + if device_name not in self.devices_: + self.devices_[device_name] = {} + self.copyUsableStats(self.devices_[device_name], results) + + def removeDuplicated(self, stats_history): + new_stats_history = [] + prev_timestamp = None + for stats in stats_history: + if stats['timestamp'] == prev_timestamp: + # TODO(sergey): Average somehow? + continue + new_stats_history.append(stats) + prev_timestamp = stats['timestamp'] + return new_stats_history + + def storeResults(self, dataset_dir): + # Firts of all, gather all possible scenes. + all_scenes = set() + for device_name, stats in self.devices_.items(): + for scene_name in stats.keys(): + all_scenes.add(scene_name) + all_scenes = sorted(list(all_scenes)) + # Gather datasets in format of lines. + datasets = [] + device_index = 0 + for device_name in sorted(self.devices_.keys()): + stats_history = self.devices_[device_name] + for scene_name in all_scenes: + if scene_name not in stats_history: + continue + scene_stats_history = stats_history[scene_name] + data = [] + sorted_scene_stats_history = sorted(scene_stats_history, + key=lambda k: k['timestamp']) + uniq_scene_stats_history = self.removeDuplicated( + sorted_scene_stats_history) + for scene_stats in uniq_scene_stats_history: + timestamp = scene_stats['timestamp'] + data.append({"x": timestamp.strftime("%d/%m/%y %H:%M"), + "y": scene_stats['pipeline_render_time']}) + dataset = { + "device_name": device_name, + "scene_name": scene_name, + "data": data, + } + datasets.append(dataset) + device_index += 1 + data = { + "scenes": all_scenes, + "datasets": datasets, + } + code = "var data = " + json.dumps(data, sort_keys=True, indent=2) + ";" + # Save dataset to disk. + filename = os.path.join(dataset_dir, "history.js") + with open(filename, "w") as f: + f.write(code) + def main(): parser = configureArgumentParser() args = parser.parse_args() @@ -239,9 +324,11 @@ def main(): return False # Construct list of all visitors whic hwe want. latest_results_visitor = LatestResultVisitor() + history_results_visitor = HistoryResultVisitor() # Do actual parse. logger.INFO("Iterating over all benchmark results...") - visitors = [latest_results_visitor] + visitors = (latest_results_visitor, + history_results_visitor,) iterateBenchmarksRoot(config['parser']['benchmark_dir'], visitors) # Store results. logger.INFO("Storing results...") diff --git a/benchmark/website/history.html b/benchmark/website/history.html new file mode 100644 index 0000000..16fa4ba --- /dev/null +++ b/benchmark/website/history.html @@ -0,0 +1,32 @@ + + + + Benchmark Results + + + + + + + + + + + +
+
+

+
+ + + diff --git a/benchmark/website/source/utils.js b/benchmark/website/source/utils.js index 0a47894..7483407 100644 --- a/benchmark/website/source/utils.js +++ b/benchmark/website/source/utils.js @@ -43,21 +43,42 @@ function secondsToHumanReadable(seconds) { return result; } -function buildChart(ctx, bare_data) { - var data = bare_data; - var max_value = 0; +function setDatasetColor(data, is_bar) { var index = 0; for (dataset of data.datasets) { var color = Chart.helpers.color(generateBarColor(index)); dataset.backgroundColor = color.alpha(0.5).rgbString(); dataset.borderColor = color.alpha(1.0).rgbString(); - dataset.borderWidth = 1; - for (value of dataset.data) { - max_value = Math.max(max_value, value); + if (is_bar) { + dataset.borderWidth = 1; + } else { + dataset.fill = false; } index++; } - window.myBar = new Chart( +} + +function findDatasetMaxValue(data) { + var max_value = 0; + for (dataset of data.datasets) { + for (value of dataset.data) { + if (value === null) { + continue; + } + if (typeof value !== "number") { + value = value.y; + } + max_value = Math.max(max_value, value); + } + } + return max_value; +} + +function buildChart(ctx, bare_data) { + var data = bare_data; + var max_value = findDatasetMaxValue(data); + setDatasetColor(data, true); + var my_chart = new Chart( ctx, { type: 'bar', @@ -99,6 +120,47 @@ function buildChart(ctx, bare_data) { }); } +function historyChartGetSceneStats(bare_data, scene_name) { + var datasets = []; + for (dataset of bare_data.datasets) { + if (dataset.scene_name != scene_name) { + continue; + } + datasets.push({"data": dataset.data, + "label": dataset.device_name}); + } + return {"datasets": datasets}; +} + +function buildHistoryChart(ctx, bare_data, scene_name) { + var data = historyChartGetSceneStats(bare_data, scene_name); + var max_value = findDatasetMaxValue(data); + setDatasetColor(data, false); + new Chart( + ctx, + { + type: 'line', + data: data, + options: { responsive: true, + legend: {position: 'top'}, + title: {display: true, + text: scene_name + ' benchmark results'}, + scales: {xAxes: [{type: "time", + time: {format: 'DD/MM/YYYY HH:mm', + tooltipFormat: 'll HH:mm'}, + scaleLabel: {display: true, + labelString: 'Date'}},], + yAxes: [{display: true, + scaleLabel: {display: true, + labelString: 'Render time (sec)'}, + ticks: {min: 0, + max: Math.ceil((max_value + 99) / 100) * 100}}]}, + + elements: { line: { tension: 0.000001 } }, + } + }); +} + function buildSpreadsheet(ctx, data) { // Generate columns for header. var columns = [{label: "", name: "", width: 300}];