Initial support for history over device scenes

This patch will generate html file which contains per-scene graph ofg all device
statistics over all period of time. There are still some possible tweaks to look
and feel of the graphs, but it's good enough for now.

benchmark/parse_results.py

@@ -207,8 +207,6 @@ class LatestResultVisitor(ResultVisitor):
                     data.append(None)
                     continue
                 scene_stats = stats[scene_name]
-                # data.append({"x": scene_name,
-                #             "y": scene_stats['pipeline_render_time']})
                 data.append(scene_stats['pipeline_render_time'])
             dataset = {
                 "label": device_name,
@@ -228,6 +226,93 @@ class LatestResultVisitor(ResultVisitor):
             f.write(code)
 
 
+class HistoryResultVisitor(ResultVisitor):
+    def __init__(self):
+        ResultVisitor.__init__(self)
+        self.devices_ = {}
+
+    def copyUsableStats(self, device, results):
+        stats = results['stats']
+        blender_version = results['blender_version']
+        timestamp = util.blenderCommitUnixTimestamp(
+            blender_version['build_commit_date'],
+            blender_version['build_commit_time'])
+        for scene_name in stats:
+            stat = stats[scene_name]
+            # Ignore benchmark results which crashed or aborted.
+            if "result" not in stat or stat['result'] != 'OK':
+                continue
+            device_scene_stat = dict(stat)
+            device_scene_stat.pop("result")
+            device_scene_stat['timestamp'] = timestamp
+            if scene_name not in device:
+                device[scene_name] = []
+            device[scene_name].append(device_scene_stat)
+
+    def handleResults(self, results):
+        if 'device_info' not in results or \
+           'stats' not in results:
+            return
+        device_info = results['device_info']
+        device_name = util.deviceInfoAsString(device_info)
+        # If there were no stats for the device,
+        if device_name not in self.devices_:
+            self.devices_[device_name] = {}
+        self.copyUsableStats(self.devices_[device_name], results)
+
+    def removeDuplicated(self, stats_history):
+        new_stats_history = []
+        prev_timestamp = None
+        for stats in stats_history:
+            if stats['timestamp'] == prev_timestamp:
+                # TODO(sergey): Average somehow?
+                continue
+            new_stats_history.append(stats)
+            prev_timestamp = stats['timestamp']
+        return new_stats_history
+
+    def storeResults(self, dataset_dir):
+        # Firts of all, gather all possible scenes.
+        all_scenes = set()
+        for device_name, stats in self.devices_.items():
+            for scene_name in stats.keys():
+                all_scenes.add(scene_name)
+        all_scenes = sorted(list(all_scenes))
+        # Gather datasets in format of lines.
+        datasets = []
+        device_index = 0
+        for device_name in sorted(self.devices_.keys()):
+            stats_history = self.devices_[device_name]
+            for scene_name in all_scenes:
+                if scene_name not in stats_history:
+                    continue
+                scene_stats_history = stats_history[scene_name]
+                data = []
+                sorted_scene_stats_history = sorted(scene_stats_history,
+                                                    key=lambda k: k['timestamp'])
+                uniq_scene_stats_history = self.removeDuplicated(
+                        sorted_scene_stats_history)
+                for scene_stats in uniq_scene_stats_history:
+                    timestamp = scene_stats['timestamp']
+                    data.append({"x": timestamp.strftime("%d/%m/%y %H:%M"),
+                                 "y": scene_stats['pipeline_render_time']})
+                dataset = {
+                    "device_name": device_name,
+                    "scene_name": scene_name,
+                    "data": data,
+                   }
+                datasets.append(dataset)
+                device_index += 1
+        data = {
+            "scenes": all_scenes,
+            "datasets": datasets,
+        }
+        code = "var data = " + json.dumps(data, sort_keys=True, indent=2) + ";"
+        # Save dataset to disk.
+        filename = os.path.join(dataset_dir, "history.js")
+        with open(filename, "w") as f:
+            f.write(code)
+
 def main():
     parser = configureArgumentParser()
     args = parser.parse_args()
@@ -239,9 +324,11 @@ def main():
         return False
     # Construct list of all visitors whic hwe want.
     latest_results_visitor = LatestResultVisitor()
+    history_results_visitor = HistoryResultVisitor()
     # Do actual parse.
     logger.INFO("Iterating over all benchmark results...")
-    visitors = [latest_results_visitor]
+    visitors = (latest_results_visitor,
+                history_results_visitor,)
     iterateBenchmarksRoot(config['parser']['benchmark_dir'], visitors)
     # Store results.
     logger.INFO("Storing results...")

benchmark/website/history.html

@@ -0,0 +1,32 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Benchmark Results</title>
+    <script src="source/chart.bundle.js"></script>
+    <script src="source/utils.js"></script>
+    <script src="source/jquery.min.js"></script>
+    <script src="source/jquery.jqGrid.min.js"></script>
+    <script src="source/bootstrap.min.js"></script>
+    <script src="dataset/history.js"></script>
+    <link rel="stylesheet" type="text/css" href="style/main.css">
+    <link rel="stylesheet" type="text/css" href="style/bootstrap.min.css" />
+    <link rel="stylesheet" type="text/css" href="style/ui.jqgrid-bootstrap.css" />
+  </head>
+  <body>
+    <div id="container">
+    </div>
+    <br><br>
+    <table id="spreadsheet"></table>
+    <script type="text/javascript">
+      window.onload = function() {
+        var container = document.getElementById("container");
+        for (scene_name of data.scenes) {
+          var canvas = document.createElement("canvas");
+          container.appendChild(canvas);
+          var chart_ctx = canvas.getContext("2d");
+          buildHistoryChart(chart_ctx, data, scene_name);
+        }
+      };
+    </script>
+  </body>
+</html>

benchmark/website/source/utils.js

@@ -43,21 +43,42 @@ function secondsToHumanReadable(seconds) {
   return result;
 }
 
-function buildChart(ctx, bare_data) {
-  var data = bare_data;
-  var max_value = 0;
+function setDatasetColor(data, is_bar) {
   var index = 0;
   for (dataset of data.datasets) {
     var color = Chart.helpers.color(generateBarColor(index));
     dataset.backgroundColor = color.alpha(0.5).rgbString();
     dataset.borderColor = color.alpha(1.0).rgbString();
-    dataset.borderWidth = 1;
-    for (value of dataset.data) {
-      max_value = Math.max(max_value, value);
+    if (is_bar) {
+      dataset.borderWidth = 1;
+    } else {
+      dataset.fill = false;
     }
     index++;
   }
-  window.myBar = new Chart(
+}
+
+function findDatasetMaxValue(data) {
+  var max_value = 0;
+  for (dataset of data.datasets) {
+    for (value of dataset.data) {
+      if (value === null) {
+        continue;
+      }
+      if (typeof value !== "number") {
+          value = value.y;
+      }
+      max_value = Math.max(max_value, value);
+    }
+  }
+  return max_value;
+}
+
+function buildChart(ctx, bare_data) {
+  var data = bare_data;
+  var max_value = findDatasetMaxValue(data);
+  setDatasetColor(data, true);
+  var my_chart = new Chart(
       ctx,
       {
         type: 'bar',
@@ -99,6 +120,47 @@ function buildChart(ctx, bare_data) {
       });
 }
 
+function historyChartGetSceneStats(bare_data, scene_name) {
+  var datasets = [];
+  for (dataset of bare_data.datasets) {
+    if (dataset.scene_name != scene_name) {
+      continue;
+    }
+    datasets.push({"data": dataset.data,
+                   "label": dataset.device_name});
+  }
+  return {"datasets": datasets};
+}
+
+function buildHistoryChart(ctx, bare_data, scene_name) {
+  var data = historyChartGetSceneStats(bare_data, scene_name);
+  var max_value = findDatasetMaxValue(data);
+  setDatasetColor(data, false);
+  new Chart(
+      ctx,
+      {
+        type: 'line',
+        data: data,
+        options: { responsive: true,
+                   legend: {position: 'top'},
+                   title: {display: true,
+                           text: scene_name + ' benchmark results'},
+                   scales: {xAxes:  [{type: "time",
+                                      time: {format: 'DD/MM/YYYY HH:mm',
+                                             tooltipFormat: 'll HH:mm'},
+                                      scaleLabel: {display: true,
+                                                   labelString: 'Date'}},],
+                            yAxes: [{display: true,
+                                     scaleLabel: {display: true,
+                                                  labelString: 'Render time (sec)'},
+                                     ticks: {min: 0,
+                                             max: Math.ceil((max_value + 99) / 100) * 100}}]},
+
+                            elements: { line: { tension: 0.000001 } },
+                }
+      });
+}
+
 function buildSpreadsheet(ctx, data) {
   // Generate columns for header.
   var columns = [{label: "", name: "", width: 300}];