benchmark: added stddev to bar charts

1 files changed, 21 insertions, 8 deletions

diff --git a/benchmark/run_benchmarks.py b/benchmark/run_benchmarks.py
index 5eebbab..7686451 100644
--- a/benchmark/run_benchmarks.py
+++ b/benchmark/run_benchmarks.py
@@ -15,6 +15,8 @@ cpu_count = multiprocessing.cpu_count()
 
 from bm_registry import BENCHMARKS
 
+NUM_RUNS_PER_TEST = 5
+
 __path__ = os.path.dirname(__file__)
 
 def time_a_single_one(args, env):
@@ -30,7 +32,8 @@ def do_a_benchmark(bm):
     print '#'*(len(title)+25)
     print '## running benchmark:', title
     print '#'*(len(title)+25)
-    results = list()
+    result_means = list()
+    result_stddevs = list()
     test_names = list()
     for run in bm['tests']:
         test_name = run['wat']
@@ -45,11 +48,18 @@ def do_a_benchmark(bm):
         env = copy.copy(env)
         if run.has_key('envextra'):
             env.update(run['envextra'])
-        r = time_a_single_one(args=args, env=env)
-        results.append(bm['to_result'](r))
-    print results
+        run_results = list()
+        for num_runs in range(NUM_RUNS_PER_TEST):
+            t = time_a_single_one(args=args, env=env)
+            run_results.append(bm['to_result'](t))
+        result_means.append(numpy.average(run_results))
+        result_stddevs.append(numpy.std(run_results))
+
+    print 'result_means', result_means
+    print 'result_stddevs', result_stddevs
 
-    bogomips = numpy.array(results)/1e6
+    bogomips = numpy.array(result_means)/1e6
+    bogomips_dev = numpy.array(result_stddevs)/1e6
     ind = numpy.arange(len(test_names))
     width = 0.35
     fig = Figure()
@@ -59,12 +69,15 @@ def do_a_benchmark(bm):
         ylabel='Performance (BogoMips)', title=title,
         xlabel='', xticks=ind+width/2., xticklabels=test_names
     )
-    rects = ax.bar(ind, bogomips, width, color='blue')
+    rects = ax.bar(ind, bogomips, width, color='blue',
+        yerr=bogomips_dev,
+        error_kw=dict(elinewidth=6, ecolor='pink'),
+    )
     ax.set_xlim(-width, max(len(ind), 4))
-    ax.set_ylim(0, max(*bogomips) + numpy.std(bogomips))
+    ax.set_ylim(0, max(*bogomips) + max(*bogomips_dev)*3)
     for rect in rects:
         height = rect.get_height()
-        ax.text(rect.get_x()+rect.get_width()/2.0, 1.03*height, '%.3f'%height, horizontalalignment='center')
+        ax.text(rect.get_x()+rect.get_width()/2.0, 1.01*height, '%.3f'%height, horizontalalignment='center')
     ax.grid(True)
     print '\n'
     return fig