implemented PythonCallingNet benchmark test

lostmsu · lostmsu · commit 2e28711d07e7 · 2019-10-21T17:02:19.000-07:00
diff --git a/src/perf_tests/BaselineComparisonConfig.cs b/src/perf_tests/BaselineComparisonConfig.cs
@@ -20,10 +20,12 @@ public BaselineComparisonConfig()
 
             var baseJob = Job.Default;
             this.Add(baseJob
+                .WithId("baseline")
                 .WithEnvironmentVariable(EnvironmentVariableName,
                     Path.Combine(deploymentRoot, "baseline", "Python.Runtime.dll"))
-                .AsBaseline());
+                .WithBaseline(true));
             this.Add(baseJob
+                .WithId("new")
                 .WithEnvironmentVariable(EnvironmentVariableName,
                     Path.Combine(deploymentRoot, "new", "Python.Runtime.dll")));
         }
diff --git a/src/perf_tests/BenchmarkTests.cs b/src/perf_tests/BenchmarkTests.cs
@@ -1,7 +1,10 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Reflection;
 
+using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Running;
 using NUnit.Framework;
 
@@ -19,8 +22,19 @@ public void SetUp()
         public void PythonCallingNet()
         {
             var summary = BenchmarkRunner.Run<PythonCallingNetBenchmark>();
-            Console.WriteLine(summary);
-            Assert.Inconclusive();
+
+            Assert.IsNotEmpty(summary.Reports);
+            Assert.IsTrue(summary.Reports.All(r => r.Success));
+
+            double optimisticPerfRatio = GetOptimisticPerfRatio(summary.Reports);
+            Assert.LessOrEqual(optimisticPerfRatio, 1.03);
+        }
+
+        static double GetOptimisticPerfRatio(IReadOnlyList<BenchmarkReport> reports) {
+            var baseline = reports.Single(r => r.BenchmarkCase.Job.ResolvedId == "baseline").ResultStatistics;
+            var @new = reports.Single(r => r.BenchmarkCase.Job.ResolvedId != "baseline").ResultStatistics;
+            double newTimeOptimistic = @new.Mean - (@new.StandardDeviation + baseline.StandardDeviation) * 0.5;
+            return newTimeOptimistic / baseline.Mean;
         }
 
         public static string DeploymentRoot => Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
