simplify benchmarking multiple similar methods

lostmsu · lostmsu · commit 327c3112bdf1 · 2019-10-22T14:06:47.000-07:00
diff --git a/src/perf_tests/BenchmarkTests.cs b/src/perf_tests/BenchmarkTests.cs
@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Reflection;
 
 using BenchmarkDotNet.Reports;
@@ -12,25 +13,36 @@ namespace Python.PerformanceTests
 {
     public class BenchmarkTests
     {
+        Summary summary;
+
         [OneTimeSetUp]
         public void SetUp()
         {
             Environment.CurrentDirectory = Path.Combine(DeploymentRoot, "new");
+            this.summary = BenchmarkRunner.Run<PythonCallingNetBenchmark>();
+            Assert.IsNotEmpty(this.summary.Reports);
+            Assert.IsTrue(this.summary.Reports.All(r => r.Success));
         }
 
         [Test]
-        public void PythonCallingNet()
+        public void ReadInt64Property()
         {
-            var summary = BenchmarkRunner.Run<PythonCallingNetBenchmark>();
-
-            Assert.IsNotEmpty(summary.Reports);
-            Assert.IsTrue(summary.Reports.All(r => r.Success));
+            double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);
+            Assert.LessOrEqual(optimisticPerfRatio, 0.68);
+        }
 
-            double optimisticPerfRatio = GetOptimisticPerfRatio(summary.Reports);
-            Assert.LessOrEqual(optimisticPerfRatio, 1.03);
+        [Test]
+        public void WriteInt64Property()
+        {
+            double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);
+            Assert.LessOrEqual(optimisticPerfRatio, 0.66);
         }
 
-        static double GetOptimisticPerfRatio(IReadOnlyList<BenchmarkReport> reports) {
+        static double GetOptimisticPerfRatio(
+            IReadOnlyList<BenchmarkReport> reports,
+            [CallerMemberName] string methodName = null)
+        {
+            reports = reports.Where(r => r.BenchmarkCase.Descriptor.WorkloadMethod.Name == methodName).ToArray();
             var baseline = reports.Single(r => r.BenchmarkCase.Job.ResolvedId == "baseline").ResultStatistics;
             var @new = reports.Single(r => r.BenchmarkCase.Job.ResolvedId != "baseline").ResultStatistics;
             double newTimeOptimistic = @new.Mean - (@new.StandardDeviation + baseline.StandardDeviation) * 0.5;
diff --git a/src/perf_tests/PythonCallingNetBenchmark.cs b/src/perf_tests/PythonCallingNetBenchmark.cs
@@ -11,23 +11,36 @@ namespace Python.PerformanceTests
     public class PythonCallingNetBenchmark: BaselineComparisonBenchmarkBase
     {
         [Benchmark]
-        public void ReadIntProperty()
+        public void ReadInt64Property()
         {
             using (Py.GIL())
             {
                 var locals = new PyDict();
                 locals.SetItem("a", new NetObject().ToPython());
-                PythonEngine.Exec(@"
+                PythonEngine.Exec($@"
 s = 0
-for i in range(1000000):
-  s += a.IntProperty
+for i in range(300000):
+  s += a.{nameof(NetObject.LongProperty)}
+", locals: locals.Handle);
+            }
+        }
+
+        [Benchmark]
+        public void WriteInt64Property() {
+            using (Py.GIL()) {
+                var locals = new PyDict();
+                locals.SetItem("a", new NetObject().ToPython());
+                PythonEngine.Exec($@"
+s = 0
+for i in range(300000):
+  a.{nameof(NetObject.LongProperty)} += i
 ", locals: locals.Handle);
             }
         }
     }
 
     class NetObject
     {
-        public int IntProperty { get; set; } = 42;
+        public long LongProperty { get; set; } = 42;
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -11,23 +11,36 @@ namespace Python.PerformanceTests`
`11`	`11`	`public class PythonCallingNetBenchmark: BaselineComparisonBenchmarkBase`
`12`	`12`	`{`
`13`	`13`	`[Benchmark]`
`14`		`- public void ReadIntProperty()`
	`14`	`+ public void ReadInt64Property()`
`15`	`15`	`{`
`16`	`16`	`using (Py.GIL())`
`17`	`17`	`{`
`18`	`18`	`var locals = new PyDict();`
`19`	`19`	`locals.SetItem("a", new NetObject().ToPython());`
`20`		`- PythonEngine.Exec(@"`
	`20`	`+ PythonEngine.Exec($@"`
`21`	`21`	`s = 0`
`22`		`-for i in range(1000000):`
`23`		`- s += a.IntProperty`
	`22`	`+for i in range(300000):`
	`23`	`+ s += a.{nameof(NetObject.LongProperty)}`
	`24`	`+", locals: locals.Handle);`
	`25`	`+ }`
	`26`	`+ }`
	`27`	`+`
	`28`	`+ [Benchmark]`
	`29`	`+ public void WriteInt64Property() {`
	`30`	`+ using (Py.GIL()) {`
	`31`	`+ var locals = new PyDict();`
	`32`	`+ locals.SetItem("a", new NetObject().ToPython());`
	`33`	`+ PythonEngine.Exec($@"`
	`34`	`+s = 0`
	`35`	`+for i in range(300000):`
	`36`	`+ a.{nameof(NetObject.LongProperty)} += i`
`24`	`37`	`", locals: locals.Handle);`
`25`	`38`	`}`
`26`	`39`	`}`
`27`	`40`	`}`
`28`	`41`
`29`	`42`	`class NetObject`
`30`	`43`	`{`
`31`		`- public int IntProperty { get; set; } = 42;`
	`44`	`+ public long LongProperty { get; set; } = 42;`
`32`	`45`	`}`
`33`	`46`	`}`