diff --git a/.travis.yml b/.travis.yml index 9689c0422..c69ccbc5d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,8 @@ python: env: matrix: - - BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ - - BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" + - BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ PERF_TESTS_PATH=net461/ + - BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" PERF_TESTS_PATH="" global: - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so @@ -45,6 +45,8 @@ install: script: - python -m pytest - $RUN_TESTS src/embed_tests/bin/$EMBED_TESTS_PATH/Python.EmbeddingTest.dll + # does not work on Linux, because NuGet package for 2.3 is Windows only + # - "if [[ $TRAVIS_PYTHON_VERSION == '3.5' && $PERF_TESTS_PATH != '' ]]; then mono $NUNIT_PATH src/perf_tests/bin/$PERF_TESTS_PATH/Python.PerformanceTests.dll; fi" after_script: # Waiting on mono-coverage, SharpCover or xr.Baboon diff --git a/ci/appveyor_run_tests.ps1 b/ci/appveyor_run_tests.ps1 index b45440fbe..f94cfb11e 100644 --- a/ci/appveyor_run_tests.ps1 +++ b/ci/appveyor_run_tests.ps1 @@ -3,6 +3,8 @@ # Test Runner framework being used for embedded tests $CS_RUNNER = "nunit3-console" +$XPLAT = $env:BUILD_OPTS -eq "--xplat" + # Needed for ARCH specific runners(NUnit2/XUnit3). Skip for NUnit3 if ($FALSE -and $env:PLATFORM -eq "x86"){ $CS_RUNNER = $CS_RUNNER + "-x86" @@ -11,7 +13,7 @@ if ($FALSE -and $env:PLATFORM -eq "x86"){ # Executable paths for OpenCover # Note if OpenCover fails, it won't affect the exit codes. $OPENCOVER = Resolve-Path .\packages\OpenCover.*\tools\OpenCover.Console.exe -if ($env:BUILD_OPTS -eq "--xplat"){ +if ($XPLAT){ $CS_RUNNER = Resolve-Path $env:USERPROFILE\.nuget\packages\nunit.consolerunner\*\tools\"$CS_RUNNER".exe } else{ @@ -42,9 +44,31 @@ Write-Host ("Starting embedded tests") -ForegroundColor "Green" $CS_STATUS = $LastExitCode if ($CS_STATUS -ne 0) { Write-Host "Embedded tests failed" -ForegroundColor "Red" +} else { + # NuGet for pythonnet-2.3 only has 64-bit binary for Python 3.5 + # the test is only built using modern stack + if (($env:PLATFORM -eq "x64") -and ($XPLAT) -and ($env:PYTHON_VERSION -eq "3.5")) { + # Run C# Performance tests + Write-Host ("Starting performance tests") -ForegroundColor "Green" + if ($XPLAT) { + $CS_PERF_TESTS = ".\src\perf_tests\bin\net461\Python.PerformanceTests.dll" + } + else { + $CS_PERF_TESTS = ".\src\perf_tests\bin\Python.PerformanceTests.dll" + } + &"$CS_RUNNER" "$CS_PERF_TESTS" + $CS_PERF_STATUS = $LastExitCode + if ($CS_PERF_STATUS -ne 0) { + Write-Host "Performance tests (C#) failed" -ForegroundColor "Red" + } + } else { + Write-Host ("Skipping performance tests for ", $env:PYTHON_VERSION) -ForegroundColor "Yellow" + Write-Host ("on platform ", $env:PLATFORM, " xplat: ", $XPLAT) -ForegroundColor "Yellow" + $CS_PERF_STATUS = 0 + } } -if ($env:BUILD_OPTS -eq "--xplat"){ +if ($XPLAT){ if ($env:PLATFORM -eq "x64") { $DOTNET_CMD = "dotnet" } @@ -54,7 +78,7 @@ if ($env:BUILD_OPTS -eq "--xplat"){ # Run Embedded tests for netcoreapp2.0 (OpenCover currently does not supports dotnet core) Write-Host ("Starting embedded tests for netcoreapp2.0") -ForegroundColor "Green" - &$DOTNET_CMD .\src\embed_tests\bin\netcoreapp2.0_publish\Python.EmbeddingTest.dll + &$DOTNET_CMD ".\src\embed_tests\bin\netcoreapp2.0_publish\Python.EmbeddingTest.dll" $CS_STATUS = $LastExitCode if ($CS_STATUS -ne 0) { Write-Host "Embedded tests for netcoreapp2.0 failed" -ForegroundColor "Red" @@ -62,7 +86,7 @@ if ($env:BUILD_OPTS -eq "--xplat"){ } # Set exit code to fail if either Python or Embedded tests failed -if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0) { +if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0 -or $CS_PERF_STATUS -ne 0) { Write-Host "Tests failed" -ForegroundColor "Red" $host.SetShouldExit(1) } diff --git a/pythonnet.15.sln b/pythonnet.15.sln index 096dfbe9a..6d1f4fcd9 100644 --- a/pythonnet.15.sln +++ b/pythonnet.15.sln @@ -19,6 +19,26 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F .editorconfig = .editorconfig EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CI", "CI", "{D301657F-5EAF-4534-B280-B858D651B2E5}" + ProjectSection(SolutionItems) = preProject + .travis.yml = .travis.yml + appveyor.yml = appveyor.yml + ci\appveyor_build_recipe.ps1 = ci\appveyor_build_recipe.ps1 + ci\appveyor_run_tests.ps1 = ci\appveyor_run_tests.ps1 + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{57F5D701-F265-4736-A5A2-07249E7A4DA3}" + ProjectSection(SolutionItems) = preProject + setup.py = setup.py + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "conda.recipe", "conda.recipe", "{7FD2404D-0CE8-4645-8DFB-766470E2150E}" + ProjectSection(SolutionItems) = preProject + conda.recipe\bld.bat = conda.recipe\bld.bat + conda.recipe\meta.yaml = conda.recipe\meta.yaml + conda.recipe\README.md = conda.recipe\README.md + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -313,54 +333,58 @@ Global {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.ActiveCfg = DebugMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.Build.0 = DebugMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.ActiveCfg = DebugMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.Build.0 = DebugMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.ActiveCfg = DebugMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.Build.0 = DebugMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|Any CPU.ActiveCfg = DebugMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.ActiveCfg = DebugMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.ActiveCfg = DebugMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.ActiveCfg = DebugWin|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.Build.0 = DebugWin|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.ActiveCfg = DebugWin|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.Build.0 = DebugWin|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.ActiveCfg = DebugWin|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.Build.0 = DebugWin|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|Any CPU.Build.0 = DebugWinPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.Build.0 = DebugWinPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.Build.0 = DebugWinPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.ActiveCfg = ReleaseMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.Build.0 = ReleaseMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.ActiveCfg = ReleaseMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.Build.0 = ReleaseMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.ActiveCfg = ReleaseMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.Build.0 = ReleaseMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.ActiveCfg = ReleaseMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.ActiveCfg = ReleaseMono|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.Build.0 = ReleaseWin|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.ActiveCfg = ReleaseWin|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.Build.0 = ReleaseWin|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.ActiveCfg = ReleaseWin|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.Build.0 = ReleaseWin|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|Any CPU.Build.0 = ReleaseWinPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|Any CPU - {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.ActiveCfg = ReleaseWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.Build.0 = ReleaseWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.Build.0 = DebugWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.Build.0 = DebugWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.ActiveCfg = DebugMono|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.Build.0 = DebugMono|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.ActiveCfg = DebugMono|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.Build.0 = DebugMono|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.ActiveCfg = DebugWin|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.Build.0 = DebugWin|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.ActiveCfg = DebugWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.Build.0 = DebugWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.ActiveCfg = ReleaseWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.Build.0 = ReleaseWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/setup.py b/setup.py index c6e4007e6..db2b4ae68 100644 --- a/setup.py +++ b/setup.py @@ -362,6 +362,16 @@ def build_extension(self, ext): ), shell=use_shell, ) + subprocess.check_call( + " ".join( + cmd + + [ + '"/t:Python_PerformanceTests:publish"', + "/p:TargetFramework=net461", + ] + ), + shell=use_shell, + ) if DEVTOOLS == "Mono" or DEVTOOLS == "dotnet": self._build_monoclr() diff --git a/src/perf_tests/BenchmarkTests.cs b/src/perf_tests/BenchmarkTests.cs index 12ba6c900..baa825ca8 100644 --- a/src/perf_tests/BenchmarkTests.cs +++ b/src/perf_tests/BenchmarkTests.cs @@ -21,21 +21,23 @@ public void SetUp() Environment.CurrentDirectory = Path.Combine(DeploymentRoot, "new"); this.summary = BenchmarkRunner.Run(); Assert.IsNotEmpty(this.summary.Reports); - Assert.IsTrue(this.summary.Reports.All(r => r.Success)); + Assert.IsTrue( + condition: this.summary.Reports.All(r => r.Success), + message: "BenchmarkDotNet failed to execute or collect results of performance tests. See logs above."); } [Test] public void ReadInt64Property() { double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports); - Assert.LessOrEqual(optimisticPerfRatio, 0.68); + AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.66); } [Test] public void WriteInt64Property() { double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports); - Assert.LessOrEqual(optimisticPerfRatio, 0.66); + AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.64); } static double GetOptimisticPerfRatio( @@ -59,5 +61,14 @@ static double GetOptimisticPerfRatio( } public static string DeploymentRoot => Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + + public static void AssertPerformanceIsBetterOrSame( + double actual, double target, + double wiggleRoom = 1.1, [CallerMemberName] string testName = null) { + double threshold = target * wiggleRoom; + Assert.LessOrEqual(actual, threshold, + $"{testName}: {actual:F3} > {threshold:F3} (target: {target:F3})" + + ": perf result is higher than the failure threshold."); + } } } diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj index 33949fdc1..1231cef69 100644 --- a/src/perf_tests/Python.PerformanceTests.csproj +++ b/src/perf_tests/Python.PerformanceTests.csproj @@ -1,14 +1,21 @@ - + net461 DebugMono;DebugMonoPY3;ReleaseMono;ReleaseMonoPY3;DebugWin;DebugWinPY3;ReleaseWin;ReleaseWinPY3 + bin\ false + + x64;x86 + + all + runtime; build; native; contentfiles; analyzers; buildtransitive +