diff --git a/.github/workflows/publish-nightly.yaml b/.github/workflows/publish-nightly.yaml index 3a3dc0bc9b..f3e025a6de 100644 --- a/.github/workflows/publish-nightly.yaml +++ b/.github/workflows/publish-nightly.yaml @@ -11,13 +11,13 @@ jobs: runs-on: ubuntu-latest if: ${{ github.repository == 'dotnet/BenchmarkDotNet' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set date run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV - name: Pack run: ./build.cmd pack /p:VersionSuffix=nightly.$DATE.$GITHUB_RUN_NUMBER - name: Upload nupkg to artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: nupkgs path: "**/*.*nupkg" diff --git a/.github/workflows/report-test-results.yaml b/.github/workflows/report-test-results.yaml index 005465b329..77aafa2fe7 100644 --- a/.github/workflows/report-test-results.yaml +++ b/.github/workflows/report-test-results.yaml @@ -12,12 +12,24 @@ jobs: runs-on: ubuntu-latest permissions: write-all steps: + # Cleanup Old Files + - name: Cleanup Old Files + run: rm -rf $GITHUB_WORKSPACE/*.trx + + # Download the Latest Artifacts with Unique Name - name: Download Artifacts - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v6 with: - workflow: ${{ github.event.workflow_run.workflow_id }} + run_id: ${{ github.event.workflow_run.id }} + + # Display the Structure of Downloaded Files - name: Display structure of downloaded files run: ls -R + + # Display the Contents of .trx Files + - name: Display .trx file contents + run: cat **/*.trx || echo "No .trx files found" + - name: Report tests results uses: AndreyAkinshin/test-reporter@0e2c48ebec2007001dd77dd4bcbcd450b96d5a38 with: diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 90887cbecd..2856388209 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -17,22 +17,20 @@ jobs: - name: Disable Windows Defender run: Set-MpPreference -DisableRealtimeMonitoring $true shell: powershell - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + # Build and Test - name: Run task 'build' shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - ./build.cmd build + run: ./build.cmd build - name: Run task 'in-tests-core' shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - ./build.cmd in-tests-core -e + run: ./build.cmd in-tests-core -e + # Upload Artifacts with Unique Name - name: Upload test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: - name: test-windows-core-trx + name: test-windows-core-trx-${{ github.run_id }} path: "**/*.trx" test-windows-full: @@ -41,28 +39,27 @@ jobs: - name: Disable Windows Defender run: Set-MpPreference -DisableRealtimeMonitoring $true shell: powershell - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + # Build and Test - name: Run task 'build' shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - ./build.cmd build + run: ./build.cmd build - name: Run task 'in-tests-full' shell: cmd - run: | - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" - ./build.cmd in-tests-full -e + run: ./build.cmd in-tests-full -e + # Upload Artifacts with Unique Name - name: Upload test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: - name: test-windows-full-trx + name: test-windows-full-trx-${{ github.run_id }} path: "**/*.trx" test-linux: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + # Set up the environment - name: Set up Clang uses: egor-tensin/setup-clang@v1 with: @@ -70,46 +67,80 @@ jobs: platform: x64 - name: Set up zlib-static run: sudo apt-get install -y libkrb5-dev + - name: Set up node + uses: actions/setup-node@v4 + with: + node-version: "20" + - name: Set up v8 + run: npm install jsvu -g && jsvu --os=linux64 --engines=v8 && echo "$HOME/.jsvu/bin" >> $GITHUB_PATH + - name: Install wasm-tools workload + run: ./build.cmd install-wasm-tools + # Build and Test - name: Run task 'build' run: ./build.cmd build - name: Run task 'unit-tests' run: ./build.cmd unit-tests -e - name: Run task 'in-tests-core' run: ./build.cmd in-tests-core -e + # Upload Artifacts with Unique Name - name: Upload test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: - name: test-linux-trx + name: test-linux-trx-${{ github.run_id }} path: "**/*.trx" test-macos: runs-on: macos-13 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - name: Set up node + uses: actions/setup-node@v4 + with: + node-version: "20" + - name: Set up v8 + run: npm install jsvu -g && jsvu --os=mac64 --engines=v8 && echo "$HOME/.jsvu/bin" >> $GITHUB_PATH + - name: Install wasm-tools workload + run: ./build.cmd install-wasm-tools + # Build and Test - name: Run task 'build' run: ./build.cmd build - name: Run task 'unit-tests' run: ./build.cmd unit-tests -e - name: Run task 'in-tests-core' run: ./build.cmd in-tests-core -e + # Upload Artifacts with Unique Name - name: Upload test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: - name: test-macos-trx + name: test-macos-trx-${{ github.run_id }} path: "**/*.trx" - + + test-pack: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Clang + uses: egor-tensin/setup-clang@v1 + with: + version: latest + platform: x64 + - name: Set up zlib-static + run: sudo apt-get install -y libkrb5-dev + - name: Run task 'pack' + run: ./build.cmd pack + spellcheck-docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 name: Setup node with: - node-version: "16" + node-version: "18" - name: Install cSpell - run: npm install -g cspell + run: npm install -g cspell@8.0.0 - name: Copy cSpell config run: cp ./build/cSpell.json ./cSpell.json - name: Run cSpell diff --git a/BenchmarkDotNet.sln b/BenchmarkDotNet.sln index ba31552c8a..1df6c0aabd 100644 --- a/BenchmarkDotNet.sln +++ b/BenchmarkDotNet.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.27130.2027 +# Visual Studio Version 17 +VisualStudioVersion = 17.8.34004.107 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{D6597E3A-6892-4A68-8E14-042FC941FDA2}" EndProject @@ -51,6 +51,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkDotNet.Diagnostics EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks", "tests\BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks\BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj", "{AACA2C63-A85B-47AB-99FC-72C3FF408B14}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.TestAdapter", "src\BenchmarkDotNet.TestAdapter\BenchmarkDotNet.TestAdapter.csproj", "{4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Diagnostics.dotMemory", "src\BenchmarkDotNet.Diagnostics.dotMemory\BenchmarkDotNet.Diagnostics.dotMemory.csproj", "{2E2283A3-6DA6-4482-8518-99D6D9F689AB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting", "src\BenchmarkDotNet.Exporters.Plotting\BenchmarkDotNet.Exporters.Plotting.csproj", "{B92ECCEF-7C27-4012-9E19-679F3C40A6A6}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BenchmarkDotNet.Exporters.Plotting.Tests", "tests\BenchmarkDotNet.Exporters.Plotting.Tests\BenchmarkDotNet.Exporters.Plotting.Tests.csproj", "{199AC83E-30BD-40CD-87CE-0C838AC0320D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -137,6 +145,22 @@ Global {AACA2C63-A85B-47AB-99FC-72C3FF408B14}.Debug|Any CPU.Build.0 = Debug|Any CPU {AACA2C63-A85B-47AB-99FC-72C3FF408B14}.Release|Any CPU.ActiveCfg = Release|Any CPU {AACA2C63-A85B-47AB-99FC-72C3FF408B14}.Release|Any CPU.Build.0 = Release|Any CPU + {4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC}.Release|Any CPU.Build.0 = Release|Any CPU + {2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2E2283A3-6DA6-4482-8518-99D6D9F689AB}.Release|Any CPU.Build.0 = Release|Any CPU + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6}.Release|Any CPU.Build.0 = Release|Any CPU + {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {199AC83E-30BD-40CD-87CE-0C838AC0320D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -162,6 +186,10 @@ Global {B620D10A-CD8E-4A34-8B27-FD6257E63AD0} = {63B94FD6-3F3D-4E04-9727-48E86AC4384C} {C5BDA61F-3A56-4B59-901D-0A17E78F4076} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} {AACA2C63-A85B-47AB-99FC-72C3FF408B14} = {14195214-591A-45B7-851A-19D3BA2413F9} + {4C9C89B8-7C4E-4ECF-B3C9-324C8772EDAC} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} + {2E2283A3-6DA6-4482-8518-99D6D9F689AB} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} + {B92ECCEF-7C27-4012-9E19-679F3C40A6A6} = {D6597E3A-6892-4A68-8E14-042FC941FDA2} + {199AC83E-30BD-40CD-87CE-0C838AC0320D} = {14195214-591A-45B7-851A-19D3BA2413F9} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {4D9AF12B-1F7F-45A7-9E8C-E4E46ADCBD1F} diff --git a/BenchmarkDotNet.sln.DotSettings b/BenchmarkDotNet.sln.DotSettings index 9542bd9d5b..fc698670f8 100644 --- a/BenchmarkDotNet.sln.DotSettings +++ b/BenchmarkDotNet.sln.DotSettings @@ -52,6 +52,8 @@ OSX RT <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> + <Policy><Descriptor Staticness="Instance" AccessRightKinds="Private" Description="Instance fields (private)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> + <Policy><Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static fields (private)"><ElementKinds><Kind Name="FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> True True 983040 @@ -68,6 +70,7 @@ True <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> + True <data><IncludeFilters /><ExcludeFilters><Filter ModuleMask="*" ModuleVersionMask="*" ClassMask="JetBrains.Annotations.*" FunctionMask="*" IsEnabled="True" /><Filter ModuleMask="*" ModuleVersionMask="*" ClassMask="SimpleJson.*" FunctionMask="*" IsEnabled="True" /></ExcludeFilters></data> True True @@ -162,6 +165,7 @@ True True True + True True True True @@ -173,6 +177,7 @@ True True True + True True True True diff --git a/LICENSE.md b/LICENSE.md index d20c387e81..6272f97306 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ ### The MIT License -Copyright (c) 2013–2023 .NET Foundation and contributors +Copyright (c) 2013–2025 .NET Foundation and contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/NuGet.Config b/NuGet.Config index 2b16f5257e..7507704b8b 100644 --- a/NuGet.Config +++ b/NuGet.Config @@ -12,5 +12,7 @@ + + diff --git a/README.md b/README.md index 424439148a..315215f058 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,10 @@
[![NuGet](https://img.shields.io/nuget/v/BenchmarkDotNet.svg)](https://www.nuget.org/packages/BenchmarkDotNet/) - [![MyGet](https://img.shields.io/myget/benchmarkdotnet/vpre/benchmarkdotnet?label=myget)](https://www.myget.org/feed/Packages/benchmarkdotnet) + [![MyGet](https://img.shields.io/myget/benchmarkdotnet/vpre/benchmarkdotnet?label=myget)](https://www.myget.org/feed/benchmarkdotnet/package/nuget/BenchmarkDotNet) [![Downloads](https://img.shields.io/nuget/dt/benchmarkdotnet.svg)](https://www.nuget.org/packages/BenchmarkDotNet/) [![Stars](https://img.shields.io/github/stars/dotnet/BenchmarkDotNet?color=brightgreen)](https://github.com/dotnet/BenchmarkDotNet/stargazers) - ![License](https://img.shields.io/badge/license-MIT-blue.svg) + [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/dotnet/BenchmarkDotNet/blob/master/LICENSE.md) [![Twitter](https://img.shields.io/twitter/follow/BenchmarkDotNet?style=social&label=Twitter)](https://twitter.com/BenchmarkDotNet)
@@ -30,7 +30,7 @@ It's no harder than writing unit tests! Under the hood, it performs a lot of [magic](#automation) that guarantees [reliable and precise](#reliability) results thanks to the [perfolizer](https://github.com/AndreyAkinshin/perfolizer) statistical engine. BenchmarkDotNet protects you from popular benchmarking mistakes and warns you if something is wrong with your benchmark design or obtained measurements. The results are presented in a [user-friendly](#friendliness) form that highlights all the important facts about your experiment. -BenchmarkDotNet is already adopted by [17400+ GitHub projects](https://github.com/dotnet/BenchmarkDotNet/network/dependents) including +BenchmarkDotNet is already adopted by [25800+ GitHub projects](https://github.com/dotnet/BenchmarkDotNet/network/dependents) including [.NET Runtime](https://github.com/dotnet/runtime), [.NET Compiler](https://github.com/dotnet/roslyn), [.NET Performance](https://github.com/dotnet/performance), @@ -125,7 +125,7 @@ Four aspects define the design of these features: ### Simplicity -You shouldn't be an experienced performance engineer if you want to write benchmarks. +You shouldn't have to be an experienced performance engineer if you want to write benchmarks. You can design very complicated performance experiments in the declarative style using simple APIs. For example, if you want to [parameterize](https://benchmarkdotnet.org/articles/features/parameterization.html) your benchmark, @@ -135,8 +135,8 @@ If you want to compare benchmarks with each other, mark one of the benchmarks as the [baseline](https://benchmarkdotnet.org/articles/features/baselines.html) via `[Benchmark(Baseline = true)]`: BenchmarkDotNet will compare it with all of the other benchmarks. If you want to compare performance in different environments, use [jobs](https://benchmarkdotnet.org/articles/configs/jobs.html). -For example, you can run all the benchmarks on .NET Core 3.0 and Mono via - `[SimpleJob(RuntimeMoniker.NetCoreApp30)]` and `[SimpleJob(RuntimeMoniker.Mono)]`. +For example, you can run all the benchmarks on .NET 8.0 and Mono via + `[SimpleJob(RuntimeMoniker.Net80)]` and `[SimpleJob(RuntimeMoniker.Mono)]`. If you don't like attributes, you can call most of the APIs via the fluent style and write code like this: diff --git a/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj b/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj index 3b5596f3b5..c7b97bc90a 100644 --- a/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj +++ b/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj @@ -1,15 +1,15 @@ Exe - net7.0 + net8.0 $(MSBuildProjectDirectory) enable - - - - - + + + + + - \ No newline at end of file + diff --git a/build/BenchmarkDotNet.Build/BuildContext.cs b/build/BenchmarkDotNet.Build/BuildContext.cs index a13c77c5ed..0a165721e0 100644 --- a/build/BenchmarkDotNet.Build/BuildContext.cs +++ b/build/BenchmarkDotNet.Build/BuildContext.cs @@ -59,6 +59,9 @@ public BuildContext(ICakeContext context) BuildDirectory = RootDirectory.Combine("build"); ArtifactsDirectory = RootDirectory.Combine("artifacts"); + var toolFileName = context.IsRunningOnWindows() ? "dotnet.exe" : "dotnet"; + var toolFilePath = RootDirectory.Combine(".dotnet").CombineWithFilePath(toolFileName); + context.Tools.RegisterFile(toolFilePath); SolutionFile = RootDirectory.CombineWithFilePath("BenchmarkDotNet.sln"); diff --git a/build/BenchmarkDotNet.Build/Folder.DotSettings b/build/BenchmarkDotNet.Build/Folder.DotSettings index 53109cf04e..539b6fe39e 100644 --- a/build/BenchmarkDotNet.Build/Folder.DotSettings +++ b/build/BenchmarkDotNet.Build/Folder.DotSettings @@ -1,4 +1,7 @@  <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> + <Policy><Descriptor Staticness="Instance" AccessRightKinds="Private" Description="Instance fields (private)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> + <Policy><Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static fields (private)"><ElementKinds><Kind Name="FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> + True True \ No newline at end of file diff --git a/build/BenchmarkDotNet.Build/Program.cs b/build/BenchmarkDotNet.Build/Program.cs index ce291ecd0f..38a1d4c28e 100644 --- a/build/BenchmarkDotNet.Build/Program.cs +++ b/build/BenchmarkDotNet.Build/Program.cs @@ -56,6 +56,26 @@ public HelpInfo GetHelp() } } +[TaskName(Name)] +[TaskDescription("Install wasm-tools workload")] +public class InstallWasmToolsWorkload : FrostingTask, IHelpProvider +{ + private const string Name = "install-wasm-tools"; + + public override void Run(BuildContext context) => context.BuildRunner.InstallWorkload("wasm-tools"); + + public HelpInfo GetHelp() + { + return new HelpInfo + { + Examples = new[] + { + new Example(Name) + } + }; + } +} + [TaskName(Name)] [TaskDescription("Run unit tests (fast)")] [IsDependentOn(typeof(BuildTask))] @@ -93,12 +113,12 @@ public class InTestsFullTask : FrostingTask, IHelpProvider } [TaskName(Name)] -[TaskDescription("Run integration tests using .NET 7 (slow)")] +[TaskDescription("Run integration tests using .NET 8 (slow)")] [IsDependentOn(typeof(BuildTask))] public class InTestsCoreTask : FrostingTask, IHelpProvider { private const string Name = "in-tests-core"; - public override void Run(BuildContext context) => context.UnitTestRunner.RunInTests("net7.0"); + public override void Run(BuildContext context) => context.UnitTestRunner.RunInTests("net8.0"); public HelpInfo GetHelp() => new(); } @@ -223,6 +243,9 @@ public class ReleaseTask : FrostingTask, IHelpProvider new Example(Name) .WithArgument(KnownOptions.Stable) .WithArgument(KnownOptions.NextVersion, "0.1.1729") + .WithArgument(KnownOptions.Push), + new Example(Name) + .WithArgument(KnownOptions.Stable) .WithArgument(KnownOptions.Push) } }; diff --git a/build/BenchmarkDotNet.Build/Runners/BuildRunner.cs b/build/BenchmarkDotNet.Build/Runners/BuildRunner.cs index 01c490fce3..a38ce7e79d 100644 --- a/build/BenchmarkDotNet.Build/Runners/BuildRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/BuildRunner.cs @@ -5,6 +5,7 @@ using Cake.Common.Tools.DotNet.Build; using Cake.Common.Tools.DotNet.Pack; using Cake.Common.Tools.DotNet.Restore; +using Cake.Common.Tools.DotNet.Workload.Install; using Cake.Core; using Cake.Core.IO; @@ -28,6 +29,16 @@ public void Restore() }); } + public void InstallWorkload(string workloadId) + { + context.DotNetWorkloadInstall(workloadId, + new DotNetWorkloadInstallSettings + { + IncludePreviews = true, + NoCache = true + }); + } + public void Build() { context.Information("BuildSystemProvider: " + context.BuildSystem().Provider); diff --git a/build/BenchmarkDotNet.Build/Runners/DocumentationRunner.cs b/build/BenchmarkDotNet.Build/Runners/DocumentationRunner.cs index 6883abe79a..2d2128a86f 100644 --- a/build/BenchmarkDotNet.Build/Runners/DocumentationRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/DocumentationRunner.cs @@ -133,8 +133,8 @@ private void RunDocfx() var currentDirectory = Directory.GetCurrentDirectory(); Directory.SetCurrentDirectory(docfxJsonFile.GetDirectory().FullPath); - Microsoft.DocAsCode.Dotnet.DotnetApiCatalog.GenerateManagedReferenceYamlFiles(docfxJsonFile.FullPath).Wait(); - Microsoft.DocAsCode.Docset.Build(docfxJsonFile.FullPath).Wait(); + Docfx.Dotnet.DotnetApiCatalog.GenerateManagedReferenceYamlFiles(docfxJsonFile.FullPath).Wait(); + Docfx.Docset.Build(docfxJsonFile.FullPath).Wait(); Directory.SetCurrentDirectory(currentDirectory); } diff --git a/build/BenchmarkDotNet.Build/Runners/ReleaseRunner.cs b/build/BenchmarkDotNet.Build/Runners/ReleaseRunner.cs index 0562639a9b..e9d1fcb9de 100644 --- a/build/BenchmarkDotNet.Build/Runners/ReleaseRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/ReleaseRunner.cs @@ -1,7 +1,6 @@ using System; using System.Linq; using System.Text; -using System.Text.RegularExpressions; using System.Threading.Tasks; using BenchmarkDotNet.Build.Helpers; using BenchmarkDotNet.Build.Meta; @@ -34,9 +33,15 @@ public void Run() else EnvVar.NuGetToken.SetEmpty(); - var nextVersion = KnownOptions.NextVersion.AssertHasValue(context); var currentVersion = context.VersionHistory.CurrentVersion; var tag = "v" + currentVersion; + var nextVersion = KnownOptions.NextVersion.Resolve(context); + if (nextVersion == "") + { + var version = Version.Parse(currentVersion); + nextVersion = $"{version.Major}.{version.Minor}.{version.Build + 1}"; + context.Information($"Evaluated NextVersion: {nextVersion}"); + } context.GitRunner.Tag(tag); @@ -138,6 +143,7 @@ private void PublishGitHubRelease() Draft = false, Prerelease = false, GenerateReleaseNotes = false, + DiscussionCategoryName = "Announcements", Body = notes }).Wait(); context.Information(" Success"); diff --git a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs index 48d5183e14..9b10d54ea4 100644 --- a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs @@ -7,28 +7,25 @@ namespace BenchmarkDotNet.Build.Runners; -public class UnitTestRunner +public class UnitTestRunner(BuildContext context) { - private readonly BuildContext context; + private FilePath UnitTestsProjectFile { get; } = context.RootDirectory + .Combine("tests") + .Combine("BenchmarkDotNet.Tests") + .CombineWithFilePath("BenchmarkDotNet.Tests.csproj"); - private FilePath UnitTestsProjectFile { get; } - private FilePath IntegrationTestsProjectFile { get; } - private DirectoryPath TestOutputDirectory { get; } + private FilePath ExporterTestsProjectFile { get; } = context.RootDirectory + .Combine("tests") + .Combine("BenchmarkDotNet.Exporters.Plotting.Tests") + .CombineWithFilePath("BenchmarkDotNet.Exporters.Plotting.Tests.csproj"); - public UnitTestRunner(BuildContext context) - { - this.context = context; - UnitTestsProjectFile = context.RootDirectory - .Combine("tests") - .Combine("BenchmarkDotNet.Tests") - .CombineWithFilePath("BenchmarkDotNet.Tests.csproj"); - IntegrationTestsProjectFile = context.RootDirectory - .Combine("tests") - .Combine("BenchmarkDotNet.IntegrationTests") - .CombineWithFilePath("BenchmarkDotNet.IntegrationTests.csproj"); - TestOutputDirectory = context.RootDirectory - .Combine("TestResults"); - } + private FilePath IntegrationTestsProjectFile { get; } = context.RootDirectory + .Combine("tests") + .Combine("BenchmarkDotNet.IntegrationTests") + .CombineWithFilePath("BenchmarkDotNet.IntegrationTests.csproj"); + + private DirectoryPath TestOutputDirectory { get; } = context.RootDirectory + .Combine("TestResults"); private DotNetTestSettings GetTestSettingsParameters(FilePath logFile, string tfm) { @@ -58,14 +55,15 @@ private void RunTests(FilePath projectFile, string alias, string tfm) context.DotNetTest(projectFile.FullPath, settings); } - private void RunUnitTests(string tfm) => RunTests(UnitTestsProjectFile, "unit", tfm); + private void RunUnitTests(string tfm) + { + RunTests(UnitTestsProjectFile, "unit", tfm); + RunTests(ExporterTestsProjectFile, "exporters", tfm); + } public void RunUnitTests() { - var targetFrameworks = context.IsRunningOnWindows() - ? new[] { "net462", "net7.0" } - : new[] { "net7.0" }; - + string[] targetFrameworks = context.IsRunningOnWindows() ? ["net462", "net8.0"] : ["net8.0"]; foreach (var targetFramework in targetFrameworks) RunUnitTests(targetFramework); } diff --git a/build/build.ps1 b/build/build.ps1 index 2f6d5a5d11..d4db557409 100755 --- a/build/build.ps1 +++ b/build/build.ps1 @@ -58,11 +58,10 @@ if (!(Test-Path $InstallPath)) { $ScriptPath = Join-Path $InstallPath 'dotnet-install.ps1' (New-Object System.Net.WebClient).DownloadFile($DotNetInstallerUri, $ScriptPath); & $ScriptPath -JSonFile $GlobalJsonPath -InstallDir $InstallPath; - - Remove-PathVariable "$InstallPath" - $env:PATH = "$InstallPath;$env:PATH" } +Remove-PathVariable "$InstallPath" +$env:PATH = "$InstallPath;$env:PATH" $env:DOTNET_ROOT=$InstallPath ########################################################################### diff --git a/build/build.sh b/build/build.sh index ebf8ef04bd..e07aecf5ff 100755 --- a/build/build.sh +++ b/build/build.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Define variables -SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd ) +PROJECT_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd ) ########################################################################### # INSTALL .NET CORE CLI @@ -12,17 +12,17 @@ export DOTNET_CLI_TELEMETRY_OPTOUT=1 export DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER=0 export DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX=2 -if [ ! -d "$SCRIPT_DIR/.dotnet" ]; then - mkdir "$SCRIPT_DIR/.dotnet" - curl -Lsfo "$SCRIPT_DIR/.dotnet/dotnet-install.sh" https://dot.net/v1/dotnet-install.sh - bash "$SCRIPT_DIR/.dotnet/dotnet-install.sh" --jsonfile ./build/sdk/global.json --install-dir .dotnet --no-path +if [ ! -d "$PROJECT_ROOT/.dotnet" ]; then + mkdir "$PROJECT_ROOT/.dotnet" + curl -Lsfo "$PROJECT_ROOT/.dotnet/dotnet-install.sh" https://dot.net/v1/dotnet-install.sh + bash "$PROJECT_ROOT/.dotnet/dotnet-install.sh" --jsonfile ./build/sdk/global.json --install-dir .dotnet --no-path fi -export PATH="$SCRIPT_DIR/.dotnet":$PATH -export DOTNET_ROOT="$SCRIPT_DIR/.dotnet" +export PATH="$PROJECT_ROOT/.dotnet":$PATH +export DOTNET_ROOT="$PROJECT_ROOT/.dotnet" ########################################################################### # RUN BUILD SCRIPT ########################################################################### -dotnet run --configuration Release --project ./build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj -- "$@" +dotnet run --configuration Release --project "$PROJECT_ROOT/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj" -- "$@" diff --git a/build/cSpell.json b/build/cSpell.json index a224a62a9c..42795c7a70 100644 --- a/build/cSpell.json +++ b/build/cSpell.json @@ -12,6 +12,7 @@ "Cygwin", "Diagnoser", "diagnosers", + "diagsession", "disassemblers", "disassm", "Jits", @@ -29,6 +30,8 @@ "Pseudocode", "runtimes", "Serilog", + "vsprofiler", + "vstest", "Tailcall", "toolchains", "unmanaged" diff --git a/build/common.props b/build/common.props index f84648040b..7fab76cf87 100644 --- a/build/common.props +++ b/build/common.props @@ -18,10 +18,13 @@ false true True + false $(MSBuildThisFileDirectory)CodingStyle.ruleset true annotations + + true @@ -35,11 +38,11 @@ - 11.0 + 12.0 - 0.13.8 + 0.15.0 diff --git a/build/sdk/global.json b/build/sdk/global.json index 5e4624ef91..c55fa7dfa6 100644 --- a/build/sdk/global.json +++ b/build/sdk/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "7.0.305", + "version": "8.0.401", "rollForward": "disable" } } diff --git a/build/versions.txt b/build/versions.txt index 1f6634ffbc..a7615e3fb3 100644 --- a/build/versions.txt +++ b/build/versions.txt @@ -52,4 +52,10 @@ 0.13.5 0.13.6 0.13.7 -0.13.8 \ No newline at end of file +0.13.8 +0.13.9 +0.13.10 +0.13.11 +0.13.12 +0.14.0 +0.15.0 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.10.md b/docs/_changelog/footer/v0.13.10.md new file mode 100644 index 0000000000..68ef4fd0db --- /dev/null +++ b/docs/_changelog/footer/v0.13.10.md @@ -0,0 +1,11 @@ +_Date: November 01, 2023_ + +_Milestone: [v0.13.10](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.10)_ +([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.9...v0.13.10)) + +_NuGet Packages:_ +* https://www.nuget.org/packages/BenchmarkDotNet/0.13.10 +* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.10 +* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.10 +* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.10 +* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.10 diff --git a/docs/_changelog/footer/v0.13.11.md b/docs/_changelog/footer/v0.13.11.md new file mode 100644 index 0000000000..03dff00d30 --- /dev/null +++ b/docs/_changelog/footer/v0.13.11.md @@ -0,0 +1,11 @@ +_Date: December 06, 2023_ + +_Milestone: [v0.13.11](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.11)_ +([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.10...v0.13.11)) + +_NuGet Packages:_ +* https://www.nuget.org/packages/BenchmarkDotNet/0.13.11 +* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.11 +* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.11 +* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.11 +* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.11 diff --git a/docs/_changelog/footer/v0.13.12.md b/docs/_changelog/footer/v0.13.12.md new file mode 100644 index 0000000000..246864bb8a --- /dev/null +++ b/docs/_changelog/footer/v0.13.12.md @@ -0,0 +1,12 @@ +_Date: January 05, 2024_ + +_Milestone: [v0.13.12](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.12)_ +([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.11...v0.13.12)) + +_NuGet Packages:_ +* https://www.nuget.org/packages/BenchmarkDotNet/0.13.12 +* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.12 +* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.12 +* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.12 +* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.12 +* https://www.nuget.org/packages/BenchmarkDotNet.TestAdapter/0.13.12 diff --git a/docs/_changelog/footer/v0.13.9.md b/docs/_changelog/footer/v0.13.9.md new file mode 100644 index 0000000000..962bc252cc --- /dev/null +++ b/docs/_changelog/footer/v0.13.9.md @@ -0,0 +1,11 @@ +_Date: October 05, 2023_ + +_Milestone: [v0.13.9](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.9)_ +([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.8...v0.13.9)) + +_NuGet Packages:_ +* https://www.nuget.org/packages/BenchmarkDotNet/0.13.9 +* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.9 +* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.9 +* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.9 +* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.9 diff --git a/docs/_changelog/footer/v0.14.0.md b/docs/_changelog/footer/v0.14.0.md new file mode 100644 index 0000000000..47d421f49d --- /dev/null +++ b/docs/_changelog/footer/v0.14.0.md @@ -0,0 +1,14 @@ +_Date: August 06, 2024_ + +_Milestone: [v0.14.0](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.14.0)_ +([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.12...v0.14.0)) + +_NuGet Packages:_ +* https://www.nuget.org/packages/BenchmarkDotNet/0.14.0 +* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.14.0 +* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotMemory/0.14.0 +* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.14.0 +* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.14.0 +* https://www.nuget.org/packages/BenchmarkDotNet.Exporters.Plotting/0.14.0 +* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.14.0 +* https://www.nuget.org/packages/BenchmarkDotNet.TestAdapter/0.14.0 diff --git a/docs/_changelog/footer/v0.15.0.md b/docs/_changelog/footer/v0.15.0.md new file mode 100644 index 0000000000..c3ddd0fb17 --- /dev/null +++ b/docs/_changelog/footer/v0.15.0.md @@ -0,0 +1,14 @@ +_Date: TBA_ + +_Milestone: [v0.15.0](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.15.0)_ +([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.14.0...v0.15.0)) + +_NuGet Packages:_ +* https://www.nuget.org/packages/BenchmarkDotNet/0.15.0 +* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.15.0 +* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotMemory/0.15.0 +* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.15.0 +* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.15.0 +* https://www.nuget.org/packages/BenchmarkDotNet.Exporters.Plotting/0.15.0 +* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.15.0 +* https://www.nuget.org/packages/BenchmarkDotNet.TestAdapter/0.15.0 diff --git a/docs/_changelog/header/v0.12.1.md b/docs/_changelog/header/v0.12.1.md index 8378e7da70..5c02ea5c0a 100644 --- a/docs/_changelog/header/v0.12.1.md +++ b/docs/_changelog/header/v0.12.1.md @@ -241,7 +241,7 @@ public class IntroEventPipeProfiler Once the benchmark run is finished, you get a `.speedscope.json` file that can be opened in [SpeedScope](https://www.speedscope.app/): -![](https://wojciechnagorski.com/images/EventPipeProfiler/SpeedScope.png#mid) +![](https://wojciechnagorski.github.io/images/EventPipeProfiler/SpeedScope.png#mid) The new profiler supports several modes: @@ -250,7 +250,7 @@ The new profiler supports several modes: * `GcCollect` - Tracks GC collections only at very low overhead. * `Jit` - Logging when Just in time (JIT) compilation occurs. Logging of the internal workings of the Just In Time compiler. This is fairly verbose. It details decisions about interesting optimization (like inlining and tail call) -Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.com/2020/04/cross-platform-profiling-.net-code-with-benchmarkdotnet/) for all the details. +Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.github.io/2020/04/cross-platform-profiling-.net-code-with-benchmarkdotnet/) for all the details. Special thanks to [@WojciechNagorski](https://github.com/WojciechNagorski) for the implementation! diff --git a/docs/_changelog/header/v0.13.0.md b/docs/_changelog/header/v0.13.0.md index d0fca1471d..daf8ed8ea6 100644 --- a/docs/_changelog/header/v0.13.0.md +++ b/docs/_changelog/header/v0.13.0.md @@ -340,7 +340,7 @@ Big thanks to [@lukasz-pyrzyk](https://github.com/lukasz-pyrzyk), [@fleckert](ht * `LangVersion` set to a non-numeric value like `latest` was crashing the build. Fixed by [@martincostello](https://github.com/martincostello) in [#1420](https://github.com/dotnet/BenchmarkDotNet/pull/1420). * Windows 10 November 201**9** was being recognized as 201**8**. Fixed by [@kapsiR](https://github.com/kapsiR) in [#1437](https://github.com/dotnet/BenchmarkDotNet/pull/1437). * Assemblies loaded via streams were not supported. Fixed by [@jeremyosterhoudt](https://github.com/jeremyosterhoudt) in [#1443](https://github.com/dotnet/BenchmarkDotNet/pull/1443). -* [NativeMemoryProfiler](https://wojciechnagorski.com/2019/08/analyzing-native-memory-allocation-with-benchmarkdotnet/) was detecting small leaks that were false positives. Fixed by [@WojciechNagorski](https://github.com/WojciechNagorski) in [#1451](https://github.com/dotnet/BenchmarkDotNet/pull/1451) and [#1600](https://github.com/dotnet/BenchmarkDotNet/pull/1600). +* [NativeMemoryProfiler](https://wojciechnagorski.github.io/2019/08/analyzing-native-memory-allocation-with-benchmarkdotnet/) was detecting small leaks that were false positives. Fixed by [@WojciechNagorski](https://github.com/WojciechNagorski) in [#1451](https://github.com/dotnet/BenchmarkDotNet/pull/1451) and [#1600](https://github.com/dotnet/BenchmarkDotNet/pull/1600). * [DisassemblyDiagnoser](https://adamsitnik.com/Disassembly-Diagnoser/) was crashing on Linux. Fixed by [@damageboy](https://github.com/damageboy) in [#1459](https://github.com/dotnet/BenchmarkDotNet/pull/1459). * Target framework moniker was being printed as toolchain name for Full .NET Framework benchmarks. Fixed by [@svick](https://github.com/svick) in [#1471](https://github.com/dotnet/BenchmarkDotNet/pull/1471). * `[ParamsSource]` returning `IEnumerable` was not working properly when combined with `[Arguments]`. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1478](https://github.com/dotnet/BenchmarkDotNet/pull/1478). diff --git a/docs/_changelog/header/v0.13.10.md b/docs/_changelog/header/v0.13.10.md new file mode 100644 index 0000000000..9f55725d19 --- /dev/null +++ b/docs/_changelog/header/v0.13.10.md @@ -0,0 +1,3 @@ +## Highlights + +Initial support of .NET 9 and minor bug fixes. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.11.md b/docs/_changelog/header/v0.13.11.md new file mode 100644 index 0000000000..ef2d1b9579 --- /dev/null +++ b/docs/_changelog/header/v0.13.11.md @@ -0,0 +1,3 @@ +## Highlights + +Small improvements. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.12.md b/docs/_changelog/header/v0.13.12.md new file mode 100644 index 0000000000..c1d788166c --- /dev/null +++ b/docs/_changelog/header/v0.13.12.md @@ -0,0 +1,10 @@ +## Highlights + +The biggest highlight of this release if our new VSTest Adapter, + which allows to run benchmarks as unit tests in your favorite IDE! +The detailed guide can be found [here](xref:docs.vstest). + +This release also includes to a minor bug fix that caused incorrect job id generation: + fixed job id generation ([#2491](https://github.com/dotnet/BenchmarkDotNet/pull/2491)). + +Also, the target framework in the BenchmarkDotNet templates was bumped to .NET 8.0. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.9.md b/docs/_changelog/header/v0.13.9.md new file mode 100644 index 0000000000..07e4813c9f --- /dev/null +++ b/docs/_changelog/header/v0.13.9.md @@ -0,0 +1,3 @@ +## Highlights + +This release contains bug fixes. \ No newline at end of file diff --git a/docs/_changelog/header/v0.14.0.md b/docs/_changelog/header/v0.14.0.md new file mode 100644 index 0000000000..1c2f6693ce --- /dev/null +++ b/docs/_changelog/header/v0.14.0.md @@ -0,0 +1,15 @@ +## Highlights + +* Introduce `BenchmarkDotNet.Diagnostics.dotMemory` [#2549](https://github.com/dotnet/BenchmarkDotNet/pull/2549): memory allocation profile of your benchmarks using [dotMemory](https://www.jetbrains.com/dotmemory/), see @BenchmarkDotNet.Samples.IntroDotMemoryDiagnoser +* Introduce `BenchmarkDotNet.Exporters.Plotting` [#2560](https://github.com/dotnet/BenchmarkDotNet/pull/2560): plotting via [ScottPlot](https://scottplot.net/) (initial version) +* Multiple bugfixes +* The default build toolchains have been updated to pass `IntermediateOutputPath`, `OutputPath`, and `OutDir` properties to the `dotnet build` command. This change forces all build outputs to be placed in a new directory generated by BenchmarkDotNet, and fixes many issues that have been reported with builds. You can also access these paths in your own `.csproj` and `.props` from those properties if you need to copy custom files to the output. + +## Bug fixes + +* Fixed multiple build-related bugs including passing MsBuildArguments and .Net 8's `UseArtifactsOutput`. + +## Breaking Changes + +* `DotNetCliBuilder` removed `retryFailedBuildWithNoDeps` constructor option. +* `DotNetCliCommand` removed `RetryFailedBuildWithNoDeps` property and `BuildNoRestoreNoDependencies()` and `PublishNoBuildAndNoRestore()` methods (replaced with `PublishNoRestore()`). \ No newline at end of file diff --git a/docs/_changelog/header/v0.15.0.md b/docs/_changelog/header/v0.15.0.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/articles/configs/diagnosers.md b/docs/articles/configs/diagnosers.md index cfb5532a40..e81943310a 100644 --- a/docs/articles/configs/diagnosers.md +++ b/docs/articles/configs/diagnosers.md @@ -35,10 +35,10 @@ The current Diagnosers are: Please see Adam Sitnik's [blog post](https://adamsitnik.com/ConcurrencyVisualizer-Profiler/) for all the details. - Native Memory Profiler (`NativeMemoryProfiler`) It uses `EtwProfiler` to profile the code using ETW and adds the extra columns `Allocated native memory` and `Native memory leak`. - Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.com/2019/08/analyzing-native-memory-allocation-with-benchmarkdotnet/) for all the details. + Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.github.io/2019/08/analyzing-native-memory-allocation-with-benchmarkdotnet/) for all the details. - Event Pipe Profiler (`EventPipeProfiler`). It is a cross-platform profiler that allows profile .NET code on every platform - Windows, Linux, macOS. - Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.com/2020/04/cross-platform-profiling-.net-code-with-benchmarkdotnet/) for all the details. + Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.github.io/2020/04/cross-platform-profiling-.net-code-with-benchmarkdotnet/) for all the details. - Threading Diagnoser (`ThreadingDiagnoser`) - .NET Core 3.0+ diagnoser that reports some Threading statistics. - Exception Diagnoser (`ExceptionDiagnoser`) - a diagnoser that reports the frequency of exceptions thrown during the operation. @@ -59,11 +59,11 @@ private class Config : ManualConfig { public Config() { - Add(MemoryDiagnoser.Default); - Add(new InliningDiagnoser()); - Add(new EtwProfiler()); - Add(ThreadingDiagnoser.Default); - Add(ExceptionDiagnoser.Default); + AddDiagnoser(MemoryDiagnoser.Default); + AddDiagnoser(new InliningDiagnoser()); + AddDiagnoser(new EtwProfiler()); + AddDiagnoser(ThreadingDiagnoser.Default); + AddDiagnoser(ExceptionDiagnoser.Default); } } ``` @@ -86,7 +86,6 @@ In BenchmarkDotNet, 1kB = 1024B, 1MB = 1024kB, and so on. The column Gen X means * In order to not affect main results we perform a separate run if any diagnoser is used. That's why it might take more time to execute benchmarks. * MemoryDiagnoser: - * Mono currently [does not](https://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes-in-mono) expose any api to get the number of allocated bytes. That's why our Mono users will get `?` in Allocated column. * In order to get the number of allocated bytes in cross platform way we are using `GC.GetAllocatedBytesForCurrentThread` which recently got [exposed](https://github.com/dotnet/corefx/pull/12489) for netcoreapp1.1. That's why BenchmarkDotNet does not support netcoreapp1.0 from version 0.10.1. * MemoryDiagnoser is `99.5%` accurate about allocated memory when using default settings or Job.ShortRun (or any longer job than it). * Threading Diagnoser: diff --git a/docs/articles/configs/jobs.md b/docs/articles/configs/jobs.md index a6085e0bdb..ad0cf4559c 100644 --- a/docs/articles/configs/jobs.md +++ b/docs/articles/configs/jobs.md @@ -98,20 +98,20 @@ public class MyBenchmarks { public Config() { - Add( - new Job("MySuperJob", RunMode.Dry, EnvMode.RyuJitX64) + AddJob( + new Job("MySuperJob", RunMode.Dry, EnvironmentMode.RyuJitX64) { - Env = { Runtime = Runtime.Core }, + Environment = { Runtime = CoreRuntime.Core90 }, Run = { LaunchCount = 5, IterationTime = TimeInterval.Millisecond * 200 }, - Accuracy = { MaxStdErrRelative = 0.01 } + Accuracy = { MaxRelativeError = 0.01 } }); // The same, using the .With() factory methods: - Add( + AddJob( Job.Dry .WithPlatform(Platform.X64) .WithJit(Jit.RyuJit) - .WithRuntime(Runtime.Core) + .WithRuntime(CoreRuntime.Core90) .WithLaunchCount(5) .WithIterationTime(TimeInterval.Millisecond * 200) .WithMaxRelativeError(0.01) @@ -122,26 +122,26 @@ public class MyBenchmarks } ``` -Basically, it's a good idea to start with predefined values (e.g. `EnvMode.RyuJitX64` and `RunMode.Dry` passed as constructor args) and modify rest of the properties using property setters or with help of object initializer syntax. +Basically, it's a good idea to start with predefined values (e.g. `EnvironmentMode.RyuJitX64` and `RunMode.Dry` passed as constructor args) and modify rest of the properties using property setters or with help of object initializer syntax. Note that the job cannot be modified after it's added into config. Trying to set a value on property of the frozen job will throw an `InvalidOperationException`. Use the `Job.Frozen` property to determine if the code properties can be altered. If you do want to create a new job based on frozen one (all predefined job values are frozen) you can use the `.With()` extension method ```cs - var newJob = Job.Dry.With(Platform.X64); + var newJob = Job.Dry.WithPlatform(Platform.X64); ``` or pass the frozen value as a constructor argument ```c# - var newJob = new Job(Job.Dry) { Env = { Platform = Platform.X64 } }; + var newJob = new Job(Job.Dry) { Environment = { Platform = Platform.X64 } }; ``` or use the `.Apply()` method on unfrozen job ```c# - var newJob = new Job() { Env = { Platform = Platform.X64 } }.Apply(Job.Dry); + var newJob = new Job() { Environment = { Platform = Platform.X64 } }.Apply(Job.Dry); ``` in any case the Id property will not be transfered and you must pass it explicitly (using the .ctor id argument or the `.WithId()` extension method). @@ -152,7 +152,9 @@ You can also add new jobs via attributes. Examples: ```cs [DryJob] -[ClrJob, CoreJob, MonoJob] +[MonoJob] +[SimpleJob(RuntimeMoniker.Net90)] +[SimpleJob(RuntimeMoniker.NetCoreApp31)] [LegacyJitX86Job, LegacyJitX64Job, RyuJitX64Job] [SimpleJob(RunStrategy.ColdStart, launchCount: 1, warmupCount: 5, iterationCount: 5, id: "FastAndDirtyJob")] public class MyBenchmarkClass @@ -212,7 +214,7 @@ public class MySuperJobAttribute : Attribute, IConfigSource { var job = new Job("MySuperJob", RunMode.Core); job.Env.Platform = Platform.X64; - Config = ManualConfig.CreateEmpty().With(job); + Config = ManualConfig.CreateEmpty().AddJob(job); } public IConfig Config { get; } diff --git a/docs/articles/configs/toolchains.md b/docs/articles/configs/toolchains.md index 8f73bc3784..b47e994a94 100644 --- a/docs/articles/configs/toolchains.md +++ b/docs/articles/configs/toolchains.md @@ -18,14 +18,14 @@ When you run your benchmarks without specifying the toolchain in an explicit way If you want to test multiple frameworks, your project file **MUST target all of them** and you **MUST install the corresponding SDKs**: ```xml -netcoreapp3.0;netcoreapp2.1;net48 +netcoreapp3.1;net8.0;net48 ``` If you run your benchmarks without specifying any custom settings, BenchmarkDotNet is going to run the benchmarks **using the same framework as the host process**: ```cmd -dotnet run -c Release -f netcoreapp2.1 # is going to run the benchmarks using .NET Core 2.1 -dotnet run -c Release -f netcoreapp3.0 # is going to run the benchmarks using .NET Core 3.0 +dotnet run -c Release -f netcoreapp3.1 # is going to run the benchmarks using .NET Core 3.1 +dotnet run -c Release -f net8.0 # is going to run the benchmarks using .NET 8.0 dotnet run -c Release -f net48 # is going to run the benchmarks using .NET 4.8 mono $pathToExe # is going to run the benchmarks using Mono from your PATH ``` @@ -33,8 +33,8 @@ mono $pathToExe # is going to run the benchmarks using Mo To run the benchmarks for multiple runtimes with a single command, you need to specify the target framework moniker names via `--runtimes|-r` console argument: ```cmd -dotnet run -c Release -f netcoreapp2.1 --runtimes netcoreapp2.1 netcoreapp3.0 # is going to run the benchmarks using .NET Core 2.1 and .NET Core 3.0 -dotnet run -c Release -f netcoreapp2.1 --runtimes netcoreapp2.1 net48 # is going to run the benchmarks using .NET Core 2.1 and .NET 4.8 +dotnet run -c Release -f net8.0 --runtimes net8.0 netcoreapp3.1 # is going to run the benchmarks using .NET 8.0 and .NET Core 3.1 +dotnet run -c Release -f net8.0 --runtimes net8.0 net48 # is going to run the benchmarks using .NET 8.0 and .NET 4.8 ``` What is going to happen if you provide multiple Full .NET Framework monikers? Let's say: @@ -67,8 +67,8 @@ namespace BenchmarkDotNet.Samples { [SimpleJob(RuntimeMoniker.Net48)] [SimpleJob(RuntimeMoniker.Mono)] - [SimpleJob(RuntimeMoniker.NetCoreApp21)] - [SimpleJob(RuntimeMoniker.NetCoreApp30)] + [SimpleJob(RuntimeMoniker.NetCoreApp31)] + [SimpleJob(RuntimeMoniker.Net80)] public class TheClassWithBenchmarks ``` @@ -87,10 +87,9 @@ namespace BenchmarkDotNet.Samples static void Main(string[] args) { var config = DefaultConfig.Instance - .With(Job.Default.With(CoreRuntime.Core21)) - .With(Job.Default.With(CoreRuntime.Core30)) - .With(Job.Default.With(ClrRuntime.Net48)) - .With(Job.Default.With(MonoRuntime.Default)); + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80)) + .AddJob(Job.Default.WithRuntime(ClrRuntime.Net48)) + .AddJob(Job.Default.WithRuntime(MonoRuntime.Default)); BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) @@ -115,9 +114,9 @@ public class MyConfig : ManualConfig Add(Job.Default.With( CsProjCoreToolchain.From( new NetCoreAppSettings( - targetFrameworkMoniker: "netcoreapp2.1", - runtimeFrameworkVersion: "2.1.0-preview2-25628-01", - name: ".NET Core 2.1")))); + targetFrameworkMoniker: "net8.0-windows", + runtimeFrameworkVersion: "8.0.101", + name: ".NET 8.0 Windows")))); } } ``` @@ -130,8 +129,8 @@ It's possible to benchmark a private build of .NET Runtime. All you need to do i BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) .Run(args, - DefaultConfig.Instance.With( - Job.ShortRun.With(ClrRuntime.CreateForLocalFullNetFrameworkBuild(version: "4.0")))); + DefaultConfig.Instance.AddJob( + Job.ShortRun.WithRuntime(ClrRuntime.CreateForLocalFullNetFrameworkBuild(version: "4.0")))); ``` This sends the provided version as a `COMPLUS_Version` env var to the benchmarked process. @@ -146,15 +145,15 @@ public class CustomPathsConfig : ManualConfig public CustomPathsConfig() { var dotnetCli32bit = NetCoreAppSettings - .NetCoreApp20 + .NetCoreApp31 .WithCustomDotNetCliPath(@"C:\Program Files (x86)\dotnet\dotnet.exe", "32 bit cli"); var dotnetCli64bit = NetCoreAppSettings - .NetCoreApp20 + .NetCoreApp31 .WithCustomDotNetCliPath(@"C:\Program Files\dotnet\dotnet.exe", "64 bit cli"); - Add(Job.RyuJitX86.With(CsProjCoreToolchain.From(dotnetCli32bit)).WithId("32 bit cli")); - Add(Job.RyuJitX64.With(CsProjCoreToolchain.From(dotnetCli64bit)).WithId("64 bit cli")); + AddJob(Job.RyuJitX86.WithToolchain(CsProjCoreToolchain.From(dotnetCli32bit)).WithId("32 bit cli")); + AddJob(Job.RyuJitX64.WithToolchain(CsProjCoreToolchain.From(dotnetCli64bit)).WithId("64 bit cli")); } } ``` @@ -208,7 +207,7 @@ or: ```cs var config = DefaultConfig.Instance - .With(Job.Default.With(NativeAotRuntime.Net70)); // compiles the benchmarks as net7.0 and uses the latest NativeAOT to build a native app + .AddJob(Job.Default.WithRuntime(NativeAotRuntime.Net70)); // compiles the benchmarks as net7.0 and uses the latest NativeAOT to build a native app BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) @@ -231,8 +230,8 @@ If you want to benchmark some particular version of NativeAOT (or from a differe ```cs var config = DefaultConfig.Instance - .With(Job.ShortRun - .With(NativeAotToolchain.CreateBuilder() + .AddJob(Job.ShortRun + .WithToolchain(NativeAotToolchain.CreateBuilder() .UseNuGet( microsoftDotNetILCompilerVersion: "7.0.0-*", // the version goes here nuGetFeedUrl: "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json") // this address might change over time @@ -338,8 +337,8 @@ or explicitly in the code: ```cs var config = DefaultConfig.Instance - .With(Job.ShortRun - .With(NativeAotToolchain.CreateBuilder() + .AddJob(Job.ShortRun + .WithToolchain(NativeAotToolchain.CreateBuilder() .UseLocalBuild(@"C:\Projects\runtime\artifacts\packages\Release\Shipping\") .DisplayName("NativeAOT local build") .TargetFrameworkMoniker("net7.0") diff --git a/docs/articles/features/event-pipe-profiler.md b/docs/articles/features/event-pipe-profiler.md index 3b0ed69bc6..25d1e90386 100644 --- a/docs/articles/features/event-pipe-profiler.md +++ b/docs/articles/features/event-pipe-profiler.md @@ -7,7 +7,7 @@ name: EventPipeProfiler `EventPipeProfiler` is a cross-platform profiler that allows profile .NET code on every platform - Windows, Linux, macOS. Collected data are exported to trace files (`.speedscope.json` and `.nettrace`) which can be analyzed using [SpeedScope](https://www.speedscope.app/), [PerfView](https://github.com/Microsoft/perfview), and [Visual Studio Profiler](https://learn.microsoft.com/visualstudio/profiling/profiling-feature-tour). This new profiler is available from the 0.12.1 version. -![](https://wojciechnagorski.com/images/EventPipeProfiler/SpeedScopeAdvance.png) +![](https://wojciechnagorski.github.io/images/EventPipeProfiler/SpeedScopeAdvance.png) # Configuration diff --git a/docs/articles/features/toc.yml b/docs/articles/features/toc.yml index fe55761610..61e3a2cd54 100644 --- a/docs/articles/features/toc.yml +++ b/docs/articles/features/toc.yml @@ -11,4 +11,8 @@ - name: EtwProfiler href: etwprofiler.md - name: EventPipeProfiler - href: event-pipe-profiler.md \ No newline at end of file + href: event-pipe-profiler.md +- name: VSProfiler + href: vsprofiler.md +- name: VSTest + href: vstest.md \ No newline at end of file diff --git a/docs/articles/features/vsprofiler.md b/docs/articles/features/vsprofiler.md new file mode 100644 index 0000000000..9e8f52712d --- /dev/null +++ b/docs/articles/features/vsprofiler.md @@ -0,0 +1,71 @@ +--- +uid: docs.vsprofiler +name: VS Profiler +--- + +# Running with Visual Studio profiler +Visual Studio supports [profiler integration with BenchmarkDotNet](https://learn.microsoft.com/visualstudio/profiling/profiling-with-benchmark-dotnet) on Windows through its [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package. Once installed, Visual Studio specific diagnosers will capture performance data in runs and automatically open traces if launched through Visual Studio + +![](../../images/vs-profiler-demo.png) + +## How it works + +First, install the [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package in your benchmarking project. Next add one or more of the Visual Studio diagnosers to your benchmark to capture the relevant profiling information while benchmarking. Lastly, run your benchmarks and a diagsession will be generated. If run from Visual Studio the diagsession will automatically be opened. + +## Available Diagnosers + +* `[CPUUsageDiagnoser]` - Enables the [CPU Usage tool](https://learn.microsoft.com/visualstudio/profiling/cpu-usage). +* `[DatabaseDiagnoser]` - Enables the [Database tool](https://learn.microsoft.com/visualstudio/profiling/analyze-database) +* `[DotNetCountersDiagnoser]` - Enables the [.NET Counters tool](https://learn.microsoft.com/visualstudio/profiling/dotnet-counters-tool) +* `[DotNetObjectAllocDiagnoser]` - Enables the [.NET Object Allocation tool](https://learn.microsoft.com/visualstudio/profiling/dotnet-alloc-tool). When using this tool, you must also specify `[DotNetObjectAllocJobConfiguration]` on the benchmark. If this is missing the run will fail and you will receive an error indicating you need to add it. +* `[EventsDiagnoser]` - Enables the [Events tool](https://learn.microsoft.com/visualstudio/profiling/events-viewer) +* `[FileIODiagnoser]` - Enables the [File IO tool](https://learn.microsoft.com/visualstudio/profiling/use-file-io) + +## How to use it? + +After installing the [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package add the following code as a benchmark: + +```cs +using System; +using System.Security.Cryptography; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Running; +using Microsoft.VSDiagnostics; + +namespace MyBenchmarks +{ + [CPUUsageDiagnoser] + public class Md5VsSha256 + { + private const int N = 10000; + private readonly byte[] data; + + private readonly SHA256 sha256 = SHA256.Create(); + private readonly MD5 md5 = MD5.Create(); + + public Md5VsSha256() + { + data = new byte[N]; + new Random(42).NextBytes(data); + } + + [Benchmark] + public byte[] Sha256() => sha256.ComputeHash(data); + + [Benchmark] + public byte[] Md5() => md5.ComputeHash(data); + } + + public class Program + { + public static void Main(string[] args) + { + var summary = BenchmarkRunner.Run(typeof(Program).Assembly); + } + } +} +``` + +In this case we have added the `[CpuUsageDiagnoser]` to capture a CPU sampling trace. From here run the benchmark in Visual Studio (Ctrl+F5), and after the benchmark run the resulting diagsession will be displayed. Double clicking on one of the benchmark rows shown under the Benchmarks tab will filter the time selection to the specific benchmark allowing you to better isolate and investigate. + +![](../../images/vs-profiler-demo.png) \ No newline at end of file diff --git a/docs/articles/features/vstest.md b/docs/articles/features/vstest.md new file mode 100644 index 0000000000..9163cb43e5 --- /dev/null +++ b/docs/articles/features/vstest.md @@ -0,0 +1,139 @@ +--- +uid: docs.vstest +name: Running with VSTest +--- + +# Running with VSTest + +BenchmarkDotNet supports discovering and executing benchmarks through VSTest. +This provides an alternative user experience to running benchmarks with the CLI + and may be preferable for those who like their IDE's VSTest integrations that they may have used when running unit tests. + +Below is an example of running some benchmarks from the BenchmarkDotNet samples project in Visual Studio's Test Explorer. + +![](../../images/vs-testexplorer-demo.png) + +## About VSTest + +VSTest is one of the most popular test platforms in use in the .NET ecosystem, + with test frameworks such as MSTest, xUnit, and NUnit providing support for it. +Many IDEs, including Visual Studio and Rider, provide UIs for running tests through VSTest + which some users may find more accessible than running them through the command line. + +It may seem counterintuitive to run performance tests on a platform + that is designed for unit tests that expect a boolean outcome of "Passed" or "Failed". +However, VSTest provides good value as a protocol for discovering and executing tests. +In addition, we can still make use of this boolean output to indicate + if the benchmark had validation errors that caused it to fail to run. + +## Caveats and things to know + +* **The benchmark measurements may be affected by the VSTest host and your IDE!** + If you want to have accurate measurements, + it is recommended to run benchmarks through the CLI without other processes on the machine impacting performance. + This does not mean that the measurements are useless though, + it will still be able to provide useful measurements during development when comparing different approaches. +* **The test adapter will not display or execute benchmarks if optimizations are disabled.** + Please ensure you are compiling in Release mode or with `Optimize` set to true. + Using an `InProcess` toolchain will let you run your benchmarks with optimizations disabled + and will let you attach the debugger as well. +* **The VSTest adapter will not call your application's entry point.** + If you use the entry point to customize how your benchmarks are run, + you will need to do this through other means such as an assembly-level `IConfigSource`, + as shown [here](#setting-a-default-configuration). +* **The test adapter will generate an entry point for you automatically.** + The generated entry point will pass the command line arguments + and the current assembly into `BenchmarkSwitcher`, + so you can still use it in your CLI as well as in VSTest. + This means you can delete your entry point and only need to define your benchmarks. + If you want to use a custom entry point, you can still do so by setting `GenerateProgramFile` to `false` in your project file. + +## Getting started + +* **Step 1.** Install the NuGet packages. + You need to install two packages into your benchmark project: + * `BenchmarkDotNet.TestAdapter`: Implements the VSTest protocol for BenchmarkDotNet + * `Microsoft.NET.Test.Sdk`: Includes all the pieces needed for the VSTest host to run and load the VSTest adapter. +* **Step 2.** Make sure that the entry point is configured correctly. + As mentioned in the caveats section, `BenchmarkDotNet.TestAdapter` will generate an entry point for you automatically. + So, if you have an entry point already, + you will either need to delete it or set `GenerateProgramFile` to `false` in your project file to continue using your existing one. + Here is an example of a `.csproj` file based on the default Console Application template: + +```xml + + + + Exe + net8.0 + enable + enable + + false + + + + + + + + +``` + +* **Step 3.** Make sure that your IDE supports VSTest integration. + In Visual Studio, everything works out of the box. + In Rider/R# 2023.3, the VSTest integration should be activated: + * Go to the "Unit Testing" settings page. + * Rider: Settings -> Build, Execution, Deployment -> Unit Testing -> VSTest + * R#: Extensions -> ReSharper -> Options -> Tools -> Unit Testing -> Test Frameworks -> VSTest + * Make sure that the "Enable VSTest adapter support" checkbox is checked. + In future versions of Rider, this should be enabled by default. +* **Step 4.** Switch to the `Release` configuration. + As mentioned above, the TestAdapter is not able to discover and run benchmarks with optimizations disabled (by design). +* **Step 5.** Build the project. + In order to discover the benchmarks, the VSTest adapter needs to be able to find the assembly. + Once you build the project, you should observe the discovered benchmarks in your IDE's Unit Test Explorer. + +If you correctly performed all the steps above, you should be able to run your benchmarks in your IDE using embedded unit testing features. +If this doesn't work for you, don't hesitate to file [a new GitHub issue](https://github.com/dotnet/BenchmarkDotNet/issues/new). + +## Setting a default configuration + +Previously, it was common for the default configuration to be defined inside the entry point. +Since the entry point is not used when running benchmarks through VSTest, + the default configuration must be specified using a `Config` attribute that is set on the assembly instead. + +First, create a class that extends `ManualConfig` or `IConfig` which sets the default configuration you want: + +```csharp +class MyDefaultConfig : ManualConfig +{ + public MyDefaultConfig() + { + AddJob(Job.Dry); + AddLogger(Loggers.ConsoleLogger.Default); + AddValidator(JitOptimizationsValidator.DontFailOnError); + } +} +``` + +Then, set an assembly attribute with the following. + +```csharp +[assembly: Config(typeof(MyDefaultConfig))] +``` + +By convention, assembly attributes are usually defined inside `AssemblyInfo.cs` in a directory called `Properties`. + +## Viewing the results + +The full output from BenchmarkDotNet that you would have been used to seeing in the past will be sent to the "Tests" output of your IDE. +Use this view if you want to see the tabular view that compares multiple benchmarks with each other or + if you want to see the results for each individual iteration. + +One more place where you can view the results is in each individual test's output messages. +In Visual Studio, this can be viewed by clicking on the test in the Test Explorer after running it and looking at the Test Detail Summary. +Since this only displays statistics for a single benchmark case, + it does not show the tabulated view that compares multiple benchmark cases. +Instead, it displays a histogram and various other useful statistics. +Not all IDEs support displaying these output messages, so you may only be able to view the results using the "Tests" output. diff --git a/docs/articles/guides/console-args.md b/docs/articles/guides/console-args.md index 6fc5ddb245..765ad2e2bb 100644 --- a/docs/articles/guides/console-args.md +++ b/docs/articles/guides/console-args.md @@ -117,28 +117,29 @@ You can also filter the benchmarks by categories: The `--runtimes` or just `-r` allows you to run the benchmarks for selected Runtimes. Available options are: * Clr - BDN will either use Roslyn (if you run it as .NET app) or latest installed .NET SDK to build the benchmarks (if you run it as .NET Core app). -* Core - if you run it as .NET Core app, BDN will use the same target framework moniker, if you run it as .NET app it's going to use netcoreapp2.1. +* Core - if you run it as .NET Core app, BDN will use the same target framework moniker, if you run it as .NET app it's going to use net8.0. * Mono - it's going to use the Mono from `$Path`, you can override it with `--monoPath`. -* net46, net461, net462, net47, net471, net472 - to build and run benchmarks against specific .NET framework version. -* netcoreapp2.0, netcoreapp2.1, netcoreapp2.2, netcoreapp3.0, netcoreapp3.1, net5.0, net6.0, net7.0 - to build and run benchmarks against specific .NET Core version. -* nativeaot5.0, nativeaot6.0, nativeaot7.0 - to build and run benchmarks using NativeAOT. Can be customized with additional options: `--ilcPath`, `--ilCompilerVersion`. +* net46, net461, net462, net47, net471, net472, net48, net481 - to build and run benchmarks against specific .NET Framework version. +* netcoreapp3.1, net5.0, net6.0, net7.0, net8.0 - to build and run benchmarks against specific .NET (Core) version. +* nativeaot5.0, nativeaot6.0, nativeaot7.0, nativeaot8.0 - to build and run benchmarks using NativeAOT. Can be customized with additional options: `--ilcPackages`, `--ilCompilerVersion`. +* mono6.0, mono7.0, mono8.0 - to build and run benchmarks with .Net 6+ using MonoVM. -Example: run the benchmarks for .NET 4.7.2 and .NET Core 2.1: +Example: run the benchmarks for .NET 4.7.2 and .NET 8.0: ```log -dotnet run -c Release -- --runtimes net472 netcoreapp2.1 +dotnet run -c Release -- --runtimes net472 net8.0 ``` -Example: run the benchmarks for .NET Core 3.0 and latest .NET SDK installed on your PC: +Example: run the benchmarks for .NET Core 3.1 and latest .NET SDK installed on your PC: ```log -dotnet run -c Release -f netcoreapp3.0 -- --runtimes clr core +dotnet run -c Release -f netcoreapp3.1 -- --runtimes clr core ``` -But same command executed with `-f netcoreapp2.0` is going to run the benchmarks for .NET Core 2.0: +But same command executed with `-f net6.0` is going to run the benchmarks for .NET 6.0: ```log -dotnet run -c Release -f netcoreapp2.0 -- --runtimes clr core +dotnet run -c Release -f net6.0 -- --runtimes clr core ``` ## Number of invocations and iterations @@ -207,10 +208,10 @@ To perform a Mann–Whitney U Test and display the results in a dedicated column * `--statisticalTest`- Threshold for Mann–Whitney U Test. Examples: 5%, 10ms, 100ns, 1s -Example: run Mann–Whitney U test with relative ratio of 5% for all benchmarks for .NET Core 2.0 (base) vs .NET Core 2.1 (diff). .NET Core 2.0 will be baseline because it was first. +Example: run Mann–Whitney U test with relative ratio of 5% for all benchmarks for .NET 6.0 (base) vs .NET 8.0 (diff). .NET 6.0 will be baseline because it was first. ```log -dotnet run -c Release -- --filter * --runtimes netcoreapp2.0 netcoreapp2.1 --statisticalTest 5% +dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5% ``` ## More @@ -228,7 +229,7 @@ dotnet run -c Release -- --filter * --runtimes netcoreapp2.0 netcoreapp2.1 --sta * `--cli` path to dotnet cli (optional). * `--packages` the directory to restore packages to (optional). * `--coreRun` path(s) to CoreRun (optional). -* `--ilcPath` path to ILCompiler for NativeAOT. +* `--ilcPackages` path to ILCompiler for NativeAOT. * `--info` prints environment configuration including BenchmarkDotNet, OS, CPU and .NET version * `--stopOnFirstError` stop on first error. * `--help` display this help screen. @@ -242,6 +243,7 @@ dotnet run -c Release -- --filter * --runtimes netcoreapp2.0 netcoreapp2.1 --sta * `--platform` the Platform that should be used. If not specified, the host process platform is used (default). AnyCpu/X86/X64/Arm/Arm64/LoongArch64. * `--runOncePerIteration` run the benchmark exactly once per iteration. * `--buildTimeout` build timeout in seconds. +* `--wakeLock` prevents the system from entering sleep or turning off the display. None/System/Display. * `--wasmEngine` full path to a java script engine used to run the benchmarks, used by Wasm toolchain. * `--wasmMainJS` path to the test-main.js file used by Wasm toolchain. Mandatory when using \"--runtimes wasm\" * `--expose_wasm` arguments for the JavaScript engine used by Wasm toolchain. diff --git a/docs/articles/guides/good-practices.md b/docs/articles/guides/good-practices.md index 854ead5679..da4329de2b 100644 --- a/docs/articles/guides/good-practices.md +++ b/docs/articles/guides/good-practices.md @@ -4,8 +4,8 @@ Never use the Debug build for benchmarking. *Never*. The debug version of the target method can run 10–100 times slower. The release mode means that you should have `true` in your csproj file -or use [/optimize](https://learn.microsoft.com/dotnet/csharp/language-reference/compiler-options/) for `csc`. Also your never -should use an attached debugger (e.g. Visual Studio or WinDbg) during the benchmarking. The best way is +or use [/optimize](https://learn.microsoft.com/dotnet/csharp/language-reference/compiler-options/) for `csc`. Also, never +use an attached debugger (e.g. Visual Studio or WinDbg) during the benchmarking. The best way is build our benchmark in the Release mode and run it from the command line. ## Try different environments diff --git a/docs/articles/guides/nuget.md b/docs/articles/guides/nuget.md index 0e2de62a79..d1359c03d9 100644 --- a/docs/articles/guides/nuget.md +++ b/docs/articles/guides/nuget.md @@ -13,6 +13,7 @@ We have the following set of NuGet packages (you can install it directly from `n * `BenchmarkDotNet.Annotations`: Basic BenchmarkDotNet annotations for your benchmarks. * `BenchmarkDotNet.Diagnostics.Windows`: an additional optional package that provides a set of Windows diagnosers. * `BenchmarkDotNet.Diagnostics.dotTrace`: an additional optional package that provides DotTraceDiagnoser. +* `BenchmarkDotNet.Diagnostics.dotMemory`: an additional optional package that provides DotMemoryDiagnoser. * `BenchmarkDotNet.Templates`: Templates for BenchmarkDotNet. You might find other NuGet packages that start with `BenchmarkDotNet` name, but they are internal BDN packages that should not be installed manually. All that matters are the three packages mentioned above. diff --git a/docs/articles/license.md b/docs/articles/license.md index d20c387e81..6272f97306 100644 --- a/docs/articles/license.md +++ b/docs/articles/license.md @@ -1,6 +1,6 @@ ### The MIT License -Copyright (c) 2013–2023 .NET Foundation and contributors +Copyright (c) 2013–2025 .NET Foundation and contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/docs/articles/samples/IntroDotMemoryDiagnoser.md b/docs/articles/samples/IntroDotMemoryDiagnoser.md new file mode 100644 index 0000000000..e16e96a83c --- /dev/null +++ b/docs/articles/samples/IntroDotMemoryDiagnoser.md @@ -0,0 +1,22 @@ +--- +uid: BenchmarkDotNet.Samples.IntroDotMemoryDiagnoser +--- + +## Sample: IntroDotMemoryDiagnoser + +If you want to get a memory allocation profile of your benchmarks, just add the `[DotMemoryDiagnoser]` attribute, as shown below. +As a result, BenchmarkDotNet performs bonus benchmark runs using attached + [dotMemory Command-Line Profiler](https://www.jetbrains.com/help/dotmemory/Working_with_dotMemory_Command-Line_Profiler.html). +The obtained dotMemory workspaces are saved to the `artifacts` folder. +These dotMemory workspaces can be opened using the [standalone dotMemory](https://www.jetbrains.com/dotmemory/), + or [dotMemory in Rider](https://www.jetbrains.com/help/rider/Memory_profiling_of_.NET_code.html). + +### Source code + +[!code-csharp[IntroDotMemoryDiagnoser.cs](../../../samples/BenchmarkDotNet.Samples/IntroDotMemoryDiagnoser.cs)] + +### Links + +* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroDotMemoryDiagnoser + +--- \ No newline at end of file diff --git a/docs/articles/samples/IntroSummaryStyle.md b/docs/articles/samples/IntroSummaryStyle.md new file mode 100644 index 0000000000..470bb901a1 --- /dev/null +++ b/docs/articles/samples/IntroSummaryStyle.md @@ -0,0 +1,51 @@ +--- +uid: BenchmarkDotNet.SummaryStyle +--- + +## SummaryStyle in BenchmarkDotNet + +`SummaryStyle` is a class in BenchmarkDotNet that allows customization of the summary reports of benchmark results. It offers several properties to fine-tune how the results are displayed. + +### Usage + +You can customize the summary report by specifying various properties of `SummaryStyle`. These properties include formatting options like whether to print units in the header or content, setting the maximum width for parameter columns, and choosing units for size and time measurements. + +### Source Code + +[!code-csharp[IntroSummaryStyle.cs](../../../samples/BenchmarkDotNet.Samples/IntroSummaryStyle.cs)] + +### Properties + +- `PrintUnitsInHeader`: Boolean to indicate if units should be printed in the header. +- `PrintUnitsInContent`: Boolean to control unit printing in the content. +- `PrintZeroValuesInContent`: Determines if zero values should be printed. +- `MaxParameterColumnWidth`: Integer defining the max width for parameter columns. +- `SizeUnit`: Optional `SizeUnit` to specify the unit for size measurements. +- `TimeUnit`: Optional `TimeUnit` for time measurement units. +- `CultureInfo`: `CultureInfo` to define culture-specific formatting. + +### Example Output + +Using SummaryStyle options: + +```markdown +| Method | N | Mean [ns] | Error [ns] | StdDev [ns] | +|------- |---- |--------------:|-----------:|------------:| +| Sleep | 10 | 15,644,973.1 | 32,808.7 | 30,689.3 | +| Sleep | 100 | 109,440,686.7 | 236,673.8 | 221,384.8 | +``` + +Default: + +```markdown +| Method | N | Mean | Error | StdDev | +|------- |---- |----------:|---------:|---------:| +| Sleep | 10 | 15.65 ms | 0.039 ms | 0.034 ms | +| Sleep | 100 | 109.20 ms | 0.442 ms | 0.392 ms | +``` + +### Links + +* @docs.SummaryStyle +* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroSummaryStyle + diff --git a/docs/articles/samples/IntroVisualStudioProfiler.md b/docs/articles/samples/IntroVisualStudioProfiler.md new file mode 100644 index 0000000000..8d35498f21 --- /dev/null +++ b/docs/articles/samples/IntroVisualStudioProfiler.md @@ -0,0 +1,23 @@ +--- +uid: BenchmarkDotNet.Samples.IntroVisualStudioProfiler +--- + +## Sample: Visual Studio Profiler + +Using the [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package you can capture performance profiles of your benchmarks that can be opened in Visual Studio. + +### Source code + +[!code-csharp[IntroVisualStudioDiagnoser.cs](../../../samples/BenchmarkDotNet.Samples/IntroVisualStudioDiagnoser.cs)] + +### Output +The output will contain a path to the collected diagsession and automatically open in Visual Studio when launched from it. + +```markdown +// * Diagnostic Output - VSDiagnosticsDiagnoser * +Collection result moved to 'C:\Work\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\net8.0\BenchmarkDotNet.Artifacts\BenchmarkDotNet_IntroVisualStudioProfiler_20241205_192056.diagsession'. +Session : {d54ebddb-2d6d-404f-b1da-10acbc89635f} + Stopped +Exported diagsession file: C:\Work\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\net8.0\BenchmarkDotNet.Artifacts\BenchmarkDotNet_IntroVisualStudioProfiler_20241205_192056.diagsession. +Opening diagsession in VisualStudio: 15296 +``` \ No newline at end of file diff --git a/docs/articles/samples/IntroWakeLock.md b/docs/articles/samples/IntroWakeLock.md new file mode 100644 index 0000000000..333b3e4345 --- /dev/null +++ b/docs/articles/samples/IntroWakeLock.md @@ -0,0 +1,32 @@ +--- +uid: BenchmarkDotNet.Samples.IntroWakeLock +--- + +## Sample: IntroWakeLock + +Running benchmarks may sometimes take enough time such that the system enters sleep or turns off the display. + +Using a WakeLock prevents the Windows system doing so. + +### Source code + +[!code-csharp[IntroWakeLock.cs](../../../samples/BenchmarkDotNet.Samples/IntroWakeLock.cs)] + +### Command line + +``` +--wakeLock None +``` +``` +--wakeLock System +``` +``` +--wakeLock Display +``` + +### Links + +* @BenchmarkDotNet.Attributes.WakeLockAttribute +* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroWakeLock + +--- diff --git a/docs/articles/samples/toc.yml b/docs/articles/samples/toc.yml index 8098a637c3..8dab3b5c6b 100644 --- a/docs/articles/samples/toc.yml +++ b/docs/articles/samples/toc.yml @@ -38,6 +38,8 @@ href: IntroDisassemblyRyuJit.md - name: IntroDotTraceDiagnoser href: IntroDotTraceDiagnoser.md +- name: IntroDotMemoryDiagnoser + href: IntroDotMemoryDiagnoser.md - name: IntroEnvVars href: IntroEnvVars.md - name: IntroEventPipeProfiler @@ -122,6 +124,10 @@ href: IntroTagColumn.md - name: IntroTailcall href: IntroTailcall.md +- name: IntroVisualStudioProfiler + href: IntroVisualStudioProfiler.md +- name: IntroWakeLock + href: IntroWakeLock.md - name: IntroWasm href: IntroWasm.md - name: IntroUnicode diff --git a/docs/articles/toc.yml b/docs/articles/toc.yml index 29ad2cce16..60f1d5e11a 100644 --- a/docs/articles/toc.yml +++ b/docs/articles/toc.yml @@ -14,5 +14,5 @@ href: faq.md - name: Team href: team.md -- name: Licence - href: license.md \ No newline at end of file +- name: License + href: license.md diff --git a/docs/docfx.json b/docs/docfx.json index 52ed261bb6..291ff58dbd 100644 --- a/docs/docfx.json +++ b/docs/docfx.json @@ -64,7 +64,7 @@ "globalMetadata" : { "_appTitle" : "BenchmarkDotNet", - "_appFooter" : "Copyright © 2013–2023 .NET Foundation and contributors", + "_appFooter" : "Copyright © 2013–2024 .NET Foundation and contributors", "_appLogoPath" : "logo/icon.svg", "_appFaviconPath": "logo/icon-32.png", "_enableSearch": true, diff --git a/docs/images/vs-profiler-demo.png b/docs/images/vs-profiler-demo.png new file mode 100644 index 0000000000..928f012df1 Binary files /dev/null and b/docs/images/vs-profiler-demo.png differ diff --git a/docs/images/vs-testexplorer-demo.png b/docs/images/vs-testexplorer-demo.png new file mode 100644 index 0000000000..91232e30d6 Binary files /dev/null and b/docs/images/vs-testexplorer-demo.png differ diff --git a/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj b/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj index 55045658d8..457b660115 100644 --- a/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj +++ b/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj @@ -5,13 +5,15 @@ Exe - net462;net7.0 + net462;net8.0 + false + @@ -19,11 +21,8 @@ - - all - runtime; build; native; contentfiles; analyzers - + diff --git a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj index 180148454a..7cdc359d7d 100644 --- a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj +++ b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet.Samples - net7.0;net462 + net8.0;net462 true BenchmarkDotNet.Samples Exe @@ -11,6 +11,8 @@ AnyCPU true $(NoWarn);CA1018;CA5351;CA1825 + + false @@ -19,10 +21,16 @@ + + + + + + diff --git a/samples/BenchmarkDotNet.Samples/IntroDotMemoryDiagnoser.cs b/samples/BenchmarkDotNet.Samples/IntroDotMemoryDiagnoser.cs new file mode 100644 index 0000000000..894bfc6c34 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroDotMemoryDiagnoser.cs @@ -0,0 +1,46 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Diagnostics.dotMemory; +using System.Collections.Generic; + +namespace BenchmarkDotNet.Samples +{ + // Profile benchmarks via dotMemory SelfApi profiling for all jobs + [DotMemoryDiagnoser] + [SimpleJob] // external-process execution + [InProcess] // in-process execution + public class IntroDotMemoryDiagnoser + { + [Params(1024)] + public int Size; + + private byte[] dataArray; + private IEnumerable dataEnumerable; + + [GlobalSetup] + public void Setup() + { + dataArray = new byte[Size]; + dataEnumerable = dataArray; + } + + [Benchmark] + public int IterateArray() + { + var count = 0; + foreach (var _ in dataArray) + count++; + + return count; + } + + [Benchmark] + public int IterateEnumerable() + { + var count = 0; + foreach (var _ in dataEnumerable) + count++; + + return count; + } + } +} \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroDotTraceDiagnoser.cs b/samples/BenchmarkDotNet.Samples/IntroDotTraceDiagnoser.cs index 351207c78b..047e6ee059 100644 --- a/samples/BenchmarkDotNet.Samples/IntroDotTraceDiagnoser.cs +++ b/samples/BenchmarkDotNet.Samples/IntroDotTraceDiagnoser.cs @@ -3,16 +3,11 @@ namespace BenchmarkDotNet.Samples { - // Enables dotTrace profiling for all jobs + // Profile benchmarks via dotTrace SelfApi profiling for all jobs + // See: https://www.nuget.org/packages/JetBrains.Profiler.SelfApi [DotTraceDiagnoser] - // Adds the default "external-process" job - // Profiling is performed using dotTrace command-line Tools - // See: https://www.jetbrains.com/help/profiler/Performance_Profiling__Profiling_Using_the_Command_Line.html - [SimpleJob] - // Adds an "in-process" job - // Profiling is performed using dotTrace SelfApi - // NuGet reference: https://www.nuget.org/packages/JetBrains.Profiler.SelfApi - [InProcess] + [SimpleJob] // external-process execution + [InProcess] // in-process execution public class IntroDotTraceDiagnoser { [Benchmark] diff --git a/samples/BenchmarkDotNet.Samples/IntroEnvVars.cs b/samples/BenchmarkDotNet.Samples/IntroEnvVars.cs index 59bd2db82f..66f5197119 100644 --- a/samples/BenchmarkDotNet.Samples/IntroEnvVars.cs +++ b/samples/BenchmarkDotNet.Samples/IntroEnvVars.cs @@ -10,13 +10,14 @@ public class IntroEnvVars { private class ConfigWithCustomEnvVars : ManualConfig { - private const string JitNoInline = "COMPlus_JitNoInline"; - public ConfigWithCustomEnvVars() { - AddJob(Job.Default.WithRuntime(CoreRuntime.Core21).WithId("Inlining enabled")); - AddJob(Job.Default.WithRuntime(CoreRuntime.Core21) - .WithEnvironmentVariables(new EnvironmentVariable(JitNoInline, "1")) + AddJob(Job.Default.WithRuntime(CoreRuntime.Core80).WithId("Inlining enabled")); + AddJob(Job.Default.WithRuntime(CoreRuntime.Core80) + .WithEnvironmentVariables([ + new EnvironmentVariable("DOTNET_JitNoInline", "1"), + new EnvironmentVariable("COMPlus_JitNoInline", "1") + ]) .WithId("Inlining disabled")); } } @@ -27,4 +28,4 @@ public void Foo() // Benchmark body } } -} \ No newline at end of file +} diff --git a/samples/BenchmarkDotNet.Samples/IntroExceptionDiagnoser.cs b/samples/BenchmarkDotNet.Samples/IntroExceptionDiagnoser.cs index a43844e592..7e0f847c95 100644 --- a/samples/BenchmarkDotNet.Samples/IntroExceptionDiagnoser.cs +++ b/samples/BenchmarkDotNet.Samples/IntroExceptionDiagnoser.cs @@ -16,7 +16,10 @@ public void ThrowExceptionRandomly() throw new Exception(); } } - catch { } + catch + { + // ignored + } } } } diff --git a/samples/BenchmarkDotNet.Samples/IntroFluentConfigBuilder.cs b/samples/BenchmarkDotNet.Samples/IntroFluentConfigBuilder.cs index 94c34c1cbd..4ac29919ee 100644 --- a/samples/BenchmarkDotNet.Samples/IntroFluentConfigBuilder.cs +++ b/samples/BenchmarkDotNet.Samples/IntroFluentConfigBuilder.cs @@ -38,7 +38,7 @@ public static void Run() .Run( DefaultConfig.Instance .AddJob(Job.Default.WithRuntime(ClrRuntime.Net462)) - .AddJob(Job.Default.WithRuntime(CoreRuntime.Core21)) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80)) .AddValidator(ExecutionValidator.FailOnError)); } } diff --git a/samples/BenchmarkDotNet.Samples/IntroOrderManual.cs b/samples/BenchmarkDotNet.Samples/IntroOrderManual.cs index 625e78a816..a624edd205 100644 --- a/samples/BenchmarkDotNet.Samples/IntroOrderManual.cs +++ b/samples/BenchmarkDotNet.Samples/IntroOrderManual.cs @@ -22,7 +22,7 @@ private class Config : ManualConfig private class FastestToSlowestOrderer : IOrderer { public IEnumerable GetExecutionOrder(ImmutableArray benchmarksCase, - IEnumerable order = null) => + IEnumerable? order = null) => from benchmark in benchmarksCase orderby benchmark.Parameters["X"] descending, benchmark.Descriptor.WorkloadMethodDisplayInfo @@ -39,7 +39,7 @@ public string GetLogicalGroupKey(ImmutableArray allBenchmarksCase benchmarkCase.Job.DisplayInfo + "_" + benchmarkCase.Parameters.DisplayInfo; public IEnumerable> GetLogicalGroupOrder(IEnumerable> logicalGroups, - IEnumerable order = null) => + IEnumerable? order = null) => logicalGroups.OrderBy(it => it.Key); public bool SeparateLogicalGroups => true; diff --git a/samples/BenchmarkDotNet.Samples/IntroSmokeEmptyBasic.cs b/samples/BenchmarkDotNet.Samples/IntroSmokeEmptyBasic.cs new file mode 100644 index 0000000000..39783dc74a --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroSmokeEmptyBasic.cs @@ -0,0 +1,82 @@ +using BenchmarkDotNet.Attributes; + +namespace BenchmarkDotNet.Samples; + +[DisassemblyDiagnoser] +public class IntroSmokeEmptyBasic +{ + [Benchmark] public void Void1() {} + [Benchmark] public void Void2() {} + [Benchmark] public void Void3() {} + [Benchmark] public void Void4() {} + + [Benchmark] public byte Byte1() => 0; + [Benchmark] public byte Byte2() => 0; + [Benchmark] public byte Byte3() => 0; + [Benchmark] public byte Byte4() => 0; + + [Benchmark] public sbyte Sbyte1() => 0; + [Benchmark] public sbyte Sbyte2() => 0; + [Benchmark] public sbyte Sbyte3() => 0; + [Benchmark] public sbyte Sbyte4() => 0; + + [Benchmark] public short Short1() => 0; + [Benchmark] public short Short2() => 0; + [Benchmark] public short Short3() => 0; + [Benchmark] public short Short4() => 0; + + [Benchmark] public ushort Ushort1() => 0; + [Benchmark] public ushort Ushort2() => 0; + [Benchmark] public ushort Ushort3() => 0; + [Benchmark] public ushort Ushort4() => 0; + + [Benchmark] public int Int1() => 0; + [Benchmark] public int Int2() => 0; + [Benchmark] public int Int3() => 0; + [Benchmark] public int Int4() => 0; + + [Benchmark] public uint Uint1() => 0u; + [Benchmark] public uint Uint2() => 0u; + [Benchmark] public uint Uint3() => 0u; + [Benchmark] public uint Uint4() => 0u; + + [Benchmark] public bool Bool1() => false; + [Benchmark] public bool Bool2() => false; + [Benchmark] public bool Bool3() => false; + [Benchmark] public bool Bool4() => false; + + [Benchmark] public char Char1() => 'a'; + [Benchmark] public char Char2() => 'a'; + [Benchmark] public char Char3() => 'a'; + [Benchmark] public char Char4() => 'a'; + + [Benchmark] public float Float1() => 0f; + [Benchmark] public float Float2() => 0f; + [Benchmark] public float Float3() => 0f; + [Benchmark] public float Float4() => 0f; + + [Benchmark] public double Double1() => 0d; + [Benchmark] public double Double2() => 0d; + [Benchmark] public double Double3() => 0d; + [Benchmark] public double Double4() => 0d; + + [Benchmark] public long Long1() => 0L; + [Benchmark] public long Long2() => 0L; + [Benchmark] public long Long3() => 0L; + [Benchmark] public long Long4() => 0L; + + [Benchmark] public ulong Ulong1() => 0uL; + [Benchmark] public ulong Ulong2() => 0uL; + [Benchmark] public ulong Ulong3() => 0uL; + [Benchmark] public ulong Ulong4() => 0uL; + + [Benchmark] public string String1() => ""; + [Benchmark] public string String2() => ""; + [Benchmark] public string String3() => ""; + [Benchmark] public string String4() => ""; + + [Benchmark] public object? Object1() => null; + [Benchmark] public object? Object2() => null; + [Benchmark] public object? Object3() => null; + [Benchmark] public object? Object4() => null; +} \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroSmokeIncrements.cs b/samples/BenchmarkDotNet.Samples/IntroSmokeIncrements.cs new file mode 100644 index 0000000000..6dfd15433d --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroSmokeIncrements.cs @@ -0,0 +1,138 @@ +using BenchmarkDotNet.Attributes; + +namespace BenchmarkDotNet.Samples; + +public class IntroSmokeIncrements +{ + public int Field; + + [Benchmark] + public void Increment01() + { + Field++; + } + + [Benchmark] + public void Increment02() + { + Field++; + Field++; + } + + [Benchmark] + public void Increment03() + { + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment04() + { + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment05() + { + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment06() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment07() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment08() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment09() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment10() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment20() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } +} \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroSmokeValueTypes.cs b/samples/BenchmarkDotNet.Samples/IntroSmokeValueTypes.cs new file mode 100644 index 0000000000..66bce4d921 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroSmokeValueTypes.cs @@ -0,0 +1,70 @@ +using System; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Environments; + +namespace BenchmarkDotNet.Samples; + +[MemoryDiagnoser, DisassemblyDiagnoser] +public class IntroSmokeValueTypes +{ + [Benchmark] public Jit ReturnEnum() => Jit.RyuJit; + + [Benchmark] public DateTime ReturnDateTime() => new DateTime(); + + [Benchmark] public DateTime? ReturnNullableDateTime() => new DateTime(); + [Benchmark] public int? ReturnNullableInt() => 0; + + public struct StructWithReferencesOnly { public object _ref; } + [Benchmark] public StructWithReferencesOnly ReturnStructWithReferencesOnly() => new StructWithReferencesOnly(); + + public struct EmptyStruct { } + [Benchmark] public EmptyStruct ReturnEmptyStruct() => new EmptyStruct(); + + [Benchmark] public ValueTuple ReturnGenericStructOfValueType() => new ValueTuple(0); + [Benchmark] public ValueTuple ReturnGenericStructOfReferenceType() => new ValueTuple(null); + + [Benchmark] public ValueTask ReturnValueTaskOfValueType() => new ValueTask(0); + [Benchmark] public ValueTask ReturnValueTaskOfReferenceType() => new ValueTask(result: null); + + [Benchmark] public byte ReturnByte() => 0; + public struct Byte1 { public byte _1; } + [Benchmark] public Byte1 ReturnByte1() => new Byte1(); + public struct Byte2 { public byte _1, _2; } + [Benchmark] public Byte2 ReturnByte2() => new Byte2(); + public struct Byte3 { public byte _1, _2, _3; } + [Benchmark] public Byte3 ReturnByte3() => new Byte3(); + public struct Byte4 { public byte _1, _2, _3, _4; } + [Benchmark] public Byte4 ReturnByte4() => new Byte4(); + + [Benchmark] public short ReturnShort() => 0; + public struct Short1 { public short _1; } + [Benchmark] public Short1 ReturnShort1() => new Short1(); + public struct Short2 { public short _1, _2; } + [Benchmark] public Short2 ReturnShort2() => new Short2(); + public struct Short3 { public short _1, _2, _3; } + [Benchmark] public Short3 ReturnShort3() => new Short3(); + public struct Short4 { public short _1, _2, _3, _4; } + [Benchmark] public Short4 ReturnShort4() => new Short4(); + + [Benchmark] public int ReturnInt() => 0; + public struct Int1 { public int _1; } + [Benchmark] public Int1 ReturnInt1() => new Int1(); + public struct Int2 { public int _1, _2; } + [Benchmark] public Int2 ReturnInt2() => new Int2(); + public struct Int3 { public int _1, _2, _3; } + [Benchmark] public Int3 ReturnInt3() => new Int3(); + public struct Int4 { public int _1, _2, _3, _4; } + [Benchmark] public Int4 ReturnInt4() => new Int4(); + + [Benchmark] public long ReturnLong() => 0; + public struct Long1 { public long _1; } + [Benchmark] public Long1 ReturnLong1() => new Long1(); + public struct Long2 { public long _1, _2; } + [Benchmark] public Long2 ReturnLong2() => new Long2(); + public struct Long3 { public long _1, _2, _3; } + [Benchmark] public Long3 ReturnLong3() => new Long3(); + public struct Long4 { public long _1, _2, _3, _4; } + [Benchmark] public Long4 ReturnLong4() => new Long4(); +} +// ReSharper restore InconsistentNaming \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs b/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs index bbdf85aad5..6351281aa6 100644 --- a/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs +++ b/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs @@ -1,14 +1,10 @@ using System.Threading; using BenchmarkDotNet.Attributes; -using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; namespace BenchmarkDotNet.Samples { - [StatisticalTestColumn(StatisticalTestKind.Welch, ThresholdUnit.Microseconds, 1, true)] - [StatisticalTestColumn(StatisticalTestKind.MannWhitney, ThresholdUnit.Microseconds, 1, true)] - [StatisticalTestColumn(StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.03, true)] - [StatisticalTestColumn(StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.03, true)] + [StatisticalTestColumn("500us")] + [StatisticalTestColumn("3%")] [SimpleJob(warmupCount: 0, iterationCount: 5)] public class IntroStatisticalTesting { diff --git a/samples/BenchmarkDotNet.Samples/IntroSummaryStyle.cs b/samples/BenchmarkDotNet.Samples/IntroSummaryStyle.cs new file mode 100644 index 0000000000..6a9830be18 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroSummaryStyle.cs @@ -0,0 +1,39 @@ +using System.Globalization; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Reports; +using Perfolizer.Horology; +using Perfolizer.Metrology; + +namespace BenchmarkDotNet.Samples +{ + [Config(typeof(Config))] + public class IntroSummaryStyle + { + private class Config : ManualConfig + { + public Config() + { + // Configure the summary style here + var summaryStyle = new SummaryStyle + ( + cultureInfo: CultureInfo.InvariantCulture, + printUnitsInHeader: true, + printUnitsInContent: false, + sizeUnit: SizeUnit.KB, + timeUnit: TimeUnit.Nanosecond, + maxParameterColumnWidth: 20 + + ); + + WithSummaryStyle(summaryStyle); + } + } + + [Params(10, 100)] + public int N; + + [Benchmark] + public void Sleep() => System.Threading.Thread.Sleep(N); + } +} \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroVisualStudioDiagnoser.cs b/samples/BenchmarkDotNet.Samples/IntroVisualStudioDiagnoser.cs new file mode 100644 index 0000000000..e513f39320 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroVisualStudioDiagnoser.cs @@ -0,0 +1,23 @@ +using System; +using BenchmarkDotNet.Attributes; +using Microsoft.VSDiagnostics; + +namespace BenchmarkDotNet.Samples +{ + // Enables profiling with the CPU Usage tool + // See: https://learn.microsoft.com/visualstudio/profiling/profiling-with-benchmark-dotnet + [CPUUsageDiagnoser] + public class IntroVisualStudioProfiler + { + private readonly Random rand = new Random(42); + + [Benchmark] + public void BurnCPU() + { + for (int i = 0; i < 100000; ++i) + { + rand.Next(1, 100); + } + } + } +} \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroWakeLock.cs b/samples/BenchmarkDotNet.Samples/IntroWakeLock.cs new file mode 100644 index 0000000000..099b347c7b --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroWakeLock.cs @@ -0,0 +1,30 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using System; +using System.Threading; + +// *** Attribute Style applied to Assembly *** +[assembly: WakeLock(WakeLockType.System)] + +namespace BenchmarkDotNet.Samples; + +// *** Attribute Style *** +[WakeLock(WakeLockType.Display)] +public class IntroWakeLock +{ + [Benchmark] + public void LongRunning() => Thread.Sleep(TimeSpan.FromSeconds(10)); +} + +// *** Object Style *** +[Config(typeof(Config))] +public class IntroWakeLockObjectStyle +{ + private class Config : ManualConfig + { + public Config() => WakeLock = WakeLockType.System; + } + + [Benchmark] + public void LongRunning() => Thread.Sleep(TimeSpan.FromSeconds(10)); +} diff --git a/src/BenchmarkDotNet.Annotations/Attributes/ParamsAttribute.cs b/src/BenchmarkDotNet.Annotations/Attributes/ParamsAttribute.cs index 22148ffa64..aa47ebdf96 100644 --- a/src/BenchmarkDotNet.Annotations/Attributes/ParamsAttribute.cs +++ b/src/BenchmarkDotNet.Annotations/Attributes/ParamsAttribute.cs @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Attributes [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)] public class ParamsAttribute : PriorityAttribute { - public object?[] Values { get; } + public object?[] Values { get; protected set; } // CLS-Compliant Code requires a constructor without an array in the argument list public ParamsAttribute() => Values = new object[0]; diff --git a/src/BenchmarkDotNet.Annotations/Attributes/TargetedAttribute.cs b/src/BenchmarkDotNet.Annotations/Attributes/TargetedAttribute.cs index f026675bd7..efeb7ee0c6 100644 --- a/src/BenchmarkDotNet.Annotations/Attributes/TargetedAttribute.cs +++ b/src/BenchmarkDotNet.Annotations/Attributes/TargetedAttribute.cs @@ -20,6 +20,6 @@ public string Target set => Targets = string.IsNullOrEmpty(value) ? new string[0] : value.Split(','); // , is for backward compat } - public bool Match(MethodInfo method) => Targets == null || Targets.Length == 0 || Targets.Contains(method.Name); + public bool Match(MethodInfo method) => Targets.Length == 0 || Targets.Contains(method.Name); } } diff --git a/src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj b/src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj index 2e7361e163..ccc51bcd9a 100644 --- a/src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj +++ b/src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj @@ -2,7 +2,7 @@ Basic BenchmarkDotNet attributes that can be used to annotate your benchmarks - netstandard1.0;netstandard2.0 + netstandard2.0 $(NoWarn);1701;1702;1705;1591;3005;NU1702;CA1825 BenchmarkDotNet.Annotations BenchmarkDotNet.Annotations diff --git a/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs b/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs index ca9b5a0772..16b242d2dc 100644 --- a/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs +++ b/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs @@ -105,6 +105,16 @@ public enum RuntimeMoniker /// Net80, + /// + /// .NET 9.0 + /// + Net90, + + /// + /// .NET 10.0 + /// + Net10_0, + /// /// NativeAOT compiled as net6.0 /// @@ -120,6 +130,16 @@ public enum RuntimeMoniker /// NativeAot80, + /// + /// NativeAOT compiled as net9.0 + /// + NativeAot90, + + /// + /// NativeAOT compiled as net10.0 + /// + NativeAot10_0, + /// /// WebAssembly with default .Net version /// @@ -145,6 +165,16 @@ public enum RuntimeMoniker /// WasmNet80, + /// + /// WebAssembly with net9.0 + /// + WasmNet90, + + /// + /// WebAssembly with net10.0 + /// + WasmNet10_0, + /// /// Mono with the Ahead of Time LLVM Compiler backend /// @@ -165,6 +195,16 @@ public enum RuntimeMoniker /// MonoAOTLLVMNet80, + /// + /// Mono with the Ahead of Time LLVM Compiler backend and net9.0 + /// + MonoAOTLLVMNet90, + + /// + /// Mono with the Ahead of Time LLVM Compiler backend and net10.0 + /// + MonoAOTLLVMNet10_0, + /// /// .NET 6 using MonoVM (not CLR which is the default) /// @@ -179,5 +219,15 @@ public enum RuntimeMoniker /// .NET 8 using MonoVM (not CLR which is the default) /// Mono80, + + /// + /// .NET 9 using MonoVM (not CLR which is the default) + /// + Mono90, + + /// + /// .NET 10 using MonoVM (not CLR which is the default) + /// + Mono10_0, } } diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/BenchmarkDotNet.Diagnostics.Windows.csproj b/src/BenchmarkDotNet.Diagnostics.Windows/BenchmarkDotNet.Diagnostics.Windows.csproj index 6503c7fa7b..6c9e43a3f4 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/BenchmarkDotNet.Diagnostics.Windows.csproj +++ b/src/BenchmarkDotNet.Diagnostics.Windows/BenchmarkDotNet.Diagnostics.Windows.csproj @@ -12,6 +12,6 @@ - + diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Configs/InliningDiagnoserAttribute.cs b/src/BenchmarkDotNet.Diagnostics.Windows/Configs/InliningDiagnoserAttribute.cs index 0a3f4e0dc5..c9878ed6fc 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Configs/InliningDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/Configs/InliningDiagnoserAttribute.cs @@ -13,7 +13,7 @@ public InliningDiagnoserAttribute(bool logFailuresOnly = true, bool filterByName Config = ManualConfig.CreateEmpty().AddDiagnoser(new InliningDiagnoser(logFailuresOnly, filterByNamespace)); } - public InliningDiagnoserAttribute(bool logFailuresOnly = true, string[] allowedNamespaces = null) + public InliningDiagnoserAttribute(bool logFailuresOnly = true, string[]? allowedNamespaces = null) { Config = ManualConfig.CreateEmpty().AddDiagnoser(new InliningDiagnoser(logFailuresOnly, allowedNamespaces)); } diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs index c52006d7c6..4205195dc5 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs @@ -8,6 +8,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; using Microsoft.Diagnostics.Tracing; using Microsoft.Diagnostics.Tracing.Parsers; @@ -15,7 +16,7 @@ namespace BenchmarkDotNet.Diagnostics.Windows { - public abstract class EtwDiagnoser where TStats : new() + public abstract class EtwDiagnoser : DisposeAtProcessTermination where TStats : new() { internal readonly LogCapture Logger = new LogCapture(); protected readonly Dictionary BenchmarkToProcess = new Dictionary(); @@ -39,11 +40,6 @@ protected void Start(DiagnoserActionParameters parameters) BenchmarkToProcess.Add(parameters.BenchmarkCase, parameters.Process.Id); StatsPerProcess.TryAdd(parameters.Process.Id, GetInitializedStats(parameters)); - // Important: Must wire-up clean-up events prior to acquiring IDisposable instance (Session property) - // This is in effect the inverted sequence of actions in the Stop() method. - Console.CancelKeyPress += OnConsoleCancelKeyPress; - AppDomain.CurrentDomain.ProcessExit += OnProcessExit; - Session = CreateSession(parameters.BenchmarkCase); EnableProvider(); @@ -80,11 +76,13 @@ protected virtual void EnableProvider() protected void Stop() { WaitForDelayedEvents(); + Dispose(); + } - Session.Dispose(); - - Console.CancelKeyPress -= OnConsoleCancelKeyPress; - AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; + public override void Dispose() + { + Session?.Dispose(); + base.Dispose(); } private void Clear() @@ -93,11 +91,7 @@ private void Clear() StatsPerProcess.Clear(); } - private void OnConsoleCancelKeyPress(object sender, ConsoleCancelEventArgs e) => Session?.Dispose(); - - private void OnProcessExit(object sender, EventArgs e) => Session?.Dispose(); - - private static string GetSessionName(string prefix, BenchmarkCase benchmarkCase, ParameterInstances parameters = null) + private static string GetSessionName(string prefix, BenchmarkCase benchmarkCase, ParameterInstances? parameters = null) { if (parameters != null && parameters.Items.Count > 0) return $"{prefix}-{benchmarkCase.FolderInfo}-{parameters.FolderInfo}"; diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs b/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs index 6641dd5a45..57f57d9e1a 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs @@ -120,21 +120,10 @@ private void Start(DiagnoserActionParameters parameters) private void Stop(DiagnoserActionParameters parameters) { WaitForDelayedEvents(); - string userSessionFile; - try - { - kernelSession.Stop(); - heapSession?.Stop(); - userSession.Stop(); - - userSessionFile = userSession.FilePath; - } - finally - { - kernelSession.Dispose(); - heapSession?.Dispose(); - userSession.Dispose(); - } + string userSessionFile = userSession.FilePath; + kernelSession.Dispose(); + heapSession?.Dispose(); + userSession.Dispose(); // Merge the 'primary' etl file X.etl (userSession) with any files that match .clr*.etl .user*.etl. and .kernel.etl. TraceEventSession.MergeInPlace(userSessionFile, TextWriter.Null); diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfilerConfig.cs b/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfilerConfig.cs index 967dcc2d03..7846edd1dd 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfilerConfig.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfilerConfig.cs @@ -41,8 +41,8 @@ public EtwProfilerConfig( float cpuSampleIntervalInMilliseconds = 1.0f, KernelTraceEventParser.Keywords kernelKeywords = KernelTraceEventParser.Keywords.ImageLoad | KernelTraceEventParser.Keywords.Profile, KernelTraceEventParser.Keywords kernelStackKeywords = KernelTraceEventParser.Keywords.Profile, - IReadOnlyDictionary> intervalSelectors = null, - IReadOnlyCollection<(Guid providerGuid, TraceEventLevel providerLevel, ulong keywords, TraceEventProviderOptions options)> providers = null, + IReadOnlyDictionary>? intervalSelectors = null, + IReadOnlyCollection<(Guid providerGuid, TraceEventLevel providerLevel, ulong keywords, TraceEventProviderOptions options)>? providers = null, bool createHeapSession = false) { CreateHeapSession = createHeapSession; diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/HardwareCounters.cs b/src/BenchmarkDotNet.Diagnostics.Windows/HardwareCounters.cs index 8b3c46b4d5..d159e72a65 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/HardwareCounters.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/HardwareCounters.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Toolchains.InProcess.Emit; @@ -31,7 +32,7 @@ private static readonly Dictionary EtwTranslations public static IEnumerable Validate(ValidationParameters validationParameters, bool mandatory) { - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { yield return new ValidationError(true, "Hardware Counters and EtwProfiler are supported only on Windows"); yield break; diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/InliningDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/InliningDiagnoser.cs index 4c09957eef..a21dd1dc81 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/InliningDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/InliningDiagnoser.cs @@ -13,7 +13,7 @@ public class InliningDiagnoser : JitDiagnoser, IProfiler private readonly bool logFailuresOnly = true; private readonly bool filterByNamespace = true; - private readonly string[] allowedNamespaces = null; + private readonly string[]? allowedNamespaces = null; private string defaultNamespace; // ReSharper disable once EmptyConstructor parameterless ctor is mandatory for DiagnosersLoader.CreateDiagnoser @@ -35,7 +35,7 @@ public InliningDiagnoser(bool logFailuresOnly = true, bool filterByNamespace = t /// /// only the methods that failed to get inlined. True by default. /// list of namespaces from which inlining message should be print. - public InliningDiagnoser(bool logFailuresOnly = true, string[] allowedNamespaces = null) + public InliningDiagnoser(bool logFailuresOnly = true, string[]? allowedNamespaces = null) { this.logFailuresOnly = logFailuresOnly; this.allowedNamespaces = allowedNamespaces; diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs index b189af0acb..0d06faf75b 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Loggers; @@ -33,7 +34,7 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) public IEnumerable Validate(ValidationParameters validationParameters) { - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { yield return new ValidationError(true, $"{GetType().Name} is supported only on Windows"); } diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs index a86989aef5..a101888a0e 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs @@ -6,6 +6,7 @@ using BenchmarkDotNet.Running; using Microsoft.Diagnostics.Tracing.Parsers; using Microsoft.Diagnostics.Tracing.Session; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Diagnostics.Windows { @@ -97,7 +98,7 @@ private sealed class JitAllocatedMemoryDescriptor : IMetricDescriptor public bool TheGreaterTheBetter => false; public string NumberFormat => "N0"; public UnitType UnitType => UnitType.Size; - public string Unit => SizeUnit.B.Name; + public string Unit => SizeUnit.B.Abbreviation; public int PriorityInCategory => 0; public bool GetIsAvailable(Metric metric) => true; } diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Sessions.cs b/src/BenchmarkDotNet.Diagnostics.Windows/Sessions.cs index cf0b6fa88f..f0f5dcc475 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Sessions.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/Sessions.cs @@ -5,13 +5,13 @@ using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Running; using Microsoft.Diagnostics.Tracing; using Microsoft.Diagnostics.Tracing.Parsers; using Microsoft.Diagnostics.Tracing.Session; -using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Diagnostics.Windows { @@ -90,7 +90,7 @@ internal override Session EnableProviders() } } - internal abstract class Session : IDisposable + internal abstract class Session : DisposeAtProcessTermination { private const int MaxSessionNameLength = 128; @@ -114,27 +114,16 @@ protected Session(string sessionName, DiagnoserActionParameters details, EtwProf BufferSizeMB = config.BufferSizeInMb, CpuSampleIntervalMSec = config.CpuSampleIntervalInMilliseconds, }; - - Console.CancelKeyPress += OnConsoleCancelKeyPress; - AppDomain.CurrentDomain.ProcessExit += OnProcessExit; } - public void Dispose() => TraceEventSession.Dispose(); - - internal void Stop() + public override void Dispose() { - TraceEventSession.Stop(); - - Console.CancelKeyPress -= OnConsoleCancelKeyPress; - AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; + TraceEventSession.Dispose(); + base.Dispose(); } internal abstract Session EnableProviders(); - private void OnConsoleCancelKeyPress(object sender, ConsoleCancelEventArgs e) => Stop(); - - private void OnProcessExit(object sender, EventArgs e) => Stop(); - protected static string GetSessionName(BenchmarkCase benchmarkCase) { string benchmarkName = FullNameProvider.GetBenchmarkName(benchmarkCase); diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/EngineEventLogParser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/EngineEventLogParser.cs index c385c1c440..f7e3d8e6fd 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/EngineEventLogParser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/EngineEventLogParser.cs @@ -8,7 +8,7 @@ namespace BenchmarkDotNet.Diagnostics.Windows.Tracing { public sealed class EngineEventLogParser : TraceEventParser { - private static volatile TraceEvent[] templates; + private static volatile TraceEvent[]? templates; public EngineEventLogParser(TraceEventSource source, bool dontRegister = false) : base(source, dontRegister) { } @@ -114,69 +114,69 @@ public event Action WorkloadActualStop protected override string GetProviderName() { return ProviderName; } - private static IterationEvent BenchmarkStartTemplate(Action action) + private static IterationEvent BenchmarkStartTemplate(Action? action) { // action, eventid, taskid, taskName, taskGuid, opcode, opcodeName, providerGuid, providerName return new IterationEvent(action, EngineEventSource.BenchmarkStartEventId, (int)EngineEventSource.Tasks.Benchmark, nameof(EngineEventSource.Tasks.Benchmark), Guid.Empty, (int)EventOpcode.Start, nameof(EventOpcode.Start), ProviderGuid, ProviderName); } - private static IterationEvent BenchmarkStopTemplate(Action action) + private static IterationEvent BenchmarkStopTemplate(Action? action) { // action, eventid, taskid, taskName, taskGuid, opcode, opcodeName, providerGuid, providerName return new IterationEvent(action, EngineEventSource.BenchmarkStopEventId, (int)EngineEventSource.Tasks.Benchmark, nameof(EngineEventSource.Tasks.Benchmark), Guid.Empty, (int)EventOpcode.Stop, nameof(EventOpcode.Stop), ProviderGuid, ProviderName); } - private static IterationEvent OverheadJittingStartTemplate(Action action) + private static IterationEvent OverheadJittingStartTemplate(Action? action) => CreateIterationStartTemplate(action, EngineEventSource.OverheadJittingStartEventId, EngineEventSource.Tasks.OverheadJitting); - private static IterationEvent OverheadJittingStopTemplate(Action action) + private static IterationEvent OverheadJittingStopTemplate(Action? action) => CreateIterationStopTemplate(action, EngineEventSource.OverheadJittingStopEventId, EngineEventSource.Tasks.OverheadJitting); - private static IterationEvent WorkloadJittingStartTemplate(Action action) + private static IterationEvent WorkloadJittingStartTemplate(Action? action) => CreateIterationStartTemplate(action, EngineEventSource.WorkloadJittingStartEventId, EngineEventSource.Tasks.WorkloadJitting); - private static IterationEvent WorkloadJittingStopTemplate(Action action) + private static IterationEvent WorkloadJittingStopTemplate(Action? action) => CreateIterationStopTemplate(action, EngineEventSource.WorkloadJittingStopEventId, EngineEventSource.Tasks.WorkloadJitting); - private static IterationEvent WorkloadPilotStartTemplate(Action action) + private static IterationEvent WorkloadPilotStartTemplate(Action? action) => CreateIterationStartTemplate(action, EngineEventSource.WorkloadPilotStartEventId, EngineEventSource.Tasks.WorkloadPilot); - private static IterationEvent WorkloadPilotStopTemplate(Action action) + private static IterationEvent WorkloadPilotStopTemplate(Action? action) => CreateIterationStopTemplate(action, EngineEventSource.WorkloadPilotStopEventId, EngineEventSource.Tasks.WorkloadPilot); - private static IterationEvent OverheadWarmupStartTemplate(Action action) + private static IterationEvent OverheadWarmupStartTemplate(Action? action) => CreateIterationStartTemplate(action, EngineEventSource.OverheadWarmupStartEventId, EngineEventSource.Tasks.OverheadWarmup); - private static IterationEvent OverheadWarmupStopTemplate(Action action) + private static IterationEvent OverheadWarmupStopTemplate(Action? action) => CreateIterationStopTemplate(action, EngineEventSource.OverheadWarmupStopEventId, EngineEventSource.Tasks.OverheadWarmup); - private static IterationEvent WorkloadWarmupStartTemplate(Action action) + private static IterationEvent WorkloadWarmupStartTemplate(Action? action) => CreateIterationStartTemplate(action, EngineEventSource.WorkloadWarmupStartEventId, EngineEventSource.Tasks.WorkloadWarmup); - private static IterationEvent WorkloadWarmupStopTemplate(Action action) + private static IterationEvent WorkloadWarmupStopTemplate(Action? action) => CreateIterationStopTemplate(action, EngineEventSource.WorkloadWarmupStopEventId, EngineEventSource.Tasks.WorkloadWarmup); - private static IterationEvent OverheadActualStartTemplate(Action action) + private static IterationEvent OverheadActualStartTemplate(Action? action) => CreateIterationStartTemplate(action, EngineEventSource.OverheadActualStartEventId, EngineEventSource.Tasks.OverheadActual); - private static IterationEvent OverheadActualStopTemplate(Action action) + private static IterationEvent OverheadActualStopTemplate(Action? action) => CreateIterationStopTemplate(action, EngineEventSource.OverheadActualStopEventId, EngineEventSource.Tasks.OverheadActual); - private static IterationEvent WorkloadActualStartTemplate(Action action) + private static IterationEvent WorkloadActualStartTemplate(Action? action) => CreateIterationStartTemplate(action, EngineEventSource.WorkloadActualStartEventId, EngineEventSource.Tasks.WorkloadActual); - private static IterationEvent WorkloadActualStopTemplate(Action action) + private static IterationEvent WorkloadActualStopTemplate(Action? action) => CreateIterationStopTemplate(action, EngineEventSource.WorkloadActualStopEventId, EngineEventSource.Tasks.WorkloadActual); - private static IterationEvent CreateIterationStartTemplate(Action action, int eventId, EventTask eventTask) + private static IterationEvent CreateIterationStartTemplate(Action? action, int eventId, EventTask eventTask) { // action, eventid, taskid, taskName, taskGuid, opcode, opcodeName, providerGuid, providerName return new IterationEvent(action, eventId, (int)eventTask, eventTask.ToString(), Guid.Empty, (int)EventOpcode.Start, nameof(EventOpcode.Start), ProviderGuid, ProviderName); } - private static IterationEvent CreateIterationStopTemplate(Action action, int eventId, EventTask eventTask) + private static IterationEvent CreateIterationStopTemplate(Action? action, int eventId, EventTask eventTask) { // action, eventid, taskid, taskName, taskGuid, opcode, opcodeName, providerGuid, providerName return new IterationEvent(action, eventId, (int)eventTask, eventTask.ToString(), Guid.Empty, (int)EventOpcode.Stop, nameof(EventOpcode.Stop), ProviderGuid, ProviderName); } - protected override void EnumerateTemplates(Func eventsToObserve, Action callback) + protected override void EnumerateTemplates(Func? eventsToObserve, Action callback) { if (templates == null) { diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/IterationEvent.cs b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/IterationEvent.cs index 7908f7cf52..3367aa7c60 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/IterationEvent.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/IterationEvent.cs @@ -9,15 +9,15 @@ public sealed class IterationEvent : TraceEvent { public long TotalOperations => GetInt64At(0); - private event Action target; + private event Action? target; - internal IterationEvent(Action target, int eventID, int task, string taskName, Guid taskGuid, int opcode, string opcodeName, Guid providerGuid, string providerName) - : base(eventID, task, taskName, taskGuid, opcode, opcodeName, providerGuid, providerName) + internal IterationEvent(Action? target, int eventId, int task, string taskName, Guid taskGuid, int opcode, string opcodeName, Guid providerGuid, string providerName) + : base(eventId, task, taskName, taskGuid, opcode, opcodeName, providerGuid, providerName) { this.target = target; } - protected override Delegate Target + protected override Delegate? Target { get => target; set => target = (Action)value; @@ -34,7 +34,7 @@ public override StringBuilder ToXml(StringBuilder sb) return sb; } - public override object PayloadValue(int index) => index == 0 ? (object)TotalOperations : null; + public override object? PayloadValue(int index) => index == 0 ? TotalOperations : null; protected override void Dispatch() => target?.Invoke(this); diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs index 9e269f28fb..4d8b79c3d1 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; -using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Extensions; @@ -12,6 +11,7 @@ using Microsoft.Diagnostics.Tracing.Etlx; using Microsoft.Diagnostics.Tracing.Parsers.Kernel; using Microsoft.Diagnostics.Tracing.Stacks; +using Perfolizer.Metrology; using Address = System.UInt64; namespace BenchmarkDotNet.Diagnostics.Windows.Tracing @@ -95,7 +95,7 @@ private IEnumerable Parse(TraceLog traceLog) var heapParser = new HeapTraceProviderTraceEventParser(eventSource); // We index by heap address and then within the heap we remember the allocation stack var heaps = new Dictionary>(); - Dictionary lastHeapAllocs = null; + Dictionary? lastHeapAllocs = null; Address lastHeapHandle = 0; @@ -246,12 +246,14 @@ bool IsCallStackIn(StackSourceCallStackIndex index) var memoryAllocatedPerOperation = totalAllocation / totalOperation; var memoryLeakPerOperation = nativeLeakSize / totalOperation; - logger.WriteLine($"Native memory allocated per single operation: {SizeValue.FromBytes(memoryAllocatedPerOperation).ToString(SizeUnit.B, benchmarkCase.Config.CultureInfo)}"); + logger.WriteLine( + $"Native memory allocated per single operation: {SizeValue.FromBytes(memoryAllocatedPerOperation).ToString(SizeUnit.B, null, benchmarkCase.Config.CultureInfo)}"); logger.WriteLine($"Count of allocated object: {countOfAllocatedObject / totalOperation}"); if (nativeLeakSize != 0) { - logger.WriteLine($"Native memory leak per single operation: {SizeValue.FromBytes(memoryLeakPerOperation).ToString(SizeUnit.B, benchmarkCase.Config.CultureInfo)}"); + logger.WriteLine( + $"Native memory leak per single operation: {SizeValue.FromBytes(memoryLeakPerOperation).ToString(SizeUnit.B, null, benchmarkCase.Config.CultureInfo)}"); } var heapInfoList = heaps.Select(h => new { Address = h.Key, h.Value.Count, types = h.Value.Values }); @@ -267,7 +269,8 @@ bool IsCallStackIn(StackSourceCallStackIndex index) }; } - private static Dictionary CreateHeapCache(Address heapHandle, Dictionary> heaps, ref Dictionary lastHeapAllocs, ref Address lastHeapHandle) + private static Dictionary CreateHeapCache(Address heapHandle, Dictionary> heaps, + ref Dictionary lastHeapAllocs, ref Address lastHeapHandle) { Dictionary ret; @@ -282,4 +285,4 @@ private static Dictionary CreateHeapCache(Address heapHandle, Dic return ret; } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/BenchmarkDotNet.Diagnostics.dotMemory.csproj b/src/BenchmarkDotNet.Diagnostics.dotMemory/BenchmarkDotNet.Diagnostics.dotMemory.csproj new file mode 100644 index 0000000000..baf4c0383e --- /dev/null +++ b/src/BenchmarkDotNet.Diagnostics.dotMemory/BenchmarkDotNet.Diagnostics.dotMemory.csproj @@ -0,0 +1,20 @@ + + + + net6.0;net462;netcoreapp3.1 + $(NoWarn);1591 + BenchmarkDotNet.Diagnostics.dotMemory + BenchmarkDotNet.Diagnostics.dotMemory + BenchmarkDotNet.Diagnostics.dotMemory + enable + + + + + + + + + + + diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoser.cs new file mode 100644 index 0000000000..7616ca468c --- /dev/null +++ b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoser.cs @@ -0,0 +1,126 @@ +using System; +using System.Reflection; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Jobs; +using JetBrains.Profiler.SelfApi; + +namespace BenchmarkDotNet.Diagnostics.dotMemory; + +public class DotMemoryDiagnoser(Uri? nugetUrl = null, string? downloadTo = null) : SnapshotProfilerBase +{ + public override string ShortName => "dotMemory"; + + protected override void InitTool(Progress progress) + { + DotMemory.InitAsync(progress, nugetUrl, NuGetApi.V3, downloadTo).Wait(); + } + + protected override void AttachToCurrentProcess(string snapshotFile) + { + DotMemory.Attach(new DotMemory.Config().SaveToFile(snapshotFile)); + } + + protected override void AttachToProcessByPid(int pid, string snapshotFile) + { + DotMemory.Attach(new DotMemory.Config().ProfileExternalProcess(pid).SaveToFile(snapshotFile)); + } + + protected override void TakeSnapshot() + { + DotMemory.GetSnapshot(); + } + + protected override void Detach() + { + DotMemory.Detach(); + } + + protected override string CreateSnapshotFilePath(DiagnoserActionParameters parameters) + { + return ArtifactFileNameHelper.GetFilePath(parameters, "snapshots", DateTime.Now, "dmw", ".0000".Length); + } + + protected override string GetRunnerPath() + { + var consoleRunnerPackageField = typeof(DotMemory).GetField("ConsoleRunnerPackage", BindingFlags.NonPublic | BindingFlags.Static); + if (consoleRunnerPackageField == null) + throw new InvalidOperationException("Field 'ConsoleRunnerPackage' not found."); + + object? consoleRunnerPackage = consoleRunnerPackageField.GetValue(null); + if (consoleRunnerPackage == null) + throw new InvalidOperationException("Unable to get value of 'ConsoleRunnerPackage'."); + + var consoleRunnerPackageType = consoleRunnerPackage.GetType(); + var getRunnerPathMethod = consoleRunnerPackageType.GetMethod("GetRunnerPath"); + if (getRunnerPathMethod == null) + throw new InvalidOperationException("Method 'GetRunnerPath' not found."); + + string? runnerPath = getRunnerPathMethod.Invoke(consoleRunnerPackage, null) as string; + if (runnerPath == null) + throw new InvalidOperationException("Unable to invoke 'GetRunnerPath'."); + + return runnerPath; + } + + internal override bool IsSupported(RuntimeMoniker runtimeMoniker) + { + switch (runtimeMoniker) + { + case RuntimeMoniker.HostProcess: + case RuntimeMoniker.Net461: + case RuntimeMoniker.Net462: + case RuntimeMoniker.Net47: + case RuntimeMoniker.Net471: + case RuntimeMoniker.Net472: + case RuntimeMoniker.Net48: + case RuntimeMoniker.Net481: + case RuntimeMoniker.Net50: + case RuntimeMoniker.Net60: + case RuntimeMoniker.Net70: + case RuntimeMoniker.Net80: + case RuntimeMoniker.Net90: + case RuntimeMoniker.Net10_0: + return true; + case RuntimeMoniker.NotRecognized: + case RuntimeMoniker.Mono: + case RuntimeMoniker.NativeAot60: + case RuntimeMoniker.NativeAot70: + case RuntimeMoniker.NativeAot80: + case RuntimeMoniker.NativeAot90: + case RuntimeMoniker.NativeAot10_0: + case RuntimeMoniker.Wasm: + case RuntimeMoniker.WasmNet50: + case RuntimeMoniker.WasmNet60: + case RuntimeMoniker.WasmNet70: + case RuntimeMoniker.WasmNet80: + case RuntimeMoniker.WasmNet90: + case RuntimeMoniker.WasmNet10_0: + case RuntimeMoniker.MonoAOTLLVM: + case RuntimeMoniker.MonoAOTLLVMNet60: + case RuntimeMoniker.MonoAOTLLVMNet70: + case RuntimeMoniker.MonoAOTLLVMNet80: + case RuntimeMoniker.MonoAOTLLVMNet90: + case RuntimeMoniker.MonoAOTLLVMNet10_0: + case RuntimeMoniker.Mono60: + case RuntimeMoniker.Mono70: + case RuntimeMoniker.Mono80: + case RuntimeMoniker.Mono90: + case RuntimeMoniker.Mono10_0: +#pragma warning disable CS0618 // Type or member is obsolete + case RuntimeMoniker.NetCoreApp50: +#pragma warning restore CS0618 // Type or member is obsolete + return false; + case RuntimeMoniker.NetCoreApp20: + case RuntimeMoniker.NetCoreApp21: + case RuntimeMoniker.NetCoreApp22: + return OsDetector.IsWindows(); + case RuntimeMoniker.NetCoreApp30: + case RuntimeMoniker.NetCoreApp31: + return OsDetector.IsWindows() || OsDetector.IsLinux(); + default: + throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, $"Runtime moniker {runtimeMoniker} is not supported"); + } + } +} diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoserAttribute.cs b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoserAttribute.cs new file mode 100644 index 0000000000..c0fb55d4f1 --- /dev/null +++ b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoserAttribute.cs @@ -0,0 +1,22 @@ +using System; +using BenchmarkDotNet.Configs; + +namespace BenchmarkDotNet.Diagnostics.dotMemory; + +[AttributeUsage(AttributeTargets.Class)] +public class DotMemoryDiagnoserAttribute : Attribute, IConfigSource +{ + public IConfig Config { get; } + + public DotMemoryDiagnoserAttribute() + { + var diagnoser = new DotMemoryDiagnoser(); + Config = ManualConfig.CreateEmpty().AddDiagnoser(diagnoser); + } + + public DotMemoryDiagnoserAttribute(Uri? nugetUrl, string? downloadTo = null) + { + var diagnoser = new DotMemoryDiagnoser(nugetUrl, downloadTo); + Config = ManualConfig.CreateEmpty().AddDiagnoser(diagnoser); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet.Diagnostics.dotMemory/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..270fdc2c9c --- /dev/null +++ b/src/BenchmarkDotNet.Diagnostics.dotMemory/Properties/AssemblyInfo.cs @@ -0,0 +1,11 @@ +using System; +using System.Runtime.CompilerServices; +using BenchmarkDotNet.Properties; + +[assembly: CLSCompliant(true)] + +#if RELEASE +[assembly: InternalsVisibleTo("BenchmarkDotNet.Tests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] +#else +[assembly: InternalsVisibleTo("BenchmarkDotNet.Tests")] +#endif \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/BenchmarkDotNet.Diagnostics.dotTrace.csproj b/src/BenchmarkDotNet.Diagnostics.dotTrace/BenchmarkDotNet.Diagnostics.dotTrace.csproj index e4037e5bcf..42c838e20f 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/BenchmarkDotNet.Diagnostics.dotTrace.csproj +++ b/src/BenchmarkDotNet.Diagnostics.dotTrace/BenchmarkDotNet.Diagnostics.dotTrace.csproj @@ -6,6 +6,7 @@ BenchmarkDotNet.Diagnostics.dotTrace BenchmarkDotNet.Diagnostics.dotTrace BenchmarkDotNet.Diagnostics.dotTrace + enable @@ -13,7 +14,7 @@ - + diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs index a53c305938..8b5d3a858f 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs @@ -1,146 +1,129 @@ using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using BenchmarkDotNet.Analysers; +using System.Reflection; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains; -using BenchmarkDotNet.Validators; -using RunMode = BenchmarkDotNet.Diagnosers.RunMode; +using JetBrains.Profiler.SelfApi; -namespace BenchmarkDotNet.Diagnostics.dotTrace +namespace BenchmarkDotNet.Diagnostics.dotTrace; + +public class DotTraceDiagnoser(Uri? nugetUrl = null, string? downloadTo = null) : SnapshotProfilerBase { - public class DotTraceDiagnoser : IProfiler + public override string ShortName => "dotTrace"; + + protected override void InitTool(Progress progress) { - private readonly Uri nugetUrl; - private readonly string toolsDownloadFolder; + DotTrace.InitAsync(progress, nugetUrl, NuGetApi.V3, downloadTo).Wait(); + } - public DotTraceDiagnoser(Uri nugetUrl = null, string toolsDownloadFolder = null) - { - this.nugetUrl = nugetUrl; - this.toolsDownloadFolder = toolsDownloadFolder; - } + protected override void AttachToCurrentProcess(string snapshotFile) + { + DotTrace.Attach(new DotTrace.Config().SaveToFile(snapshotFile)); + DotTrace.StartCollectingData(); + } - public IEnumerable Ids => new[] { "DotTrace" }; - public string ShortName => "dotTrace"; + protected override void AttachToProcessByPid(int pid, string snapshotFile) + { + DotTrace.Attach(new DotTrace.Config().ProfileExternalProcess(pid).SaveToFile(snapshotFile)); + DotTrace.StartCollectingData(); + } - public RunMode GetRunMode(BenchmarkCase benchmarkCase) - { - return IsSupported(benchmarkCase.Job.Environment.GetRuntime().RuntimeMoniker) ? RunMode.ExtraRun : RunMode.None; - } + protected override void TakeSnapshot() + { + DotTrace.StopCollectingData(); + DotTrace.SaveData(); + } - private readonly List snapshotFilePaths = new (); + protected override void Detach() + { + DotTrace.Detach(); + } - public void Handle(HostSignal signal, DiagnoserActionParameters parameters) - { - var job = parameters.BenchmarkCase.Job; - bool isInProcess = job.GetToolchain().IsInProcess; - var logger = parameters.Config.GetCompositeLogger(); - DotTraceToolBase tool = isInProcess - ? new InProcessDotTraceTool(logger, nugetUrl, downloadTo: toolsDownloadFolder) - : new ExternalDotTraceTool(logger, nugetUrl, downloadTo: toolsDownloadFolder); + protected override string CreateSnapshotFilePath(DiagnoserActionParameters parameters) + { + return ArtifactFileNameHelper.GetFilePath(parameters, "snapshots", DateTime.Now, "dtp", ".0000".Length); + } - var runtimeMoniker = job.Environment.GetRuntime().RuntimeMoniker; - if (!IsSupported(runtimeMoniker)) - { - logger.WriteLineError($"Runtime '{runtimeMoniker}' is not supported by dotTrace"); - return; - } + protected override string GetRunnerPath() + { + var consoleRunnerPackageField = typeof(DotTrace).GetField("ConsoleRunnerPackage", BindingFlags.NonPublic | BindingFlags.Static); + if (consoleRunnerPackageField == null) + throw new InvalidOperationException("Field 'ConsoleRunnerPackage' not found."); - switch (signal) - { - case HostSignal.BeforeAnythingElse: - tool.Init(parameters); - break; - case HostSignal.BeforeActualRun: - snapshotFilePaths.Add(tool.Start(parameters)); - break; - case HostSignal.AfterActualRun: - tool.Stop(parameters); - break; - } - } + object? consoleRunnerPackage = consoleRunnerPackageField.GetValue(null); + if (consoleRunnerPackage == null) + throw new InvalidOperationException("Unable to get value of 'ConsoleRunnerPackage'."); - public IEnumerable Exporters => Enumerable.Empty(); - public IEnumerable Analysers => Enumerable.Empty(); + var consoleRunnerPackageType = consoleRunnerPackage.GetType(); + var getRunnerPathMethod = consoleRunnerPackageType.GetMethod("GetRunnerPath"); + if (getRunnerPathMethod == null) + throw new InvalidOperationException("Method 'GetRunnerPath' not found."); - public IEnumerable Validate(ValidationParameters validationParameters) - { - var runtimeMonikers = validationParameters.Benchmarks.Select(b => b.Job.Environment.GetRuntime().RuntimeMoniker).Distinct(); - foreach (var runtimeMoniker in runtimeMonikers) - { - if (!IsSupported(runtimeMoniker)) - yield return new ValidationError(true, $"Runtime '{runtimeMoniker}' is not supported by dotTrace"); - } - } + string? runnerPath = getRunnerPathMethod.Invoke(consoleRunnerPackage, null) as string; + if (runnerPath == null) + throw new InvalidOperationException("Unable to invoke 'GetRunnerPath'."); - internal static bool IsSupported(RuntimeMoniker runtimeMoniker) + return runnerPath; + } + + internal override bool IsSupported(RuntimeMoniker runtimeMoniker) + { + switch (runtimeMoniker) { - switch (runtimeMoniker) - { - case RuntimeMoniker.HostProcess: - case RuntimeMoniker.Net461: - case RuntimeMoniker.Net462: - case RuntimeMoniker.Net47: - case RuntimeMoniker.Net471: - case RuntimeMoniker.Net472: - case RuntimeMoniker.Net48: - case RuntimeMoniker.Net481: - case RuntimeMoniker.Net50: - case RuntimeMoniker.Net60: - case RuntimeMoniker.Net70: - case RuntimeMoniker.Net80: - return true; - case RuntimeMoniker.NotRecognized: - case RuntimeMoniker.Mono: - case RuntimeMoniker.NativeAot60: - case RuntimeMoniker.NativeAot70: - case RuntimeMoniker.NativeAot80: - case RuntimeMoniker.Wasm: - case RuntimeMoniker.WasmNet50: - case RuntimeMoniker.WasmNet60: - case RuntimeMoniker.WasmNet70: - case RuntimeMoniker.WasmNet80: - case RuntimeMoniker.MonoAOTLLVM: - case RuntimeMoniker.MonoAOTLLVMNet60: - case RuntimeMoniker.MonoAOTLLVMNet70: - case RuntimeMoniker.MonoAOTLLVMNet80: - case RuntimeMoniker.Mono60: - case RuntimeMoniker.Mono70: - case RuntimeMoniker.Mono80: + case RuntimeMoniker.HostProcess: + case RuntimeMoniker.Net461: + case RuntimeMoniker.Net462: + case RuntimeMoniker.Net47: + case RuntimeMoniker.Net471: + case RuntimeMoniker.Net472: + case RuntimeMoniker.Net48: + case RuntimeMoniker.Net481: + case RuntimeMoniker.Net50: + case RuntimeMoniker.Net60: + case RuntimeMoniker.Net70: + case RuntimeMoniker.Net80: + case RuntimeMoniker.Net90: + case RuntimeMoniker.Net10_0: + return true; + case RuntimeMoniker.NotRecognized: + case RuntimeMoniker.Mono: + case RuntimeMoniker.NativeAot60: + case RuntimeMoniker.NativeAot70: + case RuntimeMoniker.NativeAot80: + case RuntimeMoniker.NativeAot90: + case RuntimeMoniker.NativeAot10_0: + case RuntimeMoniker.Wasm: + case RuntimeMoniker.WasmNet50: + case RuntimeMoniker.WasmNet60: + case RuntimeMoniker.WasmNet70: + case RuntimeMoniker.WasmNet80: + case RuntimeMoniker.WasmNet90: + case RuntimeMoniker.WasmNet10_0: + case RuntimeMoniker.MonoAOTLLVM: + case RuntimeMoniker.MonoAOTLLVMNet60: + case RuntimeMoniker.MonoAOTLLVMNet70: + case RuntimeMoniker.MonoAOTLLVMNet80: + case RuntimeMoniker.MonoAOTLLVMNet90: + case RuntimeMoniker.MonoAOTLLVMNet10_0: + case RuntimeMoniker.Mono60: + case RuntimeMoniker.Mono70: + case RuntimeMoniker.Mono80: + case RuntimeMoniker.Mono90: + case RuntimeMoniker.Mono10_0: #pragma warning disable CS0618 // Type or member is obsolete - case RuntimeMoniker.NetCoreApp50: + case RuntimeMoniker.NetCoreApp50: #pragma warning restore CS0618 // Type or member is obsolete - return false; - case RuntimeMoniker.NetCoreApp20: - case RuntimeMoniker.NetCoreApp21: - case RuntimeMoniker.NetCoreApp22: - return RuntimeInformation.IsWindows(); - case RuntimeMoniker.NetCoreApp30: - case RuntimeMoniker.NetCoreApp31: - return RuntimeInformation.IsWindows() || RuntimeInformation.IsLinux(); - default: - throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, $"Runtime moniker {runtimeMoniker} is not supported"); - } - } - - public IEnumerable ProcessResults(DiagnoserResults results) => ImmutableArray.Empty; - - public void DisplayResults(ILogger logger) - { - if (snapshotFilePaths.Any()) - { - logger.WriteLineInfo("The following dotTrace snapshots were generated:"); - foreach (string snapshotFilePath in snapshotFilePaths) - logger.WriteLineInfo($"* {snapshotFilePath}"); - } + return false; + case RuntimeMoniker.NetCoreApp20: + case RuntimeMoniker.NetCoreApp21: + case RuntimeMoniker.NetCoreApp22: + return OsDetector.IsWindows(); + case RuntimeMoniker.NetCoreApp30: + case RuntimeMoniker.NetCoreApp31: + return OsDetector.IsWindows() || OsDetector.IsLinux(); + default: + throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, $"Runtime moniker {runtimeMoniker} is not supported"); } } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoserAttribute.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoserAttribute.cs index de803e6443..f056a98cbd 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoserAttribute.cs @@ -1,21 +1,22 @@ using System; using BenchmarkDotNet.Configs; -namespace BenchmarkDotNet.Diagnostics.dotTrace +namespace BenchmarkDotNet.Diagnostics.dotTrace; + +[AttributeUsage(AttributeTargets.Class)] +public class DotTraceDiagnoserAttribute : Attribute, IConfigSource { - [AttributeUsage(AttributeTargets.Class)] - public class DotTraceDiagnoserAttribute : Attribute, IConfigSource - { - public IConfig Config { get; } + public IConfig Config { get; } - public DotTraceDiagnoserAttribute() - { - Config = ManualConfig.CreateEmpty().AddDiagnoser(new DotTraceDiagnoser()); - } + public DotTraceDiagnoserAttribute() + { + var diagnoser = new DotTraceDiagnoser(); + Config = ManualConfig.CreateEmpty().AddDiagnoser(diagnoser); + } - public DotTraceDiagnoserAttribute(Uri nugetUrl = null, string toolsDownloadFolder = null) - { - Config = ManualConfig.CreateEmpty().AddDiagnoser(new DotTraceDiagnoser(nugetUrl, toolsDownloadFolder)); - } + public DotTraceDiagnoserAttribute(Uri? nugetUrl, string? downloadTo = null) + { + var diagnoser = new DotTraceDiagnoser(nugetUrl, downloadTo); + Config = ManualConfig.CreateEmpty().AddDiagnoser(diagnoser); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceToolBase.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceToolBase.cs deleted file mode 100644 index c41ffc53e5..0000000000 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceToolBase.cs +++ /dev/null @@ -1,145 +0,0 @@ -using System; -using System.IO; -using System.Reflection; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Loggers; -using JetBrains.Profiler.SelfApi; - -namespace BenchmarkDotNet.Diagnostics.dotTrace -{ - internal abstract class DotTraceToolBase - { - private readonly ILogger logger; - private readonly Uri nugetUrl; - private readonly NuGetApi nugetApi; - private readonly string downloadTo; - - protected DotTraceToolBase(ILogger logger, Uri nugetUrl = null, NuGetApi nugetApi = NuGetApi.V3, string downloadTo = null) - { - this.logger = logger; - this.nugetUrl = nugetUrl; - this.nugetApi = nugetApi; - this.downloadTo = downloadTo; - } - - public void Init(DiagnoserActionParameters parameters) - { - try - { - logger.WriteLineInfo("Ensuring that dotTrace prerequisite is installed..."); - var progress = new Progress(logger, "Installing DotTrace"); - DotTrace.EnsurePrerequisiteAsync(progress, nugetUrl, nugetApi, downloadTo).Wait(); - logger.WriteLineInfo("dotTrace prerequisite is installed"); - logger.WriteLineInfo($"dotTrace runner path: {GetRunnerPath()}"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - } - - protected abstract bool AttachOnly { get; } - protected abstract void Attach(DiagnoserActionParameters parameters, string snapshotFile); - protected abstract void StartCollectingData(); - protected abstract void SaveData(); - protected abstract void Detach(); - - public string Start(DiagnoserActionParameters parameters) - { - string snapshotFile = ArtifactFileNameHelper.GetFilePath(parameters, "snapshots", DateTime.Now, "dtp", ".0000".Length); - string snapshotDirectory = Path.GetDirectoryName(snapshotFile); - logger.WriteLineInfo($"Target snapshot file: {snapshotFile}"); - if (!Directory.Exists(snapshotDirectory)) - { - try - { - Directory.CreateDirectory(snapshotDirectory); - } - catch (Exception e) - { - logger.WriteLineError($"Failed to create directory: {snapshotDirectory}"); - logger.WriteLineError(e.ToString()); - } - } - - try - { - logger.WriteLineInfo("Attaching dotTrace to the process..."); - Attach(parameters, snapshotFile); - logger.WriteLineInfo("dotTrace is successfully attached"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - return snapshotFile; - } - - if (!AttachOnly) - { - try - { - logger.WriteLineInfo("Start collecting data using dataTrace..."); - StartCollectingData(); - logger.WriteLineInfo("Data collecting is successfully started"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - } - - return snapshotFile; - } - - public void Stop(DiagnoserActionParameters parameters) - { - if (!AttachOnly) - { - try - { - logger.WriteLineInfo("Saving dotTrace snapshot..."); - SaveData(); - logger.WriteLineInfo("dotTrace snapshot is successfully saved to the artifact folder"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - - try - { - logger.WriteLineInfo("Detaching dotTrace from the process..."); - Detach(); - logger.WriteLineInfo("dotTrace is successfully detached"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - } - } - - protected string GetRunnerPath() - { - var consoleRunnerPackageField = typeof(DotTrace).GetField("ConsoleRunnerPackage", BindingFlags.NonPublic | BindingFlags.Static); - if (consoleRunnerPackageField == null) - throw new InvalidOperationException("Field 'ConsoleRunnerPackage' not found."); - - object consoleRunnerPackage = consoleRunnerPackageField.GetValue(null); - if (consoleRunnerPackage == null) - throw new InvalidOperationException("Unable to get value of 'ConsoleRunnerPackage'."); - - var consoleRunnerPackageType = consoleRunnerPackage.GetType(); - var getRunnerPathMethod = consoleRunnerPackageType.GetMethod("GetRunnerPath"); - if (getRunnerPathMethod == null) - throw new InvalidOperationException("Method 'GetRunnerPath' not found."); - - string runnerPath = getRunnerPathMethod.Invoke(consoleRunnerPackage, null) as string; - if (runnerPath == null) - throw new InvalidOperationException("Unable to invoke 'GetRunnerPath'."); - - return runnerPath; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/ExternalDotTraceTool.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/ExternalDotTraceTool.cs deleted file mode 100644 index 9c5161b309..0000000000 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/ExternalDotTraceTool.cs +++ /dev/null @@ -1,84 +0,0 @@ -using System; -using System.Diagnostics; -using System.Threading.Tasks; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Loggers; -using JetBrains.Profiler.SelfApi; -using ILogger = BenchmarkDotNet.Loggers.ILogger; - -namespace BenchmarkDotNet.Diagnostics.dotTrace -{ - internal class ExternalDotTraceTool : DotTraceToolBase - { - private static readonly TimeSpan AttachTimeout = TimeSpan.FromMinutes(5); - - public ExternalDotTraceTool(ILogger logger, Uri nugetUrl = null, NuGetApi nugetApi = NuGetApi.V3, string downloadTo = null) : - base(logger, nugetUrl, nugetApi, downloadTo) { } - - protected override bool AttachOnly => true; - - protected override void Attach(DiagnoserActionParameters parameters, string snapshotFile) - { - var logger = parameters.Config.GetCompositeLogger(); - - string runnerPath = GetRunnerPath(); - int pid = parameters.Process.Id; - string arguments = $"attach {pid} --save-to=\"{snapshotFile}\" --service-output=on"; - - logger.WriteLineInfo($"Starting process: '{runnerPath} {arguments}'"); - - var processStartInfo = new ProcessStartInfo - { - FileName = runnerPath, - WorkingDirectory = "", - Arguments = arguments, - UseShellExecute = false, - CreateNoWindow = true, - RedirectStandardOutput = true, - RedirectStandardError = true - }; - - var attachWaitingTask = new TaskCompletionSource(); - var process = new Process { StartInfo = processStartInfo }; - try - { - process.OutputDataReceived += (_, args) => - { - string content = args.Data; - if (content != null) - { - logger.WriteLineInfo("[dotTrace] " + content); - if (content.Contains("##dotTrace[\"started\"")) - attachWaitingTask.TrySetResult(true); - } - }; - process.ErrorDataReceived += (_, args) => - { - string content = args.Data; - if (content != null) - logger.WriteLineError("[dotTrace] " + args.Data); - }; - process.Exited += (_, _) => { attachWaitingTask.TrySetResult(false); }; - process.Start(); - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - } - catch (Exception e) - { - attachWaitingTask.TrySetResult(false); - logger.WriteLineError(e.ToString()); - } - - if (!attachWaitingTask.Task.Wait(AttachTimeout)) - throw new Exception($"Failed to attach dotTrace to the target process (timeout: {AttachTimeout.TotalSeconds} sec"); - if (!attachWaitingTask.Task.Result) - throw new Exception($"Failed to attach dotTrace to the target process (ExitCode={process.ExitCode})"); - } - - protected override void StartCollectingData() { } - - protected override void SaveData() { } - - protected override void Detach() { } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/InProcessDotTraceTool.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/InProcessDotTraceTool.cs deleted file mode 100644 index a124e3e495..0000000000 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/InProcessDotTraceTool.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Loggers; -using JetBrains.Profiler.SelfApi; - -namespace BenchmarkDotNet.Diagnostics.dotTrace -{ - internal class InProcessDotTraceTool : DotTraceToolBase - { - public InProcessDotTraceTool(ILogger logger, Uri nugetUrl = null, NuGetApi nugetApi = NuGetApi.V3, string downloadTo = null) : - base(logger, nugetUrl, nugetApi, downloadTo) { } - - protected override bool AttachOnly => false; - - protected override void Attach(DiagnoserActionParameters parameters, string snapshotFile) - { - var config = new DotTrace.Config(); - config.SaveToFile(snapshotFile); - DotTrace.Attach(config); - } - - protected override void StartCollectingData() => DotTrace.StartCollectingData(); - - protected override void SaveData() => DotTrace.SaveData(); - - protected override void Detach() => DotTrace.Detach(); - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/Progress.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/Progress.cs deleted file mode 100644 index 1d8249f31e..0000000000 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/Progress.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System; -using System.Diagnostics; -using BenchmarkDotNet.Loggers; - -namespace BenchmarkDotNet.Diagnostics.dotTrace -{ - public class Progress : IProgress - { - private static readonly TimeSpan ReportInterval = TimeSpan.FromSeconds(0.1); - - private readonly ILogger logger; - private readonly string title; - - public Progress(ILogger logger, string title) - { - this.logger = logger; - this.title = title; - } - - private int lastProgress; - private Stopwatch stopwatch; - - public void Report(double value) - { - int progress = (int)Math.Floor(value); - bool needToReport = stopwatch == null || - (stopwatch != null && stopwatch?.Elapsed > ReportInterval) || - progress == 100; - - if (lastProgress != progress && needToReport) - { - logger.WriteLineInfo($"{title}: {progress}%"); - lastProgress = progress; - stopwatch = Stopwatch.StartNew(); - } - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs b/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs index 29ee184df3..734aa470e4 100644 --- a/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs +++ b/src/BenchmarkDotNet.Disassembler.x64/ClrMdV1Disassembler.cs @@ -93,6 +93,7 @@ private static DisassembledMethod[] Disassemble(Settings settings, State state) { var result = new List(); + using var sourceCodeProvider = new SourceCodeProvider(); while (state.Todo.Count != 0) { var methodInfo = state.Todo.Dequeue(); @@ -101,7 +102,7 @@ private static DisassembledMethod[] Disassemble(Settings settings, State state) continue; // already handled if (settings.MaxDepth >= methodInfo.Depth) - result.Add(DisassembleMethod(methodInfo, state, settings)); + result.Add(DisassembleMethod(methodInfo, state, settings, sourceCodeProvider)); } return result.ToArray(); @@ -110,7 +111,7 @@ private static DisassembledMethod[] Disassemble(Settings settings, State state) private static bool CanBeDisassembled(ClrMethod method) => !((method.ILOffsetMap is null || method.ILOffsetMap.Length == 0) && (method.HotColdInfo is null || method.HotColdInfo.HotStart == 0 || method.HotColdInfo.HotSize == 0)); - private static DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, Settings settings) + private static DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, Settings settings, SourceCodeProvider sourceCodeProvider) { var method = methodInfo.Method; @@ -133,7 +134,7 @@ private static DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State var uniqueSourceCodeLines = new HashSet(new SharpComparer()); // for getting C# code we always use the original ILOffsetMap foreach (var map in method.ILOffsetMap.Where(map => map.StartAddress < map.EndAddress && map.ILOffset >= 0).OrderBy(map => map.StartAddress)) - foreach (var sharp in SourceCodeProvider.GetSource(method, map)) + foreach (var sharp in sourceCodeProvider.GetSource(method, map)) uniqueSourceCodeLines.Add(sharp); codes.AddRange(uniqueSourceCodeLines); diff --git a/src/BenchmarkDotNet.Disassembler.x64/SourceCodeProvider.cs b/src/BenchmarkDotNet.Disassembler.x64/SourceCodeProvider.cs index 7cdb00a6d9..f827369775 100644 --- a/src/BenchmarkDotNet.Disassembler.x64/SourceCodeProvider.cs +++ b/src/BenchmarkDotNet.Disassembler.x64/SourceCodeProvider.cs @@ -8,13 +8,34 @@ namespace BenchmarkDotNet.Disassemblers { - internal static class SourceCodeProvider + // This is taken from the Samples\FileAndLineNumbers projects from microsoft/clrmd, + // and replaces the previously-available SourceLocation functionality. + + internal class SourceLocation { - private static readonly Dictionary SourceFileCache = new Dictionary(); + public string FilePath; + public int LineNumber; + public int LineNumberEnd; + public int ColStart; + public int ColEnd; + } + + internal class SourceCodeProvider : IDisposable + { + private readonly Dictionary sourceFileCache = new Dictionary(); + private readonly Dictionary pdbReaders = new Dictionary(); + + public void Dispose() + { + foreach (var reader in pdbReaders.Values) + { + reader?.Dispose(); + } + } - internal static IEnumerable GetSource(ClrMethod method, ILToNativeMap map) + internal IEnumerable GetSource(ClrMethod method, ILToNativeMap map) { - var sourceLocation = method.GetSourceLocation(map.ILOffset); + var sourceLocation = GetSourceLocation(method, map.ILOffset); if (sourceLocation == null) yield break; @@ -39,16 +60,16 @@ internal static IEnumerable GetSource(ClrMethod method, ILToNativeMap map } } - private static string ReadSourceLine(string file, int line) + private string ReadSourceLine(string file, int line) { - if (!SourceFileCache.TryGetValue(file, out string[] contents)) + if (!sourceFileCache.TryGetValue(file, out string[] contents)) { // sometimes the symbols report some disk location from MS CI machine like "E:\A\_work\308\s\src\mscorlib\shared\System\Random.cs" for .NET Core 2.0 if (!File.Exists(file)) return null; contents = File.ReadAllLines(file); - SourceFileCache.Add(file, contents); + sourceFileCache.Add(file, contents); } return line - 1 < contents.Length @@ -84,29 +105,8 @@ private static string GetSmartPointer(string sourceLine, int? start, int? end) return new string(prefix); } - } - - - // This is taken from the Samples\FileAndLineNumbers projects from microsoft/clrmd, - // and replaces the previously-available SourceLocation functionality. - - internal class SourceLocation - { - public string FilePath; - public int LineNumber; - public int LineNumberEnd; - public int ColStart; - public int ColEnd; - } - - internal static class ClrSourceExtensions - { - // TODO Not sure we want this to be a shared dictionary, especially without - // any synchronization. Probably want to put this hanging off the Context - // somewhere, or inside SymbolCache. - private static readonly Dictionary s_pdbReaders = new Dictionary(); - internal static SourceLocation GetSourceLocation(this ClrMethod method, int ilOffset) + internal SourceLocation GetSourceLocation(ClrMethod method, int ilOffset) { PdbReader reader = GetReaderForMethod(method); if (reader == null) @@ -116,7 +116,7 @@ internal static SourceLocation GetSourceLocation(this ClrMethod method, int ilOf return FindNearestLine(function, ilOffset); } - internal static SourceLocation GetSourceLocation(this ClrStackFrame frame) + internal SourceLocation GetSourceLocation(ClrStackFrame frame) { PdbReader reader = GetReaderForMethod(frame.Method); if (reader == null) @@ -134,7 +134,7 @@ private static SourceLocation FindNearestLine(PdbFunction function, int ilOffset return null; int distance = int.MaxValue; - SourceLocation nearest = null; + SourceLocation? nearest = null; foreach (PdbSequencePointCollection sequenceCollection in function.SequencePoints) { @@ -178,15 +178,15 @@ private static int FindIlOffset(ClrStackFrame frame) return last; } - private static PdbReader GetReaderForMethod(ClrMethod method) + private PdbReader GetReaderForMethod(ClrMethod method) { ClrModule module = method?.Type?.Module; PdbInfo info = module?.Pdb; - PdbReader reader = null; + PdbReader? reader = null; if (info != null) { - if (!s_pdbReaders.TryGetValue(info, out reader)) + if (!pdbReaders.TryGetValue(info, out reader)) { SymbolLocator locator = GetSymbolLocator(module); string pdbPath = locator.FindPdb(info); @@ -207,7 +207,7 @@ private static PdbReader GetReaderForMethod(ClrMethod method) } } - s_pdbReaders[info] = reader; + pdbReaders[info] = reader; } } diff --git a/src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj b/src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj new file mode 100644 index 0000000000..5a53f980fb --- /dev/null +++ b/src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj @@ -0,0 +1,19 @@ + + + + BenchmarkDotNet plotting export support. + netstandard2.0 + BenchmarkDotNet.Exporters.Plotting + BenchmarkDotNet.Exporters.Plotting + BenchmarkDotNet.Exporters.Plotting + + True + enable + + + + + + + + diff --git a/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs b/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs new file mode 100644 index 0000000000..77695d4ae7 --- /dev/null +++ b/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs @@ -0,0 +1,438 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Properties; +using BenchmarkDotNet.Reports; +using ScottPlot; +using ScottPlot.Plottables; + +namespace BenchmarkDotNet.Exporters.Plotting +{ + /// + /// Provides plot exports as .png files. + /// + public class ScottPlotExporter : IExporter + { + /// + /// Default instance of the exporter with default configuration. + /// + public static readonly IExporter Default = new ScottPlotExporter(); + + /// + /// Gets the name of the Exporter type. + /// + public string Name => nameof(ScottPlotExporter); + + /// + /// Initializes a new instance of ScottPlotExporter. + /// + /// The width of all plots in pixels (optional). Defaults to 1920. + /// The height of all plots in pixels (optional). Defaults to 1080. + public ScottPlotExporter(int width = 1920, int height = 1080) + { + this.Width = width; + this.Height = height; + this.IncludeBarPlot = true; + this.IncludeBoxPlot = true; + this.RotateLabels = true; + } + + /// + /// Gets or sets the width of all plots in pixels. + /// + public int Width { get; set; } + + /// + /// Gets or sets the height of all plots in pixels. + /// + public int Height { get; set; } + + /// + /// Gets or sets the common font size for ticks, labels etc. (defaults to 14). + /// + public int FontSize { get; set; } = 14; + + /// + /// Gets or sets the font size for the chart title. (defaults to 28). + /// + public int TitleFontSize { get; set; } = 28; + + /// + /// Gets or sets a value indicating whether labels for Plot X-axis should be rotated. + /// This allows for longer labels at the expense of chart height. + /// + public bool RotateLabels { get; set; } + + /// + /// Gets or sets a value indicating whether a bar plot for time-per-op + /// measurement values should be exported. + /// + public bool IncludeBarPlot { get; set; } + + /// + /// Gets or sets a value indicating whether a box plot or whisker plot for time-per-op + /// measurement values should be exported. + /// + public bool IncludeBoxPlot { get; set; } + + /// + /// Not supported. + /// + /// This parameter is not used. + /// This parameter is not used. + /// + public void ExportToLog(Summary summary, ILogger logger) + { + throw new NotSupportedException(); + } + + /// + /// Exports plots to .png file. + /// + /// The summary to be exported. + /// Logger to output to. + /// The file paths of every plot exported. + public IEnumerable ExportToFiles(Summary summary, ILogger consoleLogger) + { + var title = summary.Title; + var version = BenchmarkDotNetInfo.Instance.BrandTitle; + var annotations = GetAnnotations(version); + + var (timeUnit, timeScale) = GetTimeUnit(summary.Reports + .SelectMany(m => m.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Result)))); + + foreach (var benchmark in summary.Reports.GroupBy(r => r.BenchmarkCase.Descriptor.Type.Name)) + { + var benchmarkName = benchmark.Key; + + // Get the measurement nanoseconds per op, divided by time scale, grouped by target and Job [param]. + var timeStats = from report in benchmark + let jobId = report.BenchmarkCase.DisplayInfo.Replace(report.BenchmarkCase.Descriptor.DisplayInfo + ": ", string.Empty) + from measurement in report.AllMeasurements + where measurement.Is(IterationMode.Workload, IterationStage.Result) + let measurementValue = measurement.Nanoseconds / measurement.Operations + group measurementValue / timeScale by (Target: report.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo, JobId: jobId) into g + select new ChartStats(g.Key.Target, g.Key.JobId, g.ToList()); + + if (this.IncludeBarPlot) + { + // -barplot.png + yield return CreateBarPlot( + $"{title} - {benchmarkName}", + Path.Combine(summary.ResultsDirectoryPath, $"{title}-{benchmarkName}-barplot.png"), + $"Time ({timeUnit})", + "Target", + timeStats, + annotations); + } + + if (this.IncludeBoxPlot) + { + // -boxplot.png + yield return CreateBoxPlot( + $"{title} - {benchmarkName}", + Path.Combine(summary.ResultsDirectoryPath, $"{title}-{benchmarkName}-boxplot.png"), + $"Time ({timeUnit})", + "Target", + timeStats, + annotations); + } + + /* TODO: Rest of the RPlotExporter plots. + --density.png + --facetTimeline.png + --facetTimelineSmooth.png + ---timelineSmooth.png + ---timelineSmooth.png*/ + } + } + + /// + /// Calculate Standard Deviation. + /// + /// Values to calculate from. + /// Standard deviation of values. + private static double StandardError(IReadOnlyList values) + { + double average = values.Average(); + double sumOfSquaresOfDifferences = values.Select(val => (val - average) * (val - average)).Sum(); + double standardDeviation = Math.Sqrt(sumOfSquaresOfDifferences / values.Count); + return standardDeviation / Math.Sqrt(values.Count); + } + + /// + /// Gets the lowest appropriate time scale across all measurements. + /// + /// All measurements + /// A unit and scaling factor to convert from nanoseconds. + private (string Unit, double ScaleFactor) GetTimeUnit(IEnumerable values) + { + var minValue = values.Select(m => m.Nanoseconds / m.Operations).DefaultIfEmpty(0d).Min(); + if (minValue > 1000000000d) + { + return ("sec", 1000000000d); + } + + if (minValue > 1000000d) + { + return ("ms", 1000000d); + } + + if (minValue > 1000d) + { + return ("us", 1000d); + } + + return ("ns", 1d); + } + + private string CreateBarPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable data, IReadOnlyList annotations) + { + Plot plt = new Plot(); + plt.Title(title, this.TitleFontSize); + plt.YLabel(yLabel, this.FontSize); + plt.XLabel(xLabel, this.FontSize); + + var palette = new ScottPlot.Palettes.Category10(); + + var legendPalette = data.Select(d => d.JobId) + .Distinct() + .Select((jobId, index) => (jobId, index)) + .ToDictionary(t => t.jobId, t => palette.GetColor(t.index)); + + plt.Legend.IsVisible = true; + plt.Legend.Alignment = Alignment.UpperRight; + plt.Legend.FontSize = this.FontSize; + var legend = data.Select(d => d.JobId) + .Distinct() + .Select((label, index) => new LegendItem() + { + LabelText = label, + FillColor = legendPalette[label] + }) + .ToList(); + + plt.Legend.ManualItems.AddRange(legend); + + var jobCount = plt.Legend.ManualItems.Count; + var ticks = data + .Select((d, index) => new Tick(index, d.Target)) + .ToArray(); + + plt.Axes.Left.TickLabelStyle.FontSize = this.FontSize; + plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks); + plt.Axes.Bottom.MajorTickStyle.Length = 0; + plt.Axes.Bottom.TickLabelStyle.FontSize = this.FontSize; + + if (this.RotateLabels) + { + plt.Axes.Bottom.TickLabelStyle.Rotation = 45; + plt.Axes.Bottom.TickLabelStyle.Alignment = Alignment.MiddleLeft; + + // determine the width of the largest tick label + float largestLabelWidth = 0; + foreach (Tick tick in ticks) + { + PixelSize size = plt.Axes.Bottom.TickLabelStyle.Measure(tick.Label).Size; + largestLabelWidth = Math.Max(largestLabelWidth, size.Width); + } + + // ensure axis panels do not get smaller than the largest label + plt.Axes.Bottom.MinimumSize = largestLabelWidth * 2; + plt.Axes.Right.MinimumSize = largestLabelWidth; + } + + var bars = data + .Select((d, index) => new Bar() + { + Position = ticks[index].Position, + Value = d.Mean, + Error = d.StdError, + FillColor = legendPalette[d.JobId] + }); + plt.Add.Bars(bars.ToList()); + + // Tell the plot to autoscale with no padding beneath the bars + plt.Axes.Margins(bottom: 0, right: .2); + + plt.PlottableList.AddRange(annotations); + + plt.SavePng(fileName, this.Width, this.Height); + return Path.GetFullPath(fileName); + } + + private string CreateBoxPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable data, IReadOnlyList annotations) + { + Plot plt = new Plot(); + plt.Title(title, this.TitleFontSize); + plt.YLabel(yLabel, this.FontSize); + plt.XLabel(xLabel, this.FontSize); + + var palette = new ScottPlot.Palettes.Category10(); + + var legendPalette = data.Select(d => d.JobId) + .Distinct() + .Select((jobId, index) => (jobId, index)) + .ToDictionary(t => t.jobId, t => palette.GetColor(t.index)); + + plt.Legend.IsVisible = true; + plt.Legend.Alignment = Alignment.UpperRight; + plt.Legend.FontSize = this.FontSize; + var legend = data.Select(d => d.JobId) + .Distinct() + .Select((label, index) => new LegendItem() + { + LabelText = label, + FillColor = legendPalette[label] + }) + .ToList(); + + plt.Legend.ManualItems.AddRange(legend); + + var jobCount = plt.Legend.ManualItems.Count; + var ticks = data + .Select((d, index) => new Tick(index, d.Target)) + .ToArray(); + + plt.Axes.Left.TickLabelStyle.FontSize = this.FontSize; + plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks); + plt.Axes.Bottom.MajorTickStyle.Length = 0; + plt.Axes.Bottom.TickLabelStyle.FontSize = this.FontSize; + + if (this.RotateLabels) + { + plt.Axes.Bottom.TickLabelStyle.Rotation = 45; + plt.Axes.Bottom.TickLabelStyle.Alignment = Alignment.MiddleLeft; + + // determine the width of the largest tick label + float largestLabelWidth = 0; + foreach (Tick tick in ticks) + { + PixelSize size = plt.Axes.Bottom.TickLabelStyle.Measure(tick.Label).Size; + largestLabelWidth = Math.Max(largestLabelWidth, size.Width); + } + + // ensure axis panels do not get smaller than the largest label + plt.Axes.Bottom.MinimumSize = largestLabelWidth * 2; + plt.Axes.Right.MinimumSize = largestLabelWidth; + } + + int globalIndex = 0; + foreach (var (targetGroup, targetGroupIndex) in data.GroupBy(s => s.Target).Select((targetGroup, index) => (targetGroup, index))) + { + var boxes = targetGroup.Select(job => (job.JobId, Stats: job.CalculateBoxPlotStatistics())).Select((j, jobIndex) => new Box() + { + Position = ticks[globalIndex++].Position, + FillStyle = new FillStyle() { Color = legendPalette[j.JobId] }, + LineStyle = new LineStyle() { Color = Colors.Black }, + BoxMin = j.Stats.Q1, + BoxMax = j.Stats.Q3, + WhiskerMin = j.Stats.Min, + WhiskerMax = j.Stats.Max, + BoxMiddle = j.Stats.Median + }) + .ToList(); + plt.Add.Boxes(boxes); + } + + // Tell the plot to autoscale with a small padding below the boxes. + plt.Axes.Margins(bottom: 0.05, right: .2); + + plt.PlottableList.AddRange(annotations); + + plt.SavePng(fileName, this.Width, this.Height); + return Path.GetFullPath(fileName); + } + + /// + /// Provides a list of annotations to put over the data area. + /// + /// The version to be displayed. + /// A list of annotations for every plot. + private IReadOnlyList GetAnnotations(string version) + { + var versionAnnotation = new Annotation() + { + LabelStyle = + { + Text = version, + FontSize = 14, + ForeColor = new Color(0, 0, 0, 100) + }, + OffsetY = 10, + OffsetX = 20, + Alignment = Alignment.LowerRight + }; + + + return new[] { versionAnnotation }; + } + + private class ChartStats + { + public ChartStats(string Target, string JobId, IReadOnlyList Values) + { + this.Target = Target; + this.JobId = JobId; + this.Values = Values; + } + + public string Target { get; } + + public string JobId { get; } + + public IReadOnlyList Values { get; } + + public double Min => this.Values.DefaultIfEmpty(0d).Min(); + + public double Max => this.Values.DefaultIfEmpty(0d).Max(); + + public double Mean => this.Values.DefaultIfEmpty(0d).Average(); + + public double StdError => StandardError(this.Values); + + + private static (int MidPoint, double Median) CalculateMedian(ReadOnlySpan values) + { + int n = values.Length; + var midPoint = n / 2; + + // Check if count is even, if so use average of the two middle values, + // otherwise take the middle value. + var median = n % 2 == 0 ? (values[midPoint - 1] + values[midPoint]) / 2d : values[midPoint]; + return (midPoint, median); + } + + /// + /// Calculate the mid points. + /// + /// + public (double Min, double Q1, double Median, double Q3, double Max, double[] Outliers) CalculateBoxPlotStatistics() + { + var values = this.Values.ToArray(); + Array.Sort(values); + var s = values.AsSpan(); + var (midPoint, median) = CalculateMedian(s); + + var (q1Index, q1) = midPoint > 0 ? CalculateMedian(s.Slice(0, midPoint)) : (midPoint, median); + var (q3Index, q3) = midPoint + 1 < s.Length ? CalculateMedian(s.Slice(midPoint + 1)) : (midPoint, median); + var iqr = q3 - q1; + var lowerFence = q1 - 1.5d * iqr; + var upperFence = q3 + 1.5d * iqr; + var outliers = values.Where(v => v < lowerFence || v > upperFence).ToArray(); + var nonOutliers = values.Where(v => v >= lowerFence && v <= upperFence).ToArray(); + return ( + nonOutliers.FirstOrDefault(), + q1, + median, + q3, + nonOutliers.LastOrDefault(), + outliers + ); + } + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkCaseExtensions.cs b/src/BenchmarkDotNet.TestAdapter/BenchmarkCaseExtensions.cs new file mode 100644 index 0000000000..82e6d4b6ec --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkCaseExtensions.cs @@ -0,0 +1,96 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Running; +using Microsoft.TestPlatform.AdapterUtilities; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using System; + +namespace BenchmarkDotNet.TestAdapter +{ + /// + /// A set of extensions for BenchmarkCase to support converting to VSTest TestCase objects. + /// + internal static class BenchmarkCaseExtensions + { + /// + /// Converts a BDN BenchmarkCase to a VSTest TestCase. + /// + /// The BenchmarkCase to convert. + /// The dll or exe of the benchmark project. + /// Whether or not the display name should include the job name. + /// The VSTest TestCase. + internal static TestCase ToVsTestCase(this BenchmarkCase benchmarkCase, string assemblyPath, bool includeJobInName = false) + { + var benchmarkMethod = benchmarkCase.Descriptor.WorkloadMethod; + var fullClassName = benchmarkCase.Descriptor.Type.GetCorrectCSharpTypeName(); + var parametrizedMethodName = FullNameProvider.GetMethodName(benchmarkCase); + + var displayJobInfo = benchmarkCase.GetUnrandomizedJobDisplayInfo(); + var displayMethodName = parametrizedMethodName + (includeJobInName ? $" [{displayJobInfo}]" : ""); + var displayName = $"{fullClassName}.{displayMethodName}"; + + // We use displayName as FQN to workaround the Rider/R# problem with FQNs processing + // See: https://github.com/dotnet/BenchmarkDotNet/issues/2494 + var fullyQualifiedName = displayName; + + var vsTestCase = new TestCase(fullyQualifiedName, VsTestAdapter.ExecutorUri, assemblyPath) + { + DisplayName = displayName, + Id = GetTestCaseId(benchmarkCase) + }; + + var benchmarkAttribute = benchmarkMethod.ResolveAttribute(); + if (benchmarkAttribute != null) + { + vsTestCase.CodeFilePath = benchmarkAttribute.SourceCodeFile; + vsTestCase.LineNumber = benchmarkAttribute.SourceCodeLineNumber; + } + + var categories = DefaultCategoryDiscoverer.Instance.GetCategories(benchmarkMethod); + foreach (var category in categories) + vsTestCase.Traits.Add("Category", category); + + vsTestCase.Traits.Add("", "BenchmarkDotNet"); + + return vsTestCase; + } + + /// + /// If an ID is not provided, a random string is used for the ID. This method will identify if randomness was + /// used for the ID and return the Job's DisplayInfo with that randomness removed so that the same benchmark + /// can be referenced across multiple processes. + /// + /// The benchmark case. + /// The benchmark case' job's DisplayInfo without randomness. + internal static string GetUnrandomizedJobDisplayInfo(this BenchmarkCase benchmarkCase) + { + var jobDisplayInfo = benchmarkCase.Job.DisplayInfo; + if (!benchmarkCase.Job.HasValue(CharacteristicObject.IdCharacteristic) && + benchmarkCase.Job.ResolvedId.StartsWith("Job-", StringComparison.OrdinalIgnoreCase)) + { + // Replace Job-ABCDEF with Job + jobDisplayInfo = "Job" + jobDisplayInfo.Substring(benchmarkCase.Job.ResolvedId.Length); + } + + return jobDisplayInfo; + } + + /// + /// Gets an ID for a given BenchmarkCase that is uniquely identifiable from discovery to execution phase. + /// + /// The benchmark case. + /// The test case ID. + internal static Guid GetTestCaseId(this BenchmarkCase benchmarkCase) + { + var testIdProvider = new TestIdProvider(); + testIdProvider.AppendString(VsTestAdapter.ExecutorUriString); + testIdProvider.AppendString(benchmarkCase.Descriptor.Type.Namespace ?? string.Empty); + testIdProvider.AppendString(benchmarkCase.Descriptor.DisplayInfo); + testIdProvider.AppendString(benchmarkCase.GetUnrandomizedJobDisplayInfo()); + testIdProvider.AppendString(benchmarkCase.Parameters.DisplayInfo); + return testIdProvider.GetId(); + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj b/src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj new file mode 100644 index 0000000000..bd6c053bab --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj @@ -0,0 +1,28 @@ + + + + netstandard2.0;net462 + BenchmarkDotNet.TestAdapter + BenchmarkDotNet.TestAdapter + BenchmarkDotNet.TestAdapter + True + enable + + + + + + + + + + + + + + + + + + + diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs b/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs new file mode 100644 index 0000000000..daf3e2222e --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs @@ -0,0 +1,48 @@ +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +namespace BenchmarkDotNet.TestAdapter +{ + /// + /// A class used for enumerating all the benchmarks in an assembly. + /// + internal static class BenchmarkEnumerator + { + /// + /// Returns all the BenchmarkRunInfo objects from a given assembly. + /// + /// The dll or exe of the benchmark project. + /// The benchmarks inside the assembly. + public static BenchmarkRunInfo[] GetBenchmarksFromAssemblyPath(string assemblyPath) + { + var assembly = Assembly.LoadFrom(assemblyPath); + + var isDebugAssembly = assembly.IsJitOptimizationDisabled() ?? false; + + return GenericBenchmarksBuilder.GetRunnableBenchmarks(assembly.GetRunnableBenchmarks()) + .Select(type => + { + var benchmarkRunInfo = BenchmarkConverter.TypeToBenchmarks(type); + if (isDebugAssembly) + { + // If the assembly is a debug assembly, then only display them if they will run in-process + // This will allow people to debug their benchmarks using VSTest if they wish. + benchmarkRunInfo = new BenchmarkRunInfo( + benchmarkRunInfo.BenchmarksCases.Where(c => c.GetToolchain().IsInProcess).ToArray(), + benchmarkRunInfo.Type, + benchmarkRunInfo.Config); + } + + return benchmarkRunInfo; + }) + .Where(runInfo => runInfo.BenchmarksCases.Length > 0) + .ToArray(); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs b/src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs new file mode 100644 index 0000000000..f1cd64f34e --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs @@ -0,0 +1,86 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.TestAdapter.Remoting; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; + +namespace BenchmarkDotNet.TestAdapter +{ + /// + /// A class used for executing benchmarks + /// + internal class BenchmarkExecutor + { + private readonly CancellationTokenSource cts = new (); + + /// + /// Runs all the benchmarks in the given assembly, updating the TestExecutionRecorder as they get run. + /// + /// The dll or exe of the benchmark project. + /// The interface used to record the current test execution progress. + /// + /// An optional list of benchmark IDs specifying which benchmarks to run. + /// These IDs are the same as the ones generated for the VSTest TestCase. + /// + public void RunBenchmarks(string assemblyPath, TestExecutionRecorderWrapper recorder, HashSet? benchmarkIds = null) + { + var benchmarks = BenchmarkEnumerator.GetBenchmarksFromAssemblyPath(assemblyPath); + var testCases = new List(); + + var filteredBenchmarks = new List(); + foreach (var benchmark in benchmarks) + { + var needsJobInfo = benchmark.BenchmarksCases.Select(c => c.Job.DisplayInfo).Distinct().Count() > 1; + var filteredCases = new List(); + foreach (var benchmarkCase in benchmark.BenchmarksCases) + { + var testId = benchmarkCase.GetTestCaseId(); + if (benchmarkIds == null || benchmarkIds.Contains(testId)) + { + filteredCases.Add(benchmarkCase); + testCases.Add(benchmarkCase.ToVsTestCase(assemblyPath, needsJobInfo)); + } + } + + if (filteredCases.Count > 0) + { + filteredBenchmarks.Add(new BenchmarkRunInfo(filteredCases.ToArray(), benchmark.Type, benchmark.Config)); + } + } + + benchmarks = filteredBenchmarks.ToArray(); + + if (benchmarks.Length == 0) + return; + + // Create an event processor which will subscribe to events and push them to VSTest + var eventProcessor = new VsTestEventProcessor(testCases, recorder, cts.Token); + + // Create a logger which will forward all log messages in BDN to the VSTest logger. + var logger = new VsTestLogger(recorder.GetLogger()); + + // Modify all the benchmarks so that the event process and logger is added. + benchmarks = benchmarks + .Select(b => new BenchmarkRunInfo( + b.BenchmarksCases, + b.Type, + b.Config.AddEventProcessor(eventProcessor).AddLogger(logger).CreateImmutableConfig())) + .ToArray(); + + // Run all the benchmarks, and ensure that any tests that don't have a result yet are sent. + BenchmarkRunner.Run(benchmarks); + eventProcessor.SendUnsentTestResults(); + } + + /// + /// Stop the benchmarks when next able. + /// + public void Cancel() + { + cts.Cancel(); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkEnumeratorWrapper.cs b/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkEnumeratorWrapper.cs new file mode 100644 index 0000000000..b3ad68bb23 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkEnumeratorWrapper.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace BenchmarkDotNet.TestAdapter.Remoting +{ + /// + /// A wrapper around the BenchmarkEnumerator for passing data across AppDomain boundaries. + /// + internal class BenchmarkEnumeratorWrapper : MarshalByRefObject + { + /// + /// Gets a list of VSTest TestCases from the given assembly. + /// Each test case is serialized into a string so that it can be used across AppDomain boundaries. + /// + /// The dll or exe of the benchmark project. + /// The serialized test cases. + public List GetTestCasesFromAssemblyPathSerialized(string assemblyPath) + { + var serializedTestCases = new List(); + foreach (var runInfo in BenchmarkEnumerator.GetBenchmarksFromAssemblyPath(assemblyPath)) + { + // If all the benchmarks have the same job, then no need to include job info. + var needsJobInfo = runInfo.BenchmarksCases.Select(c => c.Job.DisplayInfo).Distinct().Count() > 1; + foreach (var benchmarkCase in runInfo.BenchmarksCases) + { + var testCase = benchmarkCase.ToVsTestCase(assemblyPath, needsJobInfo); + serializedTestCases.Add(SerializationHelpers.Serialize(testCase)); + } + } + + return serializedTestCases; + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkExecutorWrapper.cs b/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkExecutorWrapper.cs new file mode 100644 index 0000000000..646ae2f8be --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkExecutorWrapper.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; + +namespace BenchmarkDotNet.TestAdapter.Remoting +{ + /// + /// A wrapper around the BenchmarkExecutor that works across AppDomain boundaries. + /// + internal class BenchmarkExecutorWrapper : MarshalByRefObject + { + private readonly BenchmarkExecutor benchmarkExecutor = new (); + + public void RunBenchmarks(string assemblyPath, TestExecutionRecorderWrapper recorder, HashSet? benchmarkIds = null) + { + benchmarkExecutor.RunBenchmarks(assemblyPath, recorder, benchmarkIds); + } + + public void Cancel() + { + benchmarkExecutor.Cancel(); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Remoting/MessageLoggerWrapper.cs b/src/BenchmarkDotNet.TestAdapter/Remoting/MessageLoggerWrapper.cs new file mode 100644 index 0000000000..00c4f5325f --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Remoting/MessageLoggerWrapper.cs @@ -0,0 +1,23 @@ +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using System; + +namespace BenchmarkDotNet.TestAdapter.Remoting +{ + /// + /// A wrapper around an IMessageLogger that works across AppDomain boundaries. + /// + internal class MessageLoggerWrapper : MarshalByRefObject, IMessageLogger + { + private readonly IMessageLogger logger; + + public MessageLoggerWrapper(IMessageLogger logger) + { + this.logger = logger; + } + + public void SendMessage(TestMessageLevel testMessageLevel, string message) + { + logger.SendMessage(testMessageLevel, message); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Remoting/SerializationHelpers.cs b/src/BenchmarkDotNet.TestAdapter/Remoting/SerializationHelpers.cs new file mode 100644 index 0000000000..5b13bd5175 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Remoting/SerializationHelpers.cs @@ -0,0 +1,26 @@ +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; + +namespace BenchmarkDotNet.TestAdapter.Remoting +{ + /// + /// A set of helper methods for serializing and deserializing the VSTest TestCases and TestReports. + /// + internal static class SerializationHelpers + { + // Version number of the VSTest protocol that the adapter supports. Only needs to be updated when + // the VSTest protocol has a change and this test adapter wishes to take a dependency on it. + // A list of protocol versions and a summary of the changes that were made in them can be found here: + // https://github.com/microsoft/vstest/blob/main/docs/Overview.md#protocolversion-request + private const int VsTestProtocolVersion = 7; + + public static string Serialize(T data) + { + return JsonDataSerializer.Instance.Serialize(data, version: VsTestProtocolVersion); + } + + public static T Deserialize(string data) + { + return JsonDataSerializer.Instance.Deserialize(data, version: VsTestProtocolVersion)!; + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Remoting/TestExecutionRecorderWrapper.cs b/src/BenchmarkDotNet.TestAdapter/Remoting/TestExecutionRecorderWrapper.cs new file mode 100644 index 0000000000..0669e79019 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Remoting/TestExecutionRecorderWrapper.cs @@ -0,0 +1,39 @@ +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; +using System; + +namespace BenchmarkDotNet.TestAdapter.Remoting +{ + /// + /// A wrapper around the ITestExecutionRecorder which works across AppDomain boundaries. + /// + internal class TestExecutionRecorderWrapper : MarshalByRefObject + { + private readonly ITestExecutionRecorder testExecutionRecorder; + + public TestExecutionRecorderWrapper(ITestExecutionRecorder testExecutionRecorder) + { + this.testExecutionRecorder = testExecutionRecorder; + } + + public MessageLoggerWrapper GetLogger() + { + return new MessageLoggerWrapper(testExecutionRecorder); + } + + internal void RecordStart(string serializedTestCase) + { + testExecutionRecorder.RecordStart(SerializationHelpers.Deserialize(serializedTestCase)); + } + + internal void RecordEnd(string serializedTestCase, TestOutcome testOutcome) + { + testExecutionRecorder.RecordEnd(SerializationHelpers.Deserialize(serializedTestCase), testOutcome); + } + + internal void RecordResult(string serializedTestResult) + { + testExecutionRecorder.RecordResult(SerializationHelpers.Deserialize(serializedTestResult)); + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs b/src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs new file mode 100644 index 0000000000..eb6695de1b --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs @@ -0,0 +1,213 @@ +using BenchmarkDotNet.TestAdapter.Remoting; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading; + +namespace BenchmarkDotNet.TestAdapter +{ + /// + /// Discovers and executes benchmarks using the VSTest protocol. + /// + [ExtensionUri(ExecutorUriString)] + [DefaultExecutorUri(ExecutorUriString)] + [FileExtension(".dll")] + [FileExtension(".exe")] + public class VsTestAdapter : ITestExecutor, ITestDiscoverer + { + // This URI is used to identify the adapter. + internal const string ExecutorUriString = "executor://BenchmarkDotNet.TestAdapter"; + internal static readonly Uri ExecutorUri = new Uri(ExecutorUriString); + + /// + /// Cancellation token used to stop any benchmarks that are currently running. + /// + private CancellationTokenSource? cts = null; + + /// + /// Discovers the benchmarks. + /// + /// List of assemblies to search for benchmarks in. + /// A context that the discovery is performed in. + /// Logger that sends messages back to VSTest host. + /// Interface that provides methods for sending discovered benchmarks back to the host. + public void DiscoverTests( + IEnumerable sources, + IDiscoveryContext discoveryContext, + IMessageLogger logger, + ITestCaseDiscoverySink discoverySink) + { + foreach (var source in sources) + { + ValidateSourceIsAssemblyOrThrow(source); + foreach (var testCase in GetVsTestCasesFromAssembly(source, logger)) + { + discoverySink.SendTestCase(testCase); + } + } + } + + /// + /// Runs a given set of test cases that represent benchmarks. + /// + /// The tests to run. + /// A context that the run is performed in. + /// Interface used for communicating with the VSTest host. + public void RunTests(IEnumerable? tests, IRunContext? runContext, IFrameworkHandle? frameworkHandle) + { + if (tests == null) + throw new ArgumentNullException(nameof(tests)); + if (frameworkHandle == null) + throw new ArgumentNullException(nameof(frameworkHandle)); + + cts ??= new CancellationTokenSource(); + + foreach (var testsPerAssembly in tests.GroupBy(t => t.Source)) + RunBenchmarks(testsPerAssembly.Key, frameworkHandle, testsPerAssembly); + + cts = null; + } + + /// + /// Runs all benchmarks in the given set of sources (assemblies). + /// + /// The assemblies to run. + /// A context that the run is performed in. + /// Interface used for communicating with the VSTest host. + public void RunTests(IEnumerable? sources, IRunContext? runContext, IFrameworkHandle? frameworkHandle) + { + if (sources == null) + throw new ArgumentNullException(nameof(sources)); + if (frameworkHandle == null) + throw new ArgumentNullException(nameof(frameworkHandle)); + + cts ??= new CancellationTokenSource(); + + foreach (var source in sources) + RunBenchmarks(source, frameworkHandle); + + cts = null; + } + + /// + /// Stops any currently running benchmarks. + /// + public void Cancel() + { + cts?.Cancel(); + } + + /// + /// Gets the VSTest test cases in the given assembly. + /// + /// The dll or exe of the benchmark project. + /// A logger that sends logs to VSTest. + /// The VSTest test cases inside the given assembly. + private static List GetVsTestCasesFromAssembly(string assemblyPath, IMessageLogger logger) + { + try + { + // Ensure that the test enumeration is done inside the context of the source directory. + var enumerator = (BenchmarkEnumeratorWrapper)CreateIsolatedType(typeof(BenchmarkEnumeratorWrapper), assemblyPath); + var testCases = enumerator + .GetTestCasesFromAssemblyPathSerialized(assemblyPath) + .Select(SerializationHelpers.Deserialize) + .ToList(); + + // Validate that all test ids are unique + var idLookup = new Dictionary(); + foreach (var testCase in testCases) + { + if (idLookup.TryGetValue(testCase.Id, out var matchingCase)) + throw new Exception($"Encountered Duplicate Test ID: '{testCase.DisplayName}' and '{matchingCase}'"); + + idLookup[testCase.Id] = testCase.DisplayName; + } + + return testCases; + } + catch (Exception ex) + { + logger.SendMessage(TestMessageLevel.Error, $"Failed to load benchmarks from assembly\n{ex}"); + throw; + } + } + + /// + /// Runs the benchmarks in the given source. + /// + /// The dll or exe of the benchmark project. + /// An interface used to communicate with the VSTest host. + /// + /// The specific test cases to be run if specified. + /// If unspecified, runs all the test cases in the source. + /// + private void RunBenchmarks(string source, IFrameworkHandle frameworkHandle, IEnumerable? testCases = null) + { + ValidateSourceIsAssemblyOrThrow(source); + + // Create a HashSet of all the TestCase IDs to be run if specified. + var caseIds = testCases == null ? null : new HashSet(testCases.Select(c => c.Id)); + + try + { + // Ensure that test execution is done inside the context of the source directory. + var executor = (BenchmarkExecutorWrapper)CreateIsolatedType(typeof(BenchmarkExecutorWrapper), source); + cts?.Token.Register(executor.Cancel); + + executor.RunBenchmarks(source, new TestExecutionRecorderWrapper(frameworkHandle), caseIds); + } + catch (Exception ex) + { + frameworkHandle.SendMessage(TestMessageLevel.Error, $"Failed to run benchmarks in assembly\n{ex}"); + throw; + } + } + + /// + /// This will create the given type in a child AppDomain when used in .NET Framework. + /// If not in the .NET Framework, it will use the current AppDomain. + /// + /// The type to create. + /// The dll or exe of the benchmark project. + /// The created object. + private static object CreateIsolatedType(Type type, string assemblyPath) + { + // .NET Framework runs require a custom AppDomain to be set up to run the benchmarks in because otherwise, + // all the assemblies will be loaded from the VSTest console rather than from the directory that the BDN + // program under test lives in. .NET Core assembly resolution is smarter and will correctly load the right + // assembly versions as needed and does not require a custom AppDomain. Unfortunately, the APIs needed to + // create the AppDomain for .NET Framework are not part of .NET Standard, and so a multi-targeting solution + // such as this is required to get this to work. This same approach is also used by other .NET unit testing + // libraries as well, further justifying this approach to solving how to get the correct assemblies loaded. +#if NETFRAMEWORK + var appBase = Path.GetDirectoryName(assemblyPath); + var setup = new AppDomainSetup { ApplicationBase = appBase }; + var domainName = $"Isolated Domain for {type.Name}"; + var appDomain = AppDomain.CreateDomain(domainName, null, setup); + return appDomain.CreateInstanceAndUnwrap( + type.Assembly.FullName, type.FullName, false, BindingFlags.Default, null, null, null, null); +#else + return Activator.CreateInstance(type); +#endif + } + + private static void ValidateSourceIsAssemblyOrThrow(string source) + { + if (string.IsNullOrEmpty(source)) + throw new ArgumentException($"'{nameof(source)}' cannot be null or whitespace.", nameof(source)); + + if (!Path.HasExtension(source)) + throw new NotSupportedException($"Missing extension on source '{source}', must have the extension '.dll' or '.exe'."); + + var extension = Path.GetExtension(source); + if (!string.Equals(extension, ".dll", StringComparison.OrdinalIgnoreCase) && !string.Equals(extension, ".exe", StringComparison.OrdinalIgnoreCase)) + throw new NotSupportedException($"Unsupported extension on source '{source}', must have the extension '.dll' or '.exe'."); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs b/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs new file mode 100644 index 0000000000..35438d4d80 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs @@ -0,0 +1,198 @@ +using BenchmarkDotNet.EventProcessors; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.TestAdapter.Remoting; +using BenchmarkDotNet.Toolchains.Results; +using BenchmarkDotNet.Validators; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Perfolizer.Mathematics.Histograms; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Threading; + +namespace BenchmarkDotNet.TestAdapter +{ + /// + /// An event processor which will pass on benchmark execution information to VSTest. + /// + internal class VsTestEventProcessor : EventProcessor + { + private readonly Dictionary cases; + private readonly TestExecutionRecorderWrapper recorder; + private readonly CancellationToken cancellationToken; + private readonly Stopwatch runTimerStopwatch = new (); + private readonly Dictionary testResults = new (); + private readonly HashSet sentTestResults = new (); + + public VsTestEventProcessor( + List cases, + TestExecutionRecorderWrapper recorder, + CancellationToken cancellationToken) + { + this.cases = cases.ToDictionary(c => c.Id); + this.recorder = recorder; + this.cancellationToken = cancellationToken; + } + + public override void OnValidationError(ValidationError validationError) + { + // If the error is not linked to a benchmark case, then set the error on all benchmarks + var errorCases = validationError.BenchmarkCase == null + ? cases.Values.ToList() + : new List { cases[validationError.BenchmarkCase.GetTestCaseId()] }; + foreach (var testCase in errorCases) + { + var testResult = GetOrCreateTestResult(testCase); + + if (validationError.IsCritical) + { + // Fail if there is a critical validation error + testResult.Outcome = TestOutcome.Failed; + + // Append validation error message to end of test case error message + testResult.ErrorMessage = testResult.ErrorMessage == null + ? validationError.Message + : $"{testResult.ErrorMessage}\n{validationError.Message}"; + + // The test result is not sent yet, in case there are multiple validation errors that need to be sent. + } + else + { + // If the validation error is not critical, append it as a message + testResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, $"WARNING: {validationError.Message}\n")); + } + } + } + + public override void OnBuildComplete(BuildPartition buildPartition, BuildResult buildResult) + { + // Only need to handle build failures + if (!buildResult.IsBuildSuccess) + { + foreach (var benchmarkBuildInfo in buildPartition.Benchmarks) + { + var testCase = cases[benchmarkBuildInfo.BenchmarkCase.GetTestCaseId()]; + var testResult = GetOrCreateTestResult(testCase); + + if (buildResult.GenerateException != null) + testResult.ErrorMessage = $"// Generate Exception: {buildResult.GenerateException.Message}"; + else if (!buildResult.IsBuildSuccess && buildResult.TryToExplainFailureReason(out string reason)) + testResult.ErrorMessage = $"// Build Error: {reason}"; + else if (buildResult.ErrorMessage != null) + testResult.ErrorMessage = $"// Build Error: {buildResult.ErrorMessage}"; + testResult.Outcome = TestOutcome.Failed; + + // Send the result immediately + RecordStart(testCase); + RecordEnd(testCase, testResult.Outcome); + RecordResult(testResult); + sentTestResults.Add(testCase.Id); + } + } + } + + public override void OnStartRunBenchmark(BenchmarkCase benchmarkCase) + { + // TODO: add proper cancellation support to BDN so that we don't need to do cancellation through the event processor + cancellationToken.ThrowIfCancellationRequested(); + + var testCase = cases[benchmarkCase.GetTestCaseId()]; + var testResult = GetOrCreateTestResult(testCase); + testResult.StartTime = DateTimeOffset.UtcNow; + + RecordStart(testCase); + runTimerStopwatch.Restart(); + } + + public override void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkReport report) + { + var testCase = cases[benchmarkCase.GetTestCaseId()]; + var testResult = GetOrCreateTestResult(testCase); + testResult.EndTime = DateTimeOffset.UtcNow; + testResult.Duration = runTimerStopwatch.Elapsed; + testResult.Outcome = report.Success ? TestOutcome.Passed : TestOutcome.Failed; + + var resultRuns = report.GetResultRuns(); + + // Provide the raw result runs data. + testResult.SetPropertyValue(VsTestProperties.Measurement, resultRuns.Select(m => m.Nanoseconds.ToString()).ToArray()); + + // Add a message to the TestResult which contains the results summary. + testResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, report.BenchmarkCase.DisplayInfo + "\n")); + testResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, $"Runtime = {report.GetRuntimeInfo()}; GC = {report.GetGcInfo()}\n")); + + var statistics = resultRuns.GetStatistics(); + var cultureInfo = CultureInfo.InvariantCulture; + var formatter = statistics.CreateNanosecondFormatter(cultureInfo); + + var builder = new StringBuilder(); + var histogram = HistogramBuilder.Adaptive.Build(statistics.Sample.Values); + builder.AppendLine("-------------------- Histogram --------------------"); + builder.AppendLine(histogram.ToString(formatter)); + builder.AppendLine("---------------------------------------------------"); + + var statisticsOutput = statistics.ToString(cultureInfo, formatter, calcHistogram: false); + builder.AppendLine(statisticsOutput); + + testResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, builder.ToString())); + + RecordEnd(testResult.TestCase, testResult.Outcome); + RecordResult(testResult); + sentTestResults.Add(testCase.Id); + } + + /// + /// Iterate through all the benchmarks that were scheduled to run, and if they haven't been sent yet, send the result through. + /// + public void SendUnsentTestResults() + { + foreach (var testCase in cases.Values) + { + if (!sentTestResults.Contains(testCase.Id)) + { + var testResult = GetOrCreateTestResult(testCase); + if (testResult.Outcome == TestOutcome.None) + testResult.Outcome = TestOutcome.Skipped; + RecordStart(testCase); + RecordEnd(testCase, testResult.Outcome); + RecordResult(testResult); + } + } + } + + private TestResult GetOrCreateTestResult(TestCase testCase) + { + if (testResults.TryGetValue(testCase.Id, out var testResult)) + return testResult; + + var newResult = new TestResult(testCase) + { + ComputerName = Environment.MachineName, + DisplayName = testCase.DisplayName + }; + + testResults[testCase.Id] = newResult; + return newResult; + } + + private void RecordStart(TestCase testCase) + { + recorder.RecordStart(SerializationHelpers.Serialize(testCase)); + } + + private void RecordEnd(TestCase testCase, TestOutcome testOutcome) + { + recorder.RecordEnd(SerializationHelpers.Serialize(testCase), testOutcome); + } + + private void RecordResult(TestResult testResult) + { + recorder.RecordResult(SerializationHelpers.Serialize(testResult)); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/VSTestLogger.cs b/src/BenchmarkDotNet.TestAdapter/VSTestLogger.cs new file mode 100644 index 0000000000..c9a8de620f --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/VSTestLogger.cs @@ -0,0 +1,62 @@ +using BenchmarkDotNet.Loggers; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using System.Text; + +namespace BenchmarkDotNet.TestAdapter +{ + /// + /// A class to send logs from BDN to the VSTest output log. + /// + internal sealed class VsTestLogger : ILogger + { + private readonly IMessageLogger messageLogger; + private readonly StringBuilder currentLine = new StringBuilder(); + private TestMessageLevel currentLevel = TestMessageLevel.Informational; + + public VsTestLogger(IMessageLogger logger) + { + messageLogger = logger; + } + + public string Id => nameof(VsTestLogger); + + public int Priority => 0; + + public void Flush() + { + WriteLine(); + } + + public void Write(LogKind logKind, string text) + { + currentLine.Append(text); + + // Assume that if the log kind is an error, that the whole line is treated as an error + // The level will be reset to Informational when WriteLine() is called. + currentLevel = logKind switch + { + LogKind.Error => TestMessageLevel.Error, + LogKind.Warning => TestMessageLevel.Warning, + _ => currentLevel + }; + } + + public void WriteLine() + { + // The VSTest logger throws an error on logging empty or whitespace strings, so skip them. + if (currentLine.Length == 0) + return; + + messageLogger.SendMessage(currentLevel, currentLine.ToString()); + + currentLevel = TestMessageLevel.Informational; + currentLine.Clear(); + } + + public void WriteLine(LogKind logKind, string text) + { + Write(logKind, text); + WriteLine(); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/VSTestProperties.cs b/src/BenchmarkDotNet.TestAdapter/VSTestProperties.cs new file mode 100644 index 0000000000..6bcbdcf299 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/VSTestProperties.cs @@ -0,0 +1,22 @@ +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +namespace BenchmarkDotNet.TestAdapter +{ + /// + /// A class that contains all the custom properties that can be set on VSTest TestCase and TestResults. + /// Some of these properties are well known as they are also used by VSTest adapters for other test libraries. + /// + internal static class VsTestProperties + { + /// + /// A test property used for storing the test results so that they could be accessed + /// programmatically from a custom VSTest runner. + /// + internal static readonly TestProperty Measurement = TestProperty.Register( + "BenchmarkDotNet.TestAdapter.Measurements", + "Measurements", + typeof(string[]), + TestPropertyAttributes.Hidden, + typeof(TestResult)); + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props b/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props new file mode 100644 index 0000000000..a9e8340b30 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props @@ -0,0 +1,21 @@ + + + $(MSBuildThisFileDirectory)..\entrypoints\ + + + + + + + + + + + false + + + \ No newline at end of file diff --git a/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.cs b/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.cs new file mode 100644 index 0000000000..4f9036a00c --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.cs @@ -0,0 +1,11 @@ +// +// Exclude this file from StyleCop analysis. This file isn't generated but is added to projects. +// + +using BenchmarkDotNet.Running; +using System.Reflection; + +public class __AutoGeneratedEntryPointClass +{ + public static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(__AutoGeneratedEntryPointClass).Assembly).Run(args); +} diff --git a/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.fs b/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.fs new file mode 100644 index 0000000000..7e305320b8 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.fs @@ -0,0 +1,13 @@ +// +// Exclude this file from StyleCop analysis. This file isn't generated but is added to projects. +// + +module __AutoGeneratedEntryPointClass +open System.Reflection; +open BenchmarkDotNet.Running + +type internal __Marker = interface end // Used to help locale current assembly +[] +let main argv = + BenchmarkSwitcher.FromAssembly(typeof<__Marker>.Assembly).Run(argv) |> ignore + 0 // return an integer exit code diff --git a/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.vb b/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.vb new file mode 100644 index 0000000000..24458cafca --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/entrypoints/EntryPoint.vb @@ -0,0 +1,14 @@ +REM +REM Exclude this file from StyleCop analysis. This file isn't generated but is added to projects. +REM + +Imports System.Reflection +Imports BenchmarkDotNet.Running + +Namespace Global + Module __AutoGeneratedEntryPointClass + Sub Main(args As String()) + Dim summary = BenchmarkSwitcher.FromAssembly(MethodBase.GetCurrentMethod().Module.Assembly).Run(args) + End Sub + End Module +End Namespace \ No newline at end of file diff --git a/src/BenchmarkDotNet/Analysers/BaselineCustomAnalyzer.cs b/src/BenchmarkDotNet/Analysers/BaselineCustomAnalyzer.cs index 51663e4085..9f2ff0028c 100644 --- a/src/BenchmarkDotNet/Analysers/BaselineCustomAnalyzer.cs +++ b/src/BenchmarkDotNet/Analysers/BaselineCustomAnalyzer.cs @@ -25,13 +25,14 @@ protected override IEnumerable AnalyseSummary(Summary summary) foreach (var benchmarkCase in summary.BenchmarksCases) { - string logicalGroupKey = summary.GetLogicalGroupKey(benchmarkCase); + string? logicalGroupKey = summary.GetLogicalGroupKey(benchmarkCase); var baseline = summary.GetBaseline(logicalGroupKey); if (BaselineCustomColumn.ResultsAreInvalid(summary, benchmarkCase, baseline) == false) continue; var message = "A question mark '?' symbol indicates that it was not possible to compute the " + - $"({columnNames}) column(s) because the baseline value is too close to zero."; + $"({columnNames}) column(s) because the baseline or benchmark could not be found, or " + + $"the baseline value is too close to zero."; yield return Conclusion.CreateWarning(Id, message); } diff --git a/src/BenchmarkDotNet/Analysers/ConclusionHelper.cs b/src/BenchmarkDotNet/Analysers/ConclusionHelper.cs index 9542230606..af872414f2 100644 --- a/src/BenchmarkDotNet/Analysers/ConclusionHelper.cs +++ b/src/BenchmarkDotNet/Analysers/ConclusionHelper.cs @@ -10,7 +10,7 @@ public static class ConclusionHelper public static void Print(ILogger logger, IEnumerable conclusions) { PrintFiltered(conclusions, ConclusionKind.Error, "Errors", logger.WriteLineError); - PrintFiltered(conclusions, ConclusionKind.Warning, "Warnings", logger.WriteLineError); + PrintFiltered(conclusions, ConclusionKind.Warning, "Warnings", logger.WriteLineWarning); PrintFiltered(conclusions, ConclusionKind.Hint, "Hints", logger.WriteLineHint); } diff --git a/src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs b/src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs index 151ac70be2..e1e4465ef6 100644 --- a/src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs +++ b/src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs @@ -20,7 +20,7 @@ protected override IEnumerable AnalyseReport(BenchmarkReport report, if (statistics == null || statistics.N < EngineResolver.DefaultMinWorkloadIterationCount) yield break; - double mValue = MValueCalculator.Calculate(statistics.OriginalValues); + double mValue = MValueCalculator.Calculate(statistics.Sample.Values); if (mValue > 4.2) yield return Create("is multimodal", mValue, report, summary.GetCultureInfo()); else if (mValue > 3.2) diff --git a/src/BenchmarkDotNet/Analysers/OutliersAnalyser.cs b/src/BenchmarkDotNet/Analysers/OutliersAnalyser.cs index 6da2088e86..570ea7806e 100644 --- a/src/BenchmarkDotNet/Analysers/OutliersAnalyser.cs +++ b/src/BenchmarkDotNet/Analysers/OutliersAnalyser.cs @@ -3,6 +3,7 @@ using System.Linq; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; using JetBrains.Annotations; @@ -22,25 +23,11 @@ protected override IEnumerable AnalyseReport(BenchmarkReport report, var workloadActual = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).ToArray(); if (workloadActual.IsEmpty()) yield break; - var result = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Result)).ToArray(); var outlierMode = report.BenchmarkCase.Job.ResolveValue(AccuracyMode.OutlierModeCharacteristic, EngineResolver.Instance); // TODO: improve var statistics = workloadActual.GetStatistics(); var allOutliers = statistics.AllOutliers; var actualOutliers = statistics.GetActualOutliers(outlierMode); - if (result.Length + actualOutliers.Length != workloadActual.Length) - { - // This should never happen - yield return CreateHint( - "Something went wrong with outliers: " + - $"Size(WorkloadActual) = {workloadActual.Length}, " + - $"Size(WorkloadActual/Outliers) = {actualOutliers.Length}, " + - $"Size(Result) = {result.Length}), " + - $"OutlierMode = {outlierMode}", - report); - yield break; - } - var cultureInfo = summary.GetCultureInfo(); if (allOutliers.Any()) yield return CreateHint(GetMessage(actualOutliers, allOutliers, statistics.LowerOutliers, statistics.UpperOutliers, cultureInfo), report); @@ -67,7 +54,7 @@ string Format(int n, string verb) return $"{n} {words} {verb}"; } - var rangeMessages = new List { GetRangeMessage(lowerOutliers, cultureInfo), GetRangeMessage(upperOutliers, cultureInfo) }; + var rangeMessages = new List { GetRangeMessage(lowerOutliers), GetRangeMessage(upperOutliers) }; rangeMessages.RemoveAll(string.IsNullOrEmpty); string rangeMessage = rangeMessages.Any() ? " (" + string.Join(", ", rangeMessages) + ")" @@ -80,20 +67,17 @@ string Format(int n, string verb) return Format(actualOutliers.Length, "removed") + ", " + Format(allOutliers.Length, "detected") + rangeMessage; } - private static string? GetRangeMessage(double[] values, CultureInfo cultureInfo) + private static string? GetRangeMessage(double[] values) { - string Format(double value) => TimeInterval.FromNanoseconds(value).ToString(cultureInfo, "N2"); + string Format(double value) => TimeInterval.FromNanoseconds(value).ToDefaultString("N2"); - switch (values.Length) { - case 0: - return null; - case 1: - return Format(values.First()); - case 2: - return Format(values.Min()) + ", " + Format(values.Max()); - default: - return Format(values.Min()) + ".." + Format(values.Max()); - } + return values.Length switch + { + 0 => null, + 1 => Format(values.First()), + 2 => Format(values.Min()) + ", " + Format(values.Max()), + _ => Format(values.Min()) + ".." + Format(values.Max()) + }; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs b/src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs index 240f2e8475..4d83a5d154 100644 --- a/src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs +++ b/src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs @@ -19,7 +19,7 @@ private ZeroMeasurementAnalyser() { } protected override IEnumerable AnalyseReport(BenchmarkReport report, Summary summary) { - var currentFrequency = summary.HostEnvironmentInfo.CpuInfo.Value.MaxFrequency; + var currentFrequency = summary.HostEnvironmentInfo.Cpu.Value.MaxFrequency(); if (!currentFrequency.HasValue || currentFrequency <= 0) currentFrequency = FallbackCpuResolutionValue.ToFrequency(); @@ -28,17 +28,17 @@ protected override IEnumerable AnalyseReport(BenchmarkReport report, var workloadMeasurements = entire.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).ToArray(); if (workloadMeasurements.IsEmpty()) yield break; - var workload = workloadMeasurements.GetStatistics(); + var workloadSample = workloadMeasurements.GetStatistics().Sample; var threshold = currentFrequency.Value.ToResolution().Nanoseconds / 2; var zeroMeasurement = overheadMeasurements.Any() - ? ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workload.WithoutOutliers(), overheadMeasurements.GetStatistics().WithoutOutliers()) - : ZeroMeasurementHelper.CheckZeroMeasurementOneSample(workload.WithoutOutliers(), threshold); + ? ZeroMeasurementHelper.AreIndistinguishable(workloadSample, overheadMeasurements.GetStatistics().Sample) + : ZeroMeasurementHelper.IsNegligible(workloadSample, threshold); if (zeroMeasurement) yield return CreateWarning("The method duration is indistinguishable from the empty method duration", - report, false); + report, false); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs index 5e3f650167..9079bc54d9 100644 --- a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs +++ b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs @@ -1,30 +1,39 @@ +using BenchmarkDotNet.Mathematics; +using Perfolizer; +using Perfolizer.Horology; +using Perfolizer.Mathematics.Common; +using Perfolizer.Mathematics.GenericEstimators; using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; +using Perfolizer.Mathematics.SignificanceTesting.MannWhitney; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Analysers { - public static class ZeroMeasurementHelper + internal static class ZeroMeasurementHelper { - /// - /// Checks distribution against Zero Measurement hypothesis in case of known threshold - /// - /// True if measurement is ZeroMeasurement - public static bool CheckZeroMeasurementOneSample(double[] results, double threshold) + public static bool IsNegligible(Sample results, double threshold) => HodgesLehmannEstimator.Instance.Median(results) < threshold; + public static bool IsNoticeable(Sample results, double threshold) => !IsNegligible(results, threshold); + + public static bool AreIndistinguishable(double[] workload, double[] overhead, Threshold? threshold = null) { - if (results.Length < 3) - return false; - return !StudentTest.Instance.IsGreater(results, threshold).NullHypothesisIsRejected; + var workloadSample = new Sample(workload, TimeUnit.Nanosecond); + var overheadSample = new Sample(overhead, TimeUnit.Nanosecond); + return AreIndistinguishable(workloadSample, overheadSample, threshold); } - /// - /// Checks distribution against Zero Measurement hypothesis in case of two samples - /// - /// True if measurement is ZeroMeasurement - public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead, Threshold? threshold = null) + public static bool AreIndistinguishable(Sample workload, Sample overhead, Threshold? threshold = null) { - if (workload.Length < 3 || overhead.Length < 3) + threshold ??= MathHelper.DefaultThreshold; + var tost = new SimpleEquivalenceTest(MannWhitneyTest.Instance); + if (workload.Size == 1 || overhead.Size == 1) return false; - return !WelchTest.Instance.IsGreater(workload, overhead, threshold).NullHypothesisIsRejected; + return tost.Perform(workload, overhead, threshold, SignificanceLevel.P1E5) == ComparisonResult.Indistinguishable; } + + public static bool AreDistinguishable(double[] workload, double[] overhead, Threshold? threshold = null) => + !AreIndistinguishable(workload, overhead, threshold); + + public static bool AreDistinguishable(Sample workload, Sample overhead, Threshold? threshold = null) => + !AreIndistinguishable(workload, overhead, threshold); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/Columns/ConfidenceIntervalErrorColumnAttribute.cs b/src/BenchmarkDotNet/Attributes/Columns/ConfidenceIntervalErrorColumnAttribute.cs index d0d1f90faf..b1b87d5241 100644 --- a/src/BenchmarkDotNet/Attributes/Columns/ConfidenceIntervalErrorColumnAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Columns/ConfidenceIntervalErrorColumnAttribute.cs @@ -7,8 +7,7 @@ namespace BenchmarkDotNet.Attributes [PublicAPI] public class ConfidenceIntervalErrorColumnAttribute : ColumnConfigBaseAttribute { - public ConfidenceIntervalErrorColumnAttribute(ConfidenceLevel level = ConfidenceLevel.L999) : base(StatisticColumn.CiError(level)) - { - } + public ConfidenceIntervalErrorColumnAttribute() : base(StatisticColumn.CiError(ConfidenceLevel.L999)) { } + public ConfidenceIntervalErrorColumnAttribute(ConfidenceLevel level) : base(StatisticColumn.CiError(level)) { } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/Columns/OperationsPerSecondAttribute.cs b/src/BenchmarkDotNet/Attributes/Columns/OperationsPerSecondAttribute.cs new file mode 100644 index 0000000000..6315dc3694 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/Columns/OperationsPerSecondAttribute.cs @@ -0,0 +1,10 @@ +using BenchmarkDotNet.Columns; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Attributes +{ + public class OperationsPerSecondAttribute : ColumnConfigBaseAttribute + { + public OperationsPerSecondAttribute() : base(StatisticColumn.OperationsPerSecond) { } + } +} diff --git a/src/BenchmarkDotNet/Attributes/Columns/StdErrorColumnAttribute.cs b/src/BenchmarkDotNet/Attributes/Columns/StdErrorColumnAttribute.cs index 27a22b1e1a..23327e187e 100644 --- a/src/BenchmarkDotNet/Attributes/Columns/StdErrorColumnAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Columns/StdErrorColumnAttribute.cs @@ -4,10 +4,5 @@ namespace BenchmarkDotNet.Attributes { [PublicAPI] - public class StdErrorColumnAttribute : ColumnConfigBaseAttribute - { - public StdErrorColumnAttribute() : base(StatisticColumn.StdErr) - { - } - } + public class StdErrorColumnAttribute() : ColumnConfigBaseAttribute(StatisticColumn.StdErr) { } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/Columns/WelchTTestPValueColumnAttribute.cs b/src/BenchmarkDotNet/Attributes/Columns/WelchTTestPValueColumnAttribute.cs index 797ab57a9d..607f2c4fe2 100644 --- a/src/BenchmarkDotNet/Attributes/Columns/WelchTTestPValueColumnAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Columns/WelchTTestPValueColumnAttribute.cs @@ -1,8 +1,7 @@ using System; using BenchmarkDotNet.Columns; using JetBrains.Annotations; -using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; +using Perfolizer.Mathematics.Common; namespace BenchmarkDotNet.Attributes { @@ -10,17 +9,11 @@ namespace BenchmarkDotNet.Attributes [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)] public class StatisticalTestColumnAttribute : ColumnConfigBaseAttribute { - public StatisticalTestColumnAttribute(StatisticalTestKind testKind, ThresholdUnit thresholdUnit, double value, bool showPValues = false) - : base(StatisticalTestColumn.Create(testKind, Threshold.Create(thresholdUnit, value), showPValues)) { } + public StatisticalTestColumnAttribute() : base(StatisticalTestColumn.Create("10%", null)) { } - public StatisticalTestColumnAttribute(StatisticalTestKind testKind, bool showPValues = false) : this(testKind, ThresholdUnit.Ratio, 0.1, showPValues) { } + public StatisticalTestColumnAttribute(string threshold) : base(StatisticalTestColumn.Create(threshold, null)) { } - public StatisticalTestColumnAttribute(bool showPValues = false) : this(StatisticalTestKind.MannWhitney, showPValues) {} - } - - [Obsolete("Use StatisticalTestAttribute")] - public class WelchTTestPValueColumnAttribute : StatisticalTestColumnAttribute - { - public WelchTTestPValueColumnAttribute() : base(StatisticalTestKind.Welch) { } + public StatisticalTestColumnAttribute(string threshold, SignificanceLevel significanceLevel) + : base(StatisticalTestColumn.Create(threshold, significanceLevel)) { } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs b/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs index bff956e968..21f2903124 100644 --- a/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs @@ -9,6 +9,10 @@ public class ExceptionDiagnoserAttribute : Attribute, IConfigSource { public IConfig Config { get; } - public ExceptionDiagnoserAttribute() => Config = ManualConfig.CreateEmpty().AddDiagnoser(ExceptionDiagnoser.Default); + /// Display Exceptions column. True by default. + public ExceptionDiagnoserAttribute(bool displayExceptionsIfZeroValue = true) + { + Config = ManualConfig.CreateEmpty().AddDiagnoser(new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue))); + } } } diff --git a/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserConfig.cs b/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserConfig.cs new file mode 100644 index 0000000000..86f8d99be4 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserConfig.cs @@ -0,0 +1,20 @@ +using JetBrains.Annotations; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BenchmarkDotNet.Attributes +{ + public class ExceptionDiagnoserConfig + { + /// Determines whether the Exceptions column is displayed when its value is not calculated. True by default. + + [PublicAPI] + public ExceptionDiagnoserConfig(bool displayExceptionsIfZeroValue = true) + { + DisplayExceptionsIfZeroValue = displayExceptionsIfZeroValue; + } + + public bool DisplayExceptionsIfZeroValue { get; } + } +} diff --git a/src/BenchmarkDotNet/Attributes/Exporters/PhdExporterAttribute.cs b/src/BenchmarkDotNet/Attributes/Exporters/PhdExporterAttribute.cs new file mode 100644 index 0000000000..9169c82084 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/Exporters/PhdExporterAttribute.cs @@ -0,0 +1,10 @@ +using System; +using BenchmarkDotNet.Exporters; + +namespace BenchmarkDotNet.Attributes; + +/// +/// IMPORTANT: Not fully implemented yet +/// +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)] +public class PhdExporterAttribute() : ExporterConfigBaseAttribute(new PhdJsonExporter(), new PhdMdExporter()); \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/ThreadingDiagnoserAttribute.cs b/src/BenchmarkDotNet/Attributes/ThreadingDiagnoserAttribute.cs index 7627170b8b..4ad7651bfc 100644 --- a/src/BenchmarkDotNet/Attributes/ThreadingDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/ThreadingDiagnoserAttribute.cs @@ -1,6 +1,7 @@ using System; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; +using JetBrains.Annotations; namespace BenchmarkDotNet.Attributes { @@ -9,6 +10,15 @@ public class ThreadingDiagnoserAttribute : Attribute, IConfigSource { public IConfig Config { get; } - public ThreadingDiagnoserAttribute() => Config = ManualConfig.CreateEmpty().AddDiagnoser(ThreadingDiagnoser.Default); + //public ThreadingDiagnoserAttribute() => Config = ManualConfig.CreateEmpty().AddDiagnoser(ThreadingDiagnoser.Default); + + /// Display configuration for 'LockContentionCount' when it is empty. True (displayed) by default. + /// Display configuration for 'CompletedWorkItemCount' when it is empty. True (displayed) by default. + + [PublicAPI] + public ThreadingDiagnoserAttribute(bool displayLockContentionWhenZero = true, bool displayCompletedWorkItemCountWhenZero = true) + { + Config = ManualConfig.CreateEmpty().AddDiagnoser(new ThreadingDiagnoser(new ThreadingDiagnoserConfig(displayLockContentionWhenZero, displayCompletedWorkItemCountWhenZero))); + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/WakeLockAttribute.cs b/src/BenchmarkDotNet/Attributes/WakeLockAttribute.cs new file mode 100644 index 0000000000..1243ac8f8f --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/WakeLockAttribute.cs @@ -0,0 +1,18 @@ +using BenchmarkDotNet.Configs; +using System; + +namespace BenchmarkDotNet.Attributes +{ + /// + /// Placing a on your assembly or class controls whether the + /// Windows system enters sleep or turns off the display while benchmarks run. + /// + [AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class)] + public sealed class WakeLockAttribute : Attribute, IConfigSource + { + public WakeLockAttribute(WakeLockType wakeLockType) => + Config = ManualConfig.CreateEmpty().WithWakeLock(wakeLockType); + + public IConfig Config { get; } + } +} diff --git a/src/BenchmarkDotNet/BenchmarkDotNet.csproj b/src/BenchmarkDotNet/BenchmarkDotNet.csproj index 50e6733592..1b72486e30 100644 --- a/src/BenchmarkDotNet/BenchmarkDotNet.csproj +++ b/src/BenchmarkDotNet/BenchmarkDotNet.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet - netstandard2.0;net6.0 + netstandard2.0;net6.0;net8.0 true $(NoWarn);1701;1702;1705;1591;3005;NU1702;CS3001;CS3003 BenchmarkDotNet @@ -13,19 +13,17 @@ - - - - + + + - - - + + diff --git a/src/BenchmarkDotNet/Characteristics/Characteristic.cs b/src/BenchmarkDotNet/Characteristics/Characteristic.cs index 6ca88f5f75..d6e5815b4e 100644 --- a/src/BenchmarkDotNet/Characteristics/Characteristic.cs +++ b/src/BenchmarkDotNet/Characteristics/Characteristic.cs @@ -86,7 +86,7 @@ protected Characteristic( private object FallbackValue { get; } - public object this[CharacteristicObject obj] + public object? this[CharacteristicObject obj] { get { return obj.GetValue(this); } set { obj.SetValue(this, value); } diff --git a/src/BenchmarkDotNet/Characteristics/CharacteristicObject.cs b/src/BenchmarkDotNet/Characteristics/CharacteristicObject.cs index a6d7b37b81..6cb4a983c3 100644 --- a/src/BenchmarkDotNet/Characteristics/CharacteristicObject.cs +++ b/src/BenchmarkDotNet/Characteristics/CharacteristicObject.cs @@ -402,6 +402,12 @@ protected CharacteristicObject UnfreezeCopyCore() var newRoot = (CharacteristicObject)Activator.CreateInstance(GetType()); newRoot.ApplyCore(this); + // Preserve the IdCharacteristic of the original object + if (this.HasValue(IdCharacteristic)) + { + newRoot.SetValue(IdCharacteristic, this.GetValue(IdCharacteristic)); + } + return newRoot; } #endregion diff --git a/src/BenchmarkDotNet/Code/CodeGenerator.cs b/src/BenchmarkDotNet/Code/CodeGenerator.cs index 005564b77c..9a6228dd88 100644 --- a/src/BenchmarkDotNet/Code/CodeGenerator.cs +++ b/src/BenchmarkDotNet/Code/CodeGenerator.cs @@ -56,6 +56,7 @@ internal static string Generate(BuildPartition buildPartition) .Replace("$OverheadImplementation$", provider.OverheadImplementation) .Replace("$ConsumeField$", provider.ConsumeField) .Replace("$JobSetDefinition$", GetJobsSetDefinition(benchmark)) + .Replace("$ParamsInitializer$", GetParamsInitializer(benchmark)) .Replace("$ParamsContent$", GetParamsContent(benchmark)) .Replace("$ArgumentsDefinition$", GetArgumentsDefinition(benchmark)) .Replace("$DeclareArgumentFields$", GetDeclareArgumentFields(benchmark)) @@ -186,7 +187,15 @@ private static DeclarationsProvider GetDeclarationsProvider(Descriptor descripto return new NonVoidDeclarationsProvider(descriptor); } + private static string GetParamsInitializer(BenchmarkCase benchmarkCase) + => string.Join( + ", ", + benchmarkCase.Parameters.Items + .Where(parameter => !parameter.IsArgument && !parameter.IsStatic) + .Select(parameter => $"{parameter.Name} = default")); + // internal for tests + internal static string GetParamsContent(BenchmarkCase benchmarkCase) => string.Join( string.Empty, diff --git a/src/BenchmarkDotNet/Code/DeclarationsProvider.cs b/src/BenchmarkDotNet/Code/DeclarationsProvider.cs index ddf78eb572..7528e8ed62 100644 --- a/src/BenchmarkDotNet/Code/DeclarationsProvider.cs +++ b/src/BenchmarkDotNet/Code/DeclarationsProvider.cs @@ -63,7 +63,7 @@ private string GetMethodName(MethodInfo method) (method.ReturnType.GetGenericTypeDefinition() == typeof(Task<>) || method.ReturnType.GetGenericTypeDefinition() == typeof(ValueTask<>)))) { - return $"() => {method.Name}().GetAwaiter().GetResult()"; + return $"() => BenchmarkDotNet.Helpers.AwaitHelper.GetResult({method.Name}())"; } return method.Name; @@ -149,12 +149,10 @@ internal class TaskDeclarationsProvider : VoidDeclarationsProvider { public TaskDeclarationsProvider(Descriptor descriptor) : base(descriptor) { } - // we use GetAwaiter().GetResult() because it's fastest way to obtain the result in blocking way, - // and will eventually throw actual exception, not aggregated one public override string WorkloadMethodDelegate(string passArguments) - => $"({passArguments}) => {{ {Descriptor.WorkloadMethod.Name}({passArguments}).GetAwaiter().GetResult(); }}"; + => $"({passArguments}) => {{ BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments})); }}"; - public override string GetWorkloadMethodCall(string passArguments) => $"{Descriptor.WorkloadMethod.Name}({passArguments}).GetAwaiter().GetResult()"; + public override string GetWorkloadMethodCall(string passArguments) => $"BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments}))"; protected override Type WorkloadMethodReturnType => typeof(void); } @@ -168,11 +166,9 @@ public GenericTaskDeclarationsProvider(Descriptor descriptor) : base(descriptor) protected override Type WorkloadMethodReturnType => Descriptor.WorkloadMethod.ReturnType.GetTypeInfo().GetGenericArguments().Single(); - // we use GetAwaiter().GetResult() because it's fastest way to obtain the result in blocking way, - // and will eventually throw actual exception, not aggregated one public override string WorkloadMethodDelegate(string passArguments) - => $"({passArguments}) => {{ return {Descriptor.WorkloadMethod.Name}({passArguments}).GetAwaiter().GetResult(); }}"; + => $"({passArguments}) => {{ return BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments})); }}"; - public override string GetWorkloadMethodCall(string passArguments) => $"{Descriptor.WorkloadMethod.Name}({passArguments}).GetAwaiter().GetResult()"; + public override string GetWorkloadMethodCall(string passArguments) => $"BenchmarkDotNet.Helpers.AwaitHelper.GetResult({Descriptor.WorkloadMethod.Name}({passArguments}))"; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Code/EnumParam.cs b/src/BenchmarkDotNet/Code/EnumParam.cs index 9d0ac58950..5c0ecf6afd 100644 --- a/src/BenchmarkDotNet/Code/EnumParam.cs +++ b/src/BenchmarkDotNet/Code/EnumParam.cs @@ -24,7 +24,7 @@ private EnumParam(object value, Type type) public string ToSourceCode() => $"({type.GetCorrectCSharpTypeName()})({ToInvariantCultureString()})"; - internal static IParam FromObject(object value, Type type = null) + internal static IParam FromObject(object value, Type? type = null) { type = type ?? value.GetType(); if (!type.IsEnum) diff --git a/src/BenchmarkDotNet/Columns/BaselineCustomColumn.cs b/src/BenchmarkDotNet/Columns/BaselineCustomColumn.cs index b808d988d8..4dabe8e502 100644 --- a/src/BenchmarkDotNet/Columns/BaselineCustomColumn.cs +++ b/src/BenchmarkDotNet/Columns/BaselineCustomColumn.cs @@ -43,7 +43,7 @@ public abstract string GetValue(Summary summary, BenchmarkCase benchmarkCase, St public override string ToString() => ColumnName; public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false; - internal static bool ResultsAreInvalid(Summary summary, BenchmarkCase benchmarkCase, BenchmarkCase baseline) + internal static bool ResultsAreInvalid(Summary summary, BenchmarkCase benchmarkCase, BenchmarkCase? baseline) { return baseline == null || summary[baseline] == null || diff --git a/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs b/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs index 0677f73c77..60bb629ee8 100644 --- a/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs +++ b/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs @@ -46,7 +46,7 @@ public IEnumerable GetColumns(Summary summary) if (NeedToShow(summary, s => s.Percentiles.P95 > s.Mean + 3 * s.StandardDeviation)) yield return StatisticColumn.P95; if (NeedToShow(summary, s => s.N >= 3 && - (!s.GetConfidenceInterval(ConfidenceLevel.L99, s.N).Contains(s.Median) || + (!s.GetConfidenceInterval(ConfidenceLevel.L99).Contains(s.Median) || Math.Abs(s.Median - s.Mean) > s.Mean * 0.2))) yield return StatisticColumn.Median; if (NeedToShow(summary, s => s.StandardDeviation > 1e-9)) diff --git a/src/BenchmarkDotNet/Columns/MetricColumn.cs b/src/BenchmarkDotNet/Columns/MetricColumn.cs index 645891502a..feb615bfb0 100644 --- a/src/BenchmarkDotNet/Columns/MetricColumn.cs +++ b/src/BenchmarkDotNet/Columns/MetricColumn.cs @@ -1,8 +1,9 @@ using System.Linq; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; -using Perfolizer.Common; using Perfolizer.Horology; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Columns { @@ -26,8 +27,8 @@ public class MetricColumn : IColumn public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false; public bool IsAvailable(Summary summary) => summary.Reports.Any(report => - report.Metrics.TryGetValue(descriptor.Id, out var metric) - && metric.Descriptor.GetIsAvailable(metric)); + report.Metrics.TryGetValue(descriptor.Id, out var metric) + && metric.Descriptor.GetIsAvailable(metric)); public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => GetValue(summary, benchmarkCase, SummaryStyle.Default); @@ -43,18 +44,23 @@ public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyl var cultureInfo = summary.GetCultureInfo(); bool printUnits = style.PrintUnitsInContent || style.PrintUnitsInHeader; - UnitPresentation unitPresentation = UnitPresentation.FromVisibility(style.PrintUnitsInContent); + var unitPresentation = new UnitPresentation(style.PrintUnitsInContent, minUnitWidth: 0, gap: true); + string numberFormat = descriptor.NumberFormat; if (printUnits && descriptor.UnitType == UnitType.CodeSize) - return SizeValue.FromBytes((long) metric.Value).ToString(style.CodeSizeUnit, cultureInfo, descriptor.NumberFormat, unitPresentation); + return SizeValue.FromBytes((long)metric.Value).ToString(style.CodeSizeUnit, numberFormat, cultureInfo, unitPresentation); if (printUnits && descriptor.UnitType == UnitType.Size) - return SizeValue.FromBytes((long) metric.Value).ToString(style.SizeUnit, cultureInfo, descriptor.NumberFormat, unitPresentation); + return SizeValue.FromBytes((long)metric.Value).ToString(style.SizeUnit, numberFormat, cultureInfo, unitPresentation); if (printUnits && descriptor.UnitType == UnitType.Time) - return TimeInterval.FromNanoseconds(metric.Value).ToString(style.TimeUnit, cultureInfo, descriptor.NumberFormat, unitPresentation); + { + if (numberFormat.IsBlank()) + numberFormat = "N4"; + return TimeInterval.FromNanoseconds(metric.Value).ToString(style.TimeUnit, numberFormat, cultureInfo, unitPresentation); + } - return metric.Value.ToString(descriptor.NumberFormat, cultureInfo); + return metric.Value.ToString(numberFormat, cultureInfo); } public override string ToString() => descriptor.DisplayName; } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/SizeUnit.cs b/src/BenchmarkDotNet/Columns/SizeUnit.cs deleted file mode 100644 index b919fb7fa0..0000000000 --- a/src/BenchmarkDotNet/Columns/SizeUnit.cs +++ /dev/null @@ -1,75 +0,0 @@ -using System; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using JetBrains.Annotations; - -namespace BenchmarkDotNet.Columns -{ - [SuppressMessage("ReSharper", "InconsistentNaming")] // We want to use "KB", "MB", "GB", "TB" - public class SizeUnit : IEquatable - { - [PublicAPI] public string Name { get; } - [PublicAPI] public string Description { get; } - [PublicAPI] public long ByteAmount { get; } - - public SizeUnit(string name, string description, long byteAmount) - { - Name = name; - Description = description; - ByteAmount = byteAmount; - } - - private const long BytesInKiloByte = 1024L; // this value MUST NOT be changed - - public SizeValue ToValue(long value = 1) => new SizeValue(value, this); - - [PublicAPI] public static readonly SizeUnit B = new SizeUnit("B", "Byte", 1L); - [PublicAPI] public static readonly SizeUnit KB = new SizeUnit("KB", "Kilobyte", BytesInKiloByte); - [PublicAPI] public static readonly SizeUnit MB = new SizeUnit("MB", "Megabyte", BytesInKiloByte * BytesInKiloByte); - [PublicAPI] public static readonly SizeUnit GB = new SizeUnit("GB", "Gigabyte", BytesInKiloByte * BytesInKiloByte * BytesInKiloByte); - [PublicAPI] public static readonly SizeUnit TB = new SizeUnit("TB", "Terabyte", BytesInKiloByte * BytesInKiloByte * BytesInKiloByte * BytesInKiloByte); - [PublicAPI] public static readonly SizeUnit[] All = { B, KB, MB, GB, TB }; - - public static SizeUnit GetBestSizeUnit(params long[] values) - { - if (!values.Any()) - return B; - // Use the largest unit to display the smallest recorded measurement without loss of precision. - long minValue = values.Min(); - foreach (var sizeUnit in All) - { - if (minValue < sizeUnit.ByteAmount * BytesInKiloByte) - return sizeUnit; - } - return All.Last(); - } - - public static double Convert(long value, SizeUnit from, SizeUnit to) => value * (double)from.ByteAmount / (to ?? GetBestSizeUnit(value)).ByteAmount; - - public bool Equals(SizeUnit other) - { - if (ReferenceEquals(null, other)) - return false; - if (ReferenceEquals(this, other)) - return true; - return string.Equals(Name, other.Name) && string.Equals(Description, other.Description) && ByteAmount == other.ByteAmount; - } - - public override bool Equals(object obj) - { - if (ReferenceEquals(null, obj)) - return false; - if (ReferenceEquals(this, obj)) - return true; - if (obj.GetType() != this.GetType()) - return false; - return Equals((SizeUnit) obj); - } - - public override int GetHashCode() => HashCode.Combine(Name, Description, ByteAmount); - - public static bool operator ==(SizeUnit left, SizeUnit right) => Equals(left, right); - - public static bool operator !=(SizeUnit left, SizeUnit right) => !Equals(left, right); - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/SizeValue.cs b/src/BenchmarkDotNet/Columns/SizeValue.cs deleted file mode 100644 index 9478989eb1..0000000000 --- a/src/BenchmarkDotNet/Columns/SizeValue.cs +++ /dev/null @@ -1,61 +0,0 @@ -using System.Globalization; -using BenchmarkDotNet.Helpers; -using JetBrains.Annotations; -using Perfolizer.Common; - -namespace BenchmarkDotNet.Columns -{ - public struct SizeValue - { - public long Bytes { get; } - - public SizeValue(long bytes) => Bytes = bytes; - - public SizeValue(long bytes, SizeUnit unit) : this(bytes * unit.ByteAmount) { } - - public static readonly SizeValue B = SizeUnit.B.ToValue(); - public static readonly SizeValue KB = SizeUnit.KB.ToValue(); - public static readonly SizeValue MB = SizeUnit.MB.ToValue(); - public static readonly SizeValue GB = SizeUnit.GB.ToValue(); - public static readonly SizeValue TB = SizeUnit.TB.ToValue(); - - [Pure] public static SizeValue FromBytes(long value) => value * B; - [Pure] public static SizeValue FromKilobytes(long value) => value * KB; - [Pure] public static SizeValue FromMegabytes(long value) => value * MB; - [Pure] public static SizeValue FromGigabytes(long value) => value * GB; - [Pure] public static SizeValue FromTerabytes(long value) => value * TB; - - [Pure] public static SizeValue operator *(SizeValue value, long k) => new SizeValue(value.Bytes * k); - [Pure] public static SizeValue operator *(long k, SizeValue value) => new SizeValue(value.Bytes * k); - - [Pure] - public string ToString( - CultureInfo? cultureInfo, - string? format = "0.##", - UnitPresentation? unitPresentation = null) - { - return ToString(null, cultureInfo, format, unitPresentation); - } - - [Pure] - public string ToString( - SizeUnit? sizeUnit, - CultureInfo? cultureInfo, - string? format = "0.##", - UnitPresentation? unitPresentation = null) - { - sizeUnit = sizeUnit ?? SizeUnit.GetBestSizeUnit(Bytes); - cultureInfo = cultureInfo ?? DefaultCultureInfo.Instance; - format = format ?? "0.##"; - unitPresentation = unitPresentation ?? UnitPresentation.Default; - double unitValue = SizeUnit.Convert(Bytes, SizeUnit.B, sizeUnit); - if (unitPresentation.IsVisible) - { - string unitName = sizeUnit.Name.PadLeft(unitPresentation.MinUnitWidth); - return $"{unitValue.ToString(format, cultureInfo)} {unitName}"; - } - - return unitValue.ToString(format, cultureInfo); - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/StatisticColumn.cs b/src/BenchmarkDotNet/Columns/StatisticColumn.cs index 31e72f2446..1e1fd63b35 100644 --- a/src/BenchmarkDotNet/Columns/StatisticColumn.cs +++ b/src/BenchmarkDotNet/Columns/StatisticColumn.cs @@ -6,10 +6,10 @@ using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using JetBrains.Annotations; -using Perfolizer.Common; using Perfolizer.Horology; using Perfolizer.Mathematics.Common; using Perfolizer.Mathematics.Multimodality; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Columns { @@ -38,7 +38,7 @@ private enum Priority s => s.StandardDeviation, Priority.Main, parentColumn: Mean); public static readonly IColumn Error = new StatisticColumn(Column.Error, "Half of 99.9% confidence interval", - s => new ConfidenceInterval(s.Mean, s.StandardError, s.N, ConfidenceLevel.L999).Margin, Priority.Main, parentColumn: Mean); + s => s.GetConfidenceInterval(ConfidenceLevel.L999).Margin, Priority.Main, parentColumn: Mean); public static readonly IColumn OperationsPerSecond = new StatisticColumn(Column.OperationPerSecond, "Operation per second", s => 1.0 * 1000 * 1000 * 1000 / s.Mean, Priority.Additional, UnitType.Dimensionless); @@ -67,7 +67,7 @@ private enum Priority /// See http://www.brendangregg.com/FrequencyTrails/modes.html /// public static readonly IColumn MValue = new StatisticColumn(Column.MValue, "Modal value, see http://www.brendangregg.com/FrequencyTrails/modes.html", - s => MValueCalculator.Calculate(s.OriginalValues), Priority.Additional, UnitType.Dimensionless); + s => MValueCalculator.Calculate(s.Sample.Values), Priority.Additional, UnitType.Dimensionless); public static readonly IColumn Iterations = new StatisticColumn(Column.Iterations, "Number of target iterations", s => s.N, Priority.Additional, UnitType.Dimensionless); @@ -84,17 +84,17 @@ private enum Priority [PublicAPI] public static IColumn CiLower(ConfidenceLevel level) => new StatisticColumn( - $"CI{level.ToPercentStr()} Lower", $"Lower bound of {level.ToPercentStr()} confidence interval", + $"CI{level} Lower", $"Lower bound of {level} confidence interval", s => new ConfidenceInterval(s.Mean, s.StandardError, s.N, level).Lower, Priority.Additional); [PublicAPI] public static IColumn CiUpper(ConfidenceLevel level) => new StatisticColumn( - $"CI{level.ToPercentStr()} Upper", $"Upper bound of {level.ToPercentStr()} confidence interval", + $"CI{level} Upper", $"Upper bound of {level} confidence interval", s => new ConfidenceInterval(s.Mean, s.StandardError, s.N, level).Upper, Priority.Additional); [PublicAPI] public static IColumn CiError(ConfidenceLevel level) => new StatisticColumn( - $"CI{level.ToPercentStr()} Margin", $"Half of {level.ToPercentStr()} confidence interval", + $"CI{level} Margin", $"Half of {level} confidence interval", s => new ConfidenceInterval(s.Mean, s.StandardError, s.N, level).Margin, Priority.Additional); @@ -107,7 +107,7 @@ private enum Priority private readonly IStatisticColumn parentColumn; private StatisticColumn(string columnName, string legend, Func calc, Priority priority, UnitType type = UnitType.Time, - IStatisticColumn parentColumn = null) + IStatisticColumn? parentColumn = null) { this.calc = calc; this.priority = priority; @@ -118,33 +118,34 @@ private StatisticColumn(string columnName, string legend, Func Format(summary, benchmarkCase.Config, summary[benchmarkCase].ResultStatistics, SummaryStyle.Default); + => Format(summary, benchmarkCase.Config, summary[benchmarkCase]?.ResultStatistics, SummaryStyle.Default); public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) - => Format(summary, benchmarkCase.Config, summary[benchmarkCase].ResultStatistics, style); + => Format(summary, benchmarkCase.Config, summary[benchmarkCase]?.ResultStatistics, style); public bool IsAvailable(Summary summary) => true; public bool AlwaysShow => true; public ColumnCategory Category => ColumnCategory.Statistics; - public int PriorityInCategory => (int) priority; + public int PriorityInCategory => (int)priority; public bool IsNumeric => true; public UnitType UnitType { get; } public string Legend { get; } public List GetAllValues(Summary summary, SummaryStyle style) - => summary.Reports + { + return summary.Reports .Where(r => r.ResultStatistics != null) .Select(r => calc(r.ResultStatistics)) .Where(v => !double.IsNaN(v) && !double.IsInfinity(v)) - .Select(v => UnitType == UnitType.Time ? v / style.TimeUnit.NanosecondAmount : v) + .Select(v => UnitType == UnitType.Time && style.TimeUnit != null ? v / style.TimeUnit.BaseUnits : v) .ToList(); + } - private string Format(Summary summary, ImmutableConfig config, Statistics statistics, SummaryStyle style) + private string Format(Summary summary, ImmutableConfig config, Statistics? statistics, SummaryStyle style) { if (statistics == null) return "NA"; - int precision = summary.DisplayPrecisionManager.GetPrecision(style, this, parentColumn); string format = "N" + precision; @@ -155,9 +156,9 @@ private string Format(Summary summary, ImmutableConfig config, Statistics statis ? TimeInterval.FromNanoseconds(value) .ToString( style.TimeUnit, - style.CultureInfo, format, - UnitPresentation.FromVisibility(style.PrintUnitsInContent)) + style.CultureInfo, + new UnitPresentation(style.PrintUnitsInContent, minUnitWidth: 0, gap: true)) : value.ToString(format, style.CultureInfo); } diff --git a/src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs b/src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs index 0316b9c5f4..51edbb40f9 100644 --- a/src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs +++ b/src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs @@ -4,50 +4,57 @@ using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; +using Perfolizer.Mathematics.Common; using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; +using Perfolizer.Mathematics.SignificanceTesting.MannWhitney; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Columns { - public class StatisticalTestColumn : BaselineCustomColumn + public class StatisticalTestColumn(Threshold threshold, SignificanceLevel? significanceLevel = null) : BaselineCustomColumn { - public static StatisticalTestColumn Create(StatisticalTestKind kind, Threshold threshold, bool showPValues = false) - => new StatisticalTestColumn(kind, threshold, showPValues); + private static readonly SignificanceLevel DefaultSignificanceLevel = SignificanceLevel.P1E5; - public StatisticalTestKind Kind { get; } - public Threshold Threshold { get; } - public bool ShowPValues { get; } + public static StatisticalTestColumn CreateDefault() => new (new PercentValue(10).ToThreshold()); - public StatisticalTestColumn(StatisticalTestKind kind, Threshold threshold, bool showPValues = false) + public static StatisticalTestColumn Create(Threshold threshold, SignificanceLevel? significanceLevel = null) => new (threshold, significanceLevel); + + public static StatisticalTestColumn Create(string threshold, SignificanceLevel? significanceLevel = null) { - Kind = kind; - Threshold = threshold; - ShowPValues = showPValues; + if (!Threshold.TryParse(threshold, out var parsedThreshold)) + throw new ArgumentException($"Can't parse threshold '{threshold}'"); + return new StatisticalTestColumn(parsedThreshold, significanceLevel); } - public override string Id => nameof(StatisticalTestColumn) + "." + Kind + "." + Threshold + "." + (ShowPValues ? "WithDetails" : "WithoutDetails"); - public override string ColumnName => $"{Kind}({Threshold.ToString().Replace(" ", "")}){(ShowPValues ? "/p-values" : "")}"; + public Threshold Threshold { get; } = threshold; + public SignificanceLevel SignificanceLevel { get; } = significanceLevel ?? DefaultSignificanceLevel; + + public override string Id => $"{nameof(StatisticalTestColumn)}/{Threshold}"; + public override string ColumnName => $"MannWhitney({Threshold})"; public override string GetValue(Summary summary, BenchmarkCase benchmarkCase, Statistics baseline, IReadOnlyDictionary baselineMetrics, Statistics current, IReadOnlyDictionary currentMetrics, bool isBaseline) { - var x = baseline.OriginalValues.ToArray(); - var y = current.OriginalValues.ToArray(); - switch (Kind) + if (baseline.Sample.Values.SequenceEqual(current.Sample.Values)) + return "Baseline"; + if (current.Sample.Size == 1 && baseline.Sample.Size == 1) + return "?"; + + var test = new SimpleEquivalenceTest(MannWhitneyTest.Instance); + var comparisonResult = test.Perform(current.Sample, baseline.Sample, Threshold, SignificanceLevel); + return comparisonResult switch { - case StatisticalTestKind.Welch: - return StatisticalTestHelper.CalculateTost(WelchTest.Instance, x, y, Threshold).ToString(ShowPValues); - case StatisticalTestKind.MannWhitney: - return StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, x, y, Threshold).ToString(ShowPValues); - default: - throw new ArgumentOutOfRangeException(); - } + ComparisonResult.Greater => "Slower", + ComparisonResult.Indistinguishable => "Same", + ComparisonResult.Lesser => "Faster", + _ => throw new ArgumentOutOfRangeException() + }; } - public override int PriorityInCategory => (int) Kind; + public override int PriorityInCategory => 0; public override bool IsNumeric => false; public override UnitType UnitType => UnitType.Dimensionless; - public override string Legend => $"{Kind}-based TOST equivalence test with {Threshold} threshold{(ShowPValues ? ". Format: 'Result: p-value(Slower)|p-value(Faster)'" : "")}"; + public override string Legend => $"MannWhitney-based equivalence test (threshold={Threshold}, alpha = {SignificanceLevel})"; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/TagColumn.cs b/src/BenchmarkDotNet/Columns/TagColumn.cs index e4300f037c..425b4d2ad8 100644 --- a/src/BenchmarkDotNet/Columns/TagColumn.cs +++ b/src/BenchmarkDotNet/Columns/TagColumn.cs @@ -19,7 +19,7 @@ public TagColumn(string columnName, Func getTag) } public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false; - public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => getTag(benchmarkCase.Descriptor.WorkloadMethod.Name); + public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => getTag(benchmarkCase.Descriptor?.WorkloadMethod?.Name ?? ""); public bool IsAvailable(Summary summary) => true; public bool AlwaysShow => true; diff --git a/src/BenchmarkDotNet/Columns/UnitType.cs b/src/BenchmarkDotNet/Columns/UnitType.cs index 606a68bd2d..f9e7815fc4 100644 --- a/src/BenchmarkDotNet/Columns/UnitType.cs +++ b/src/BenchmarkDotNet/Columns/UnitType.cs @@ -1,5 +1,6 @@ namespace BenchmarkDotNet.Columns { + // TODO: migrate to Perfolizer.Metrology public enum UnitType { Dimensionless, diff --git a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs index fa0920c7f9..5177227c2a 100644 --- a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs @@ -7,6 +7,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; @@ -110,6 +111,7 @@ public static class ConfigExtensions [Obsolete("This method will soon be removed, please start using .AddLogicalGroupRules() instead.")] [EditorBrowsable(EditorBrowsableState.Never)] public static IConfig With(this IConfig config, params BenchmarkLogicalGroupRule[] rules) => config.AddLogicalGroupRules(rules); [PublicAPI] public static ManualConfig AddLogicalGroupRules(this IConfig config, params BenchmarkLogicalGroupRule[] rules) => config.With(c => c.AddLogicalGroupRules(rules)); + [PublicAPI] public static ManualConfig AddEventProcessor(this IConfig config, params EventProcessor[] eventProcessors) => config.With(c => c.AddEventProcessor(eventProcessors)); [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params string[] columnNames) => config.With(c => c.HideColumns(columnNames)); [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params IColumn[] columns) => config.With(c => c.HideColumns(columns)); diff --git a/src/BenchmarkDotNet/Configs/ConfigOptions.cs b/src/BenchmarkDotNet/Configs/ConfigOptions.cs index c1ad75d642..06bc7768c7 100644 --- a/src/BenchmarkDotNet/Configs/ConfigOptions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigOptions.cs @@ -48,7 +48,11 @@ public enum ConfigOptions /// /// Continue the execution if the last run was stopped. /// - Resume = 1 << 9 + Resume = 1 << 9, + /// + /// Determines whether parallel build of benchmark projects should be disabled. + /// + DisableParallelBuild = 1 << 10, } internal static class ConfigOptionsExtensions diff --git a/src/BenchmarkDotNet/Configs/DebugConfig.cs b/src/BenchmarkDotNet/Configs/DebugConfig.cs index 0d66540bb1..551c38fa09 100644 --- a/src/BenchmarkDotNet/Configs/DebugConfig.cs +++ b/src/BenchmarkDotNet/Configs/DebugConfig.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; @@ -61,6 +62,7 @@ public abstract class DebugConfig : IConfig public IEnumerable GetDiagnosers() => Array.Empty(); public IEnumerable GetAnalysers() => Array.Empty(); public IEnumerable GetHardwareCounters() => Array.Empty(); + public IEnumerable GetEventProcessors() => Array.Empty(); public IEnumerable GetFilters() => Array.Empty(); public IEnumerable GetColumnHidingRules() => Array.Empty(); @@ -69,6 +71,7 @@ public abstract class DebugConfig : IConfig public SummaryStyle SummaryStyle => SummaryStyle.Default; public ConfigUnionRule UnionRule => ConfigUnionRule.Union; public TimeSpan BuildTimeout => DefaultConfig.Instance.BuildTimeout; + public WakeLockType WakeLock => WakeLockType.None; public string ArtifactsPath => null; // DefaultConfig.ArtifactsPath will be used if the user does not specify it in explicit way diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index 8d1df6285b..8585a77797 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -4,7 +4,9 @@ using System.IO; using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Exporters.Csv; using BenchmarkDotNet.Filters; @@ -85,11 +87,13 @@ public IEnumerable GetValidators() public TimeSpan BuildTimeout => TimeSpan.FromSeconds(120); + public WakeLockType WakeLock => WakeLockType.System; + public string ArtifactsPath { get { - var root = RuntimeInformation.IsAndroid() ? + var root = OsDetector.IsAndroid() ? Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) : Directory.GetCurrentDirectory(); return Path.Combine(root, "BenchmarkDotNet.Artifacts"); @@ -108,6 +112,8 @@ public string ArtifactsPath public IEnumerable GetFilters() => Array.Empty(); + public IEnumerable GetEventProcessors() => Array.Empty(); + public IEnumerable GetColumnHidingRules() => Array.Empty(); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Configs/IConfig.cs b/src/BenchmarkDotNet/Configs/IConfig.cs index ba34cfeddf..9e3bf50ce6 100644 --- a/src/BenchmarkDotNet/Configs/IConfig.cs +++ b/src/BenchmarkDotNet/Configs/IConfig.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; @@ -27,6 +28,7 @@ public interface IConfig IEnumerable GetHardwareCounters(); IEnumerable GetFilters(); IEnumerable GetLogicalGroupRules(); + IEnumerable GetEventProcessors(); IEnumerable GetColumnHidingRules(); IOrderer? Orderer { get; } @@ -53,6 +55,8 @@ public interface IConfig /// TimeSpan BuildTimeout { get; } + public WakeLockType WakeLock { get; } + /// /// Collect any errors or warnings when composing the configuration /// diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs index b5d84eaf56..0043ecbe4b 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs @@ -6,6 +6,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; @@ -31,6 +32,7 @@ public sealed class ImmutableConfig : IConfig private readonly ImmutableHashSet hardwareCounters; private readonly ImmutableHashSet filters; private readonly ImmutableArray rules; + private readonly ImmutableHashSet eventProcessors; private readonly ImmutableArray columnHidingRules; internal ImmutableConfig( @@ -45,6 +47,7 @@ internal ImmutableConfig( ImmutableArray uniqueRules, ImmutableArray uniqueColumnHidingRules, ImmutableHashSet uniqueRunnableJobs, + ImmutableHashSet uniqueEventProcessors, ConfigUnionRule unionRule, string artifactsPath, CultureInfo cultureInfo, @@ -53,6 +56,7 @@ internal ImmutableConfig( SummaryStyle summaryStyle, ConfigOptions options, TimeSpan buildTimeout, + WakeLockType wakeLock, IReadOnlyList configAnalysisConclusion) { columnProviders = uniqueColumnProviders; @@ -66,6 +70,7 @@ internal ImmutableConfig( rules = uniqueRules; columnHidingRules = uniqueColumnHidingRules; jobs = uniqueRunnableJobs; + eventProcessors = uniqueEventProcessors; UnionRule = unionRule; ArtifactsPath = artifactsPath; CultureInfo = cultureInfo; @@ -74,6 +79,7 @@ internal ImmutableConfig( SummaryStyle = summaryStyle; Options = options; BuildTimeout = buildTimeout; + WakeLock = wakeLock; ConfigAnalysisConclusion = configAnalysisConclusion; } @@ -85,6 +91,7 @@ internal ImmutableConfig( public ICategoryDiscoverer CategoryDiscoverer { get; } public SummaryStyle SummaryStyle { get; } public TimeSpan BuildTimeout { get; } + public WakeLockType WakeLock { get; } public IEnumerable GetColumnProviders() => columnProviders; public IEnumerable GetExporters() => exporters; @@ -96,6 +103,7 @@ internal ImmutableConfig( public IEnumerable GetHardwareCounters() => hardwareCounters; public IEnumerable GetFilters() => filters; public IEnumerable GetLogicalGroupRules() => rules; + public IEnumerable GetEventProcessors() => eventProcessors; public IEnumerable GetColumnHidingRules() => columnHidingRules; public ILogger GetCompositeLogger() => new CompositeLogger(loggers); @@ -114,7 +122,7 @@ internal ImmutableConfig( public bool HasExtraStatsDiagnoser() => HasMemoryDiagnoser() || HasThreadingDiagnoser() || HasExceptionDiagnoser(); - public IDiagnoser GetCompositeDiagnoser(BenchmarkCase benchmarkCase, RunMode runMode) + public IDiagnoser? GetCompositeDiagnoser(BenchmarkCase benchmarkCase, RunMode runMode) { var diagnosersForGivenMode = diagnosers.Where(diagnoser => diagnoser.GetRunMode(benchmarkCase) == runMode).ToImmutableHashSet(); diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index 0540dd1426..f93e5590d0 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -53,6 +53,7 @@ public static ImmutableConfig Create(IConfig source) var uniqueHidingRules = source.GetColumnHidingRules().ToImmutableArray(); var uniqueRunnableJobs = GetRunnableJobs(source.GetJobs()).ToImmutableHashSet(); + var uniqueEventProcessors = source.GetEventProcessors().ToImmutableHashSet(); return new ImmutableConfig( uniqueColumnProviders, @@ -66,6 +67,7 @@ public static ImmutableConfig Create(IConfig source) uniqueRules, uniqueHidingRules, uniqueRunnableJobs, + uniqueEventProcessors, source.UnionRule, source.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath, source.CultureInfo, @@ -74,6 +76,7 @@ public static ImmutableConfig Create(IConfig source) source.SummaryStyle ?? SummaryStyle.Default, source.Options, source.BuildTimeout, + source.WakeLock, configAnalyse.AsReadOnly() ); } diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs index 27eca81863..cdfb64a234 100644 --- a/src/BenchmarkDotNet/Configs/ManualConfig.cs +++ b/src/BenchmarkDotNet/Configs/ManualConfig.cs @@ -6,6 +6,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Filters; @@ -33,6 +34,7 @@ public class ManualConfig : IConfig private readonly HashSet hardwareCounters = new HashSet(); private readonly List filters = new List(); private readonly List logicalGroupRules = new List(); + private readonly List eventProcessors = new List(); private readonly List columnHidingRules = new List(); public IEnumerable GetColumnProviders() => columnProviders; @@ -45,6 +47,7 @@ public class ManualConfig : IConfig public IEnumerable GetHardwareCounters() => hardwareCounters; public IEnumerable GetFilters() => filters; public IEnumerable GetLogicalGroupRules() => logicalGroupRules; + public IEnumerable GetEventProcessors() => eventProcessors; public IEnumerable GetColumnHidingRules() => columnHidingRules; [PublicAPI] public ConfigOptions Options { get; set; } @@ -55,6 +58,7 @@ public class ManualConfig : IConfig [PublicAPI] public ICategoryDiscoverer CategoryDiscoverer { get; set; } [PublicAPI] public SummaryStyle SummaryStyle { get; set; } [PublicAPI] public TimeSpan BuildTimeout { get; set; } = DefaultConfig.Instance.BuildTimeout; + [PublicAPI] public WakeLockType WakeLock { get; set; } = DefaultConfig.Instance.WakeLock; public IReadOnlyList ConfigAnalysisConclusion => emptyConclusion; @@ -106,6 +110,12 @@ public ManualConfig WithBuildTimeout(TimeSpan buildTimeout) return this; } + public ManualConfig WithWakeLock(WakeLockType wakeLockType) + { + WakeLock = wakeLockType; + return this; + } + [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("This method will soon be removed, please start using .AddColumn() instead.")] public void Add(params IColumn[] newColumns) => AddColumn(newColumns); @@ -221,6 +231,12 @@ public ManualConfig AddLogicalGroupRules(params BenchmarkLogicalGroupRule[] rule return this; } + public ManualConfig AddEventProcessor(params EventProcessor[] newEventProcessors) + { + this.eventProcessors.AddRange(newEventProcessors); + return this; + } + [PublicAPI] public ManualConfig HideColumns(params string[] columnNames) { @@ -254,6 +270,7 @@ public void Add(IConfig config) validators.AddRange(config.GetValidators()); hardwareCounters.AddRange(config.GetHardwareCounters()); filters.AddRange(config.GetFilters()); + eventProcessors.AddRange(config.GetEventProcessors()); Orderer = config.Orderer ?? Orderer; CategoryDiscoverer = config.CategoryDiscoverer ?? CategoryDiscoverer; ArtifactsPath = config.ArtifactsPath ?? ArtifactsPath; @@ -263,6 +280,7 @@ public void Add(IConfig config) columnHidingRules.AddRange(config.GetColumnHidingRules()); Options |= config.Options; BuildTimeout = GetBuildTimeout(BuildTimeout, config.BuildTimeout); + WakeLock = GetWakeLock(WakeLock, config.WakeLock); } /// @@ -317,5 +335,12 @@ private static TimeSpan GetBuildTimeout(TimeSpan current, TimeSpan other) => current == DefaultConfig.Instance.BuildTimeout ? other : TimeSpan.FromMilliseconds(Math.Max(current.TotalMilliseconds, other.TotalMilliseconds)); + + private static WakeLockType GetWakeLock(WakeLockType current, WakeLockType other) + { + if (current == DefaultConfig.Instance.WakeLock) { return other; } + if (other == DefaultConfig.Instance.WakeLock) { return current; } + return current.CompareTo(other) > 0 ? current : other; + } } } diff --git a/src/BenchmarkDotNet/Configs/WakeLockType.cs b/src/BenchmarkDotNet/Configs/WakeLockType.cs new file mode 100644 index 0000000000..f547e44764 --- /dev/null +++ b/src/BenchmarkDotNet/Configs/WakeLockType.cs @@ -0,0 +1,20 @@ +namespace BenchmarkDotNet.Configs +{ + public enum WakeLockType + { + /// + /// Allows the system to enter sleep and/or turn off the display while benchmarks are running. + /// + None, + + /// + /// Forces the system to be in the working state while benchmarks are running. + /// + System, + + /// + /// Forces the display to be on while benchmarks are running. + /// + Display + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index e3aed1fedd..6c7b48b0e2 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -2,6 +2,7 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; +using BenchmarkDotNet.Configs; using BenchmarkDotNet.ConsoleArguments.ListBenchmarks; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; @@ -179,6 +180,9 @@ public bool UseDisassemblyDiagnoser [Option("buildTimeout", Required = false, HelpText = "Build timeout in seconds.")] public int? TimeOutInSeconds { get; set; } + [Option("wakeLock", Required = false, HelpText = "Prevents the system from entering sleep or turning off the display. None/System/Display.")] + public WakeLockType? WakeLock { get; set; } + [Option("stopOnFirstError", Required = false, Default = false, HelpText = "Stop on first error.")] public bool StopOnFirstError { get; set; } @@ -237,8 +241,8 @@ public static IEnumerable Examples yield return new Example("Use Job.ShortRun for running the benchmarks", shortName, new CommandLineOptions { BaseJob = "short" }); yield return new Example("Run benchmarks in process", shortName, new CommandLineOptions { RunInProcess = true }); - yield return new Example("Run benchmarks for .NET 4.7.2, .NET Core 2.1 and Mono. .NET 4.7.2 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "net472", "netcoreapp2.1", "Mono" } }); - yield return new Example("Run benchmarks for .NET Core 2.0, .NET Core 2.1 and .NET Core 2.2. .NET Core 2.0 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "netcoreapp2.0", "netcoreapp2.1", "netcoreapp2.2" } }); + yield return new Example("Run benchmarks for .NET 4.7.2, .NET 8.0 and Mono. .NET 4.7.2 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "net472", "net8.0", "Mono" } }); + yield return new Example("Run benchmarks for .NET Core 3.1, .NET 6.0 and .NET 8.0. .NET Core 3.1 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "netcoreapp3.1", "net6.0", "net8.0" } }); yield return new Example("Use MemoryDiagnoser to get GC stats", shortName, new CommandLineOptions { UseMemoryDiagnoser = true }); yield return new Example("Use DisassemblyDiagnoser to get disassembly", shortName, new CommandLineOptions { UseDisassemblyDiagnoser = true }); yield return new Example("Use HardwareCountersDiagnoser to get hardware counter info", longName, new CommandLineOptions { HardwareCounters = new[] { nameof(HardwareCounter.CacheMisses), nameof(HardwareCounter.InstructionRetired) } }); @@ -250,8 +254,8 @@ public static IEnumerable Examples yield return new Example("Run selected benchmarks once per iteration", longName, new CommandLineOptions { RunOncePerIteration = true }); yield return new Example("Run selected benchmarks 100 times per iteration. Perform single warmup iteration and 5 actual workload iterations", longName, new CommandLineOptions { InvocationCount = 100, WarmupIterationCount = 1, IterationCount = 5}); yield return new Example("Run selected benchmarks 250ms per iteration. Perform from 9 to 15 iterations", longName, new CommandLineOptions { IterationTimeInMilliseconds = 250, MinIterationCount = 9, MaxIterationCount = 15}); - yield return new Example("Run MannWhitney test with relative ratio of 5% for all benchmarks for .NET Core 2.0 (base) vs .NET Core 2.1 (diff). .NET Core 2.0 will be baseline because it was provided as first.", longName, - new CommandLineOptions { Filters = new[] { "*"}, Runtimes = new[] { "netcoreapp2.0", "netcoreapp2.1" }, StatisticalTestThreshold = "5%" }); + yield return new Example("Run MannWhitney test with relative ratio of 5% for all benchmarks for .NET 6.0 (base) vs .NET 8.0 (diff). .NET Core 6.0 will be baseline because it was provided as first.", longName, + new CommandLineOptions { Filters = new[] { "*"}, Runtimes = new[] { "net6.0", "net8.0" }, StatisticalTestThreshold = "5%" }); yield return new Example("Run benchmarks using environment variables 'ENV_VAR_KEY_1' with value 'value_1' and 'ENV_VAR_KEY_2' with value 'value_2'", longName, new CommandLineOptions { EnvironmentVariables = new[] { "ENV_VAR_KEY_1:value_1", "ENV_VAR_KEY_2:value_2" } }); yield return new Example("Hide Mean and Ratio columns (use double quotes for multi-word columns: \"Alloc Ratio\")", shortName, new CommandLineOptions { HiddenColumns = new[] { "Mean", "Ratio" }, }); diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 76500bed44..a77a4796f8 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -28,9 +28,8 @@ using CommandLine; using Perfolizer.Horology; using Perfolizer.Mathematics.OutlierDetection; -using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; using BenchmarkDotNet.Toolchains.Mono; +using Perfolizer.Metrology; namespace BenchmarkDotNet.ConsoleArguments { @@ -72,7 +71,7 @@ public static class ConfigParser { "fullxml", new[] { XmlExporter.Full } } }; - public static (bool isSuccess, IConfig config, CommandLineOptions options) Parse(string[] args, ILogger logger, IConfig globalConfig = null) + public static (bool isSuccess, IConfig config, CommandLineOptions options) Parse(string[] args, ILogger logger, IConfig? globalConfig = null) { (bool isSuccess, IConfig config, CommandLineOptions options) result = default; @@ -361,7 +360,7 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo if (options.DisplayAllStatistics) config.AddColumn(StatisticColumn.AllStatistics); if (!string.IsNullOrEmpty(options.StatisticalTestThreshold) && Threshold.TryParse(options.StatisticalTestThreshold, out var threshold)) - config.AddColumn(new StatisticalTestColumn(StatisticalTestKind.MannWhitney, threshold)); + config.AddColumn(new StatisticalTestColumn(threshold)); if (options.ArtifactsDirectory != null) config.ArtifactsPath = options.ArtifactsDirectory.FullName; @@ -393,6 +392,9 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo if (options.TimeOutInSeconds.HasValue) config.WithBuildTimeout(TimeSpan.FromSeconds(options.TimeOutInSeconds.Value)); + if (options.WakeLock.HasValue) + config.WithWakeLock(options.WakeLock.Value); + return config; } @@ -532,6 +534,8 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.Net60: case RuntimeMoniker.Net70: case RuntimeMoniker.Net80: + case RuntimeMoniker.Net90: + case RuntimeMoniker.Net10_0: return baseJob .WithRuntime(runtimeMoniker.GetRuntime()) .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings(runtimeId, null, runtimeId, options.CliPath?.FullName, options.RestorePath?.FullName))); @@ -543,10 +547,16 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma return CreateAotJob(baseJob, options, runtimeMoniker, "6.0.0-*", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json"); case RuntimeMoniker.NativeAot70: - return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json"); + return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://api.nuget.org/v3/index.json"); case RuntimeMoniker.NativeAot80: - return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json"); + return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://api.nuget.org/v3/index.json"); + + case RuntimeMoniker.NativeAot90: + return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json"); + + case RuntimeMoniker.NativeAot10_0: + return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json"); case RuntimeMoniker.Wasm: return MakeWasmJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net5.0", runtimeMoniker); @@ -563,17 +573,29 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.WasmNet80: return MakeWasmJob(baseJob, options, "net8.0", runtimeMoniker); + case RuntimeMoniker.WasmNet90: + return MakeWasmJob(baseJob, options, "net9.0", runtimeMoniker); + + case RuntimeMoniker.WasmNet10_0: + return MakeWasmJob(baseJob, options, "net10.0", runtimeMoniker); + case RuntimeMoniker.MonoAOTLLVM: - return MakeMonoAOTLLVMJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net6.0"); + return MakeMonoAOTLLVMJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net6.0", runtimeMoniker); case RuntimeMoniker.MonoAOTLLVMNet60: - return MakeMonoAOTLLVMJob(baseJob, options, "net6.0"); + return MakeMonoAOTLLVMJob(baseJob, options, "net6.0", runtimeMoniker); case RuntimeMoniker.MonoAOTLLVMNet70: - return MakeMonoAOTLLVMJob(baseJob, options, "net7.0"); + return MakeMonoAOTLLVMJob(baseJob, options, "net7.0", runtimeMoniker); case RuntimeMoniker.MonoAOTLLVMNet80: - return MakeMonoAOTLLVMJob(baseJob, options, "net8.0"); + return MakeMonoAOTLLVMJob(baseJob, options, "net8.0", runtimeMoniker); + + case RuntimeMoniker.MonoAOTLLVMNet90: + return MakeMonoAOTLLVMJob(baseJob, options, "net9.0", runtimeMoniker); + + case RuntimeMoniker.MonoAOTLLVMNet10_0: + return MakeMonoAOTLLVMJob(baseJob, options, "net10.0", runtimeMoniker); case RuntimeMoniker.Mono60: return MakeMonoJob(baseJob, options, MonoRuntime.Mono60); @@ -584,6 +606,12 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.Mono80: return MakeMonoJob(baseJob, options, MonoRuntime.Mono80); + case RuntimeMoniker.Mono90: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono90); + + case RuntimeMoniker.Mono10_0: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono10_0); + default: throw new NotSupportedException($"Runtime {runtimeId} is not supported"); } @@ -624,9 +652,9 @@ private static Job MakeMonoJob(Job baseJob, CommandLineOptions options, MonoRunt packagesPath: options.RestorePath?.FullName))); } - private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, string msBuildMoniker) + private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, string msBuildMoniker, RuntimeMoniker moniker) { - var monoAotLLVMRuntime = new MonoAotLLVMRuntime(aotCompilerPath: options.AOTCompilerPath, aotCompilerMode: options.AOTCompilerMode, msBuildMoniker: msBuildMoniker); + var monoAotLLVMRuntime = new MonoAotLLVMRuntime(aotCompilerPath: options.AOTCompilerPath, aotCompilerMode: options.AOTCompilerMode, msBuildMoniker: msBuildMoniker, moniker: moniker); var toolChain = MonoAotLLVMToolChain.From( new NetCoreAppSettings( @@ -750,13 +778,20 @@ private static string GetCoreRunToolchainDisplayName(IReadOnlyList pat return coreRunPath.FullName.Substring(lastCommonDirectorySeparatorIndex); } - private static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker) + internal static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker) { int index = runtime.IndexOf('-'); + if (index >= 0) + { + runtime = runtime.Substring(0, index); + } - return index < 0 - ? Enum.TryParse(runtime.Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker) - : Enum.TryParse(runtime.Substring(0, index).Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker); + // Monikers older than Net 10 don't use any version delimiter, newer monikers use _ delimiter. + if (Enum.TryParse(runtime.Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker)) + { + return true; + } + return Enum.TryParse(runtime.Replace('.', '_'), ignoreCase: true, out runtimeMoniker); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/HardwareIntrinsics.cs b/src/BenchmarkDotNet/Detectors/Cpu/HardwareIntrinsics.cs similarity index 81% rename from src/BenchmarkDotNet/Portability/Cpu/HardwareIntrinsics.cs rename to src/BenchmarkDotNet/Detectors/Cpu/HardwareIntrinsics.cs index 68c48809b7..a8c7655272 100644 --- a/src/BenchmarkDotNet/Portability/Cpu/HardwareIntrinsics.cs +++ b/src/BenchmarkDotNet/Detectors/Cpu/HardwareIntrinsics.cs @@ -1,14 +1,15 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Numerics; +using System.Text; using BenchmarkDotNet.Environments; -using System.Diagnostics.CodeAnalysis; #if NET6_0_OR_GREATER using System.Runtime.Intrinsics.X86; using System.Runtime.Intrinsics.Arm; #endif -namespace BenchmarkDotNet.Portability.Cpu +namespace BenchmarkDotNet.Detectors.Cpu { internal static class HardwareIntrinsics { @@ -16,6 +17,8 @@ internal static class HardwareIntrinsics internal static string GetShortInfo() { + if (IsX86Avx512FSupported) + return GetShortAvx512Representation(); if (IsX86Avx2Supported) return "AVX2"; else if (IsX86AvxSupported) @@ -52,7 +55,9 @@ static IEnumerable GetCurrentProcessInstructionSets(Platform platform) { case Platform.X86: case Platform.X64: - if (IsX86Avx2Supported) yield return "AVX2"; + + if (IsX86Avx512FSupported) yield return GetShortAvx512Representation(); + else if (IsX86Avx2Supported) yield return "AVX2"; else if (IsX86AvxSupported) yield return "AVX"; else if (IsX86Sse42Supported) yield return "SSE4.2"; else if (IsX86Sse41Supported) yield return "SSE4.1"; @@ -90,6 +95,18 @@ static IEnumerable GetCurrentProcessInstructionSets(Platform platform) } } + private static string GetShortAvx512Representation() + { + StringBuilder avx512 = new ("AVX-512F"); + if (IsX86Avx512CDSupported) avx512.Append("+CD"); + if (IsX86Avx512BWSupported) avx512.Append("+BW"); + if (IsX86Avx512DQSupported) avx512.Append("+DQ"); + if (IsX86Avx512FVLSupported) avx512.Append("+VL"); + if (IsX86Avx512VbmiSupported) avx512.Append("+VBMI"); + + return avx512.ToString(); + } + internal static bool IsX86BaseSupported => #if NET6_0_OR_GREATER X86Base.IsSupported; @@ -153,6 +170,48 @@ static IEnumerable GetCurrentProcessInstructionSets(Platform platform) GetIsSupported("System.Runtime.Intrinsics.X86.Avx2"); #endif + internal static bool IsX86Avx512FSupported => +#if NET8_0_OR_GREATER + Avx512F.IsSupported; +#else + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512F"); +#endif + + internal static bool IsX86Avx512FVLSupported => +#if NET8_0_OR_GREATER + Avx512F.VL.IsSupported; +#else + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512F+VL"); +#endif + + internal static bool IsX86Avx512BWSupported => +#if NET8_0_OR_GREATER + Avx512BW.IsSupported; +#else + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512BW"); +#endif + + internal static bool IsX86Avx512CDSupported => +#if NET8_0_OR_GREATER + Avx512CD.IsSupported; +#else + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512CD"); +#endif + + internal static bool IsX86Avx512DQSupported => +#if NET8_0_OR_GREATER + Avx512DQ.IsSupported; +#else + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512DQ"); +#endif + + internal static bool IsX86Avx512VbmiSupported => +#if NET8_0_OR_GREATER + Avx512Vbmi.IsSupported; +#else + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512Vbmi"); +#endif + internal static bool IsX86AesSupported => #if NET6_0_OR_GREATER System.Runtime.Intrinsics.X86.Aes.IsSupported; @@ -211,8 +270,12 @@ static IEnumerable GetCurrentProcessInstructionSets(Platform platform) GetIsSupported("System.Runtime.Intrinsics.X86.AvxVnni"); #endif - // X86Serialize was introduced in .NET 7.0, BDN does not target it so we need to use reflection - internal static bool IsX86SerializeSupported => GetIsSupported("System.Runtime.Intrinsics.X86.X86Serialize"); + internal static bool IsX86SerializeSupported => +#if NET7_0_OR_GREATER + X86Serialize.IsSupported; +#else + GetIsSupported("System.Runtime.Intrinsics.X86.X86Serialize"); +#endif internal static bool IsArmBaseSupported => #if NET6_0_OR_GREATER diff --git a/src/BenchmarkDotNet/Detectors/Cpu/ICpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/ICpuDetector.cs new file mode 100644 index 0000000000..6c0fcf683c --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/ICpuDetector.cs @@ -0,0 +1,12 @@ +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Detectors.Cpu; + +/// +/// Loads the for the current hardware +/// +public interface ICpuDetector +{ + bool IsApplicable(); + PhdCpu? Detect(); +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuDetector.cs new file mode 100644 index 0000000000..fd3c98d049 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuDetector.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Portability; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Detectors.Cpu.Linux; + +/// +/// CPU information from output of the `cat /proc/cpuinfo` and `lscpu` command. +/// Linux only. +/// +internal class LinuxCpuDetector : ICpuDetector +{ + public bool IsApplicable() => OsDetector.IsLinux(); + + public PhdCpu? Detect() + { + if (!IsApplicable()) return null; + + // lscpu output respects the system locale, so we should force language invariant environment for correct parsing + var languageInvariantEnvironment = new Dictionary + { + ["LC_ALL"] = "C", + ["LANG"] = "C", + ["LANGUAGE"] = "C" + }; + + string cpuInfo = ProcessHelper.RunAndReadOutput("cat", "/proc/cpuinfo") ?? ""; + string lscpu = ProcessHelper.RunAndReadOutput("/bin/bash", "-c \"lscpu\"", environmentVariables: languageInvariantEnvironment); + return LinuxCpuInfoParser.Parse(cpuInfo, lscpu); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuInfoParser.cs b/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuInfoParser.cs new file mode 100644 index 0000000000..b1e554a6fd --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuInfoParser.cs @@ -0,0 +1,115 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Portability; +using Perfolizer.Horology; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Detectors.Cpu.Linux; + +internal static class LinuxCpuInfoParser +{ + private static class ProcCpu + { + internal const string PhysicalId = "physical id"; + internal const string CpuCores = "cpu cores"; + internal const string ModelName = "model name"; + internal const string MaxFrequency = "max freq"; + } + + private static class Lscpu + { + internal const string MaxFrequency = "CPU max MHz"; + internal const string ModelName = "Model name"; + internal const string CoresPerSocket = "Core(s) per socket"; + } + + /// Output of `cat /proc/cpuinfo` + /// Output of `lscpu` + internal static PhdCpu Parse(string? cpuInfo, string? lscpu) + { + var processorModelNames = new HashSet(); + var processorsToPhysicalCoreCount = new Dictionary(); + int logicalCoreCount = 0; + Frequency? maxFrequency = null; + + var logicalCores = SectionsHelper.ParseSections(cpuInfo, ':'); + foreach (var logicalCore in logicalCores) + { + if (logicalCore.TryGetValue(ProcCpu.PhysicalId, out string physicalId) && + logicalCore.TryGetValue(ProcCpu.CpuCores, out string cpuCoresValue) && + int.TryParse(cpuCoresValue, out int cpuCoreCount) && + cpuCoreCount > 0) + processorsToPhysicalCoreCount[physicalId] = cpuCoreCount; + + if (logicalCore.TryGetValue(ProcCpu.ModelName, out string modelName)) + { + processorModelNames.Add(modelName); + logicalCoreCount++; + } + + if (logicalCore.TryGetValue(ProcCpu.MaxFrequency, out string maxCpuFreqValue) && + Frequency.TryParseMHz(maxCpuFreqValue, out var maxCpuFreq)) + { + maxFrequency = maxCpuFreq; + } + } + + int? coresPerSocket = null; + if (lscpu != null) + { + var lscpuParts = lscpu.Split('\n') + .Where(line => line.Contains(':')) + .SelectMany(line => line.Split([':'], 2)) + .ToList(); + for (int i = 0; i + 1 < lscpuParts.Count; i += 2) + { + string name = lscpuParts[i].Trim(); + string value = lscpuParts[i + 1].Trim(); + + if (name.EqualsWithIgnoreCase(Lscpu.MaxFrequency) && + Frequency.TryParseMHz(value.Replace(',', '.'), out var maxFrequencyParsed)) // Example: `CPU max MHz: 3200,0000` + maxFrequency = maxFrequencyParsed; + + if (name.EqualsWithIgnoreCase(Lscpu.ModelName)) + processorModelNames.Add(value); + + if (name.EqualsWithIgnoreCase(Lscpu.CoresPerSocket) && + int.TryParse(value, out int coreCount)) + coresPerSocket = coreCount; + } + } + + var nominalFrequency = processorModelNames + .Select(ParseFrequencyFromBrandString) + .WhereNotNull() + .FirstOrDefault() ?? maxFrequency; + string processorName = processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null; + int? physicalProcessorCount = processorsToPhysicalCoreCount.Count > 0 ? processorsToPhysicalCoreCount.Count : null; + int? physicalCoreCount = processorsToPhysicalCoreCount.Count > 0 ? processorsToPhysicalCoreCount.Values.Sum() : coresPerSocket; + return new PhdCpu + { + ProcessorName = processorName, + PhysicalProcessorCount = physicalProcessorCount, + PhysicalCoreCount = physicalCoreCount, + LogicalCoreCount = logicalCoreCount > 0 ? logicalCoreCount : null, + NominalFrequencyHz = nominalFrequency?.Hertz.RoundToLong(), + MaxFrequencyHz = maxFrequency?.Hertz.RoundToLong() + }; + } + + internal static Frequency? ParseFrequencyFromBrandString(string brandString) + { + const string pattern = "(\\d.\\d+)GHz"; + var matches = Regex.Matches(brandString, pattern, RegexOptions.IgnoreCase); + if (matches.Count > 0 && matches[0].Groups.Count > 1) + { + string match = Regex.Matches(brandString, pattern, RegexOptions.IgnoreCase)[0].Groups[1].ToString(); + return Frequency.TryParseGHz(match, out var result) ? result : null; + } + + return null; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/MosCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/MosCpuDetector.cs new file mode 100644 index 0000000000..8c0b2718eb --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/MosCpuDetector.cs @@ -0,0 +1,64 @@ +using System.Collections.Generic; +using System.Linq; +using System.Management; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Portability; +using Perfolizer.Horology; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +internal class MosCpuDetector : ICpuDetector +{ +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatform("windows")] +#endif + public bool IsApplicable() => OsDetector.IsWindows() && + RuntimeInformation.IsFullFramework && + !RuntimeInformation.IsMono; + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatform("windows")] +#endif + public PhdCpu? Detect() + { + if (!IsApplicable()) return null; + + var processorModelNames = new HashSet(); + int physicalCoreCount = 0; + int logicalCoreCount = 0; + int processorsCount = 0; + int sumMaxFrequency = 0; + + using (var mosProcessor = new ManagementObjectSearcher("SELECT * FROM Win32_Processor")) + { + foreach (var moProcessor in mosProcessor.Get().Cast()) + { + string name = moProcessor[WmicCpuInfoKeyNames.Name]?.ToString(); + if (!string.IsNullOrEmpty(name)) + { + processorModelNames.Add(name); + processorsCount++; + physicalCoreCount += (int)(uint)moProcessor[WmicCpuInfoKeyNames.NumberOfCores]; + logicalCoreCount += (int)(uint)moProcessor[WmicCpuInfoKeyNames.NumberOfLogicalProcessors]; + sumMaxFrequency = (int)(uint)moProcessor[WmicCpuInfoKeyNames.MaxClockSpeed]; + } + } + } + + string processorName = processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null; + Frequency? maxFrequency = sumMaxFrequency > 0 && processorsCount > 0 + ? Frequency.FromMHz(sumMaxFrequency * 1.0 / processorsCount) + : null; + + return new PhdCpu + { + ProcessorName = processorName, + PhysicalProcessorCount = processorsCount > 0 ? processorsCount : null, + PhysicalCoreCount = physicalCoreCount > 0 ? physicalCoreCount : null, + LogicalCoreCount = logicalCoreCount > 0 ? logicalCoreCount : null, + NominalFrequencyHz = maxFrequency?.Hertz.RoundToLong(), + MaxFrequencyHz = maxFrequency?.Hertz.RoundToLong() + }; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/WindowsCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WindowsCpuDetector.cs new file mode 100644 index 0000000000..9969a1bca0 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WindowsCpuDetector.cs @@ -0,0 +1,3 @@ +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +internal class WindowsCpuDetector() : CpuDetector(new MosCpuDetector(), new WmicCpuDetector()); \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuDetector.cs new file mode 100644 index 0000000000..e7ae5401bc --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuDetector.cs @@ -0,0 +1,30 @@ +using System.IO; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Portability; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +/// +/// CPU information from output of the `wmic cpu get Name, NumberOfCores, NumberOfLogicalProcessors /Format:List` command. +/// Windows only. +/// +internal class WmicCpuDetector : ICpuDetector +{ + private const string DefaultWmicPath = @"C:\Windows\System32\wbem\WMIC.exe"; + + public bool IsApplicable() => OsDetector.IsWindows(); + + public PhdCpu? Detect() + { + if (!IsApplicable()) return null; + + const string argList = $"{WmicCpuInfoKeyNames.Name}, " + + $"{WmicCpuInfoKeyNames.NumberOfCores}, " + + $"{WmicCpuInfoKeyNames.NumberOfLogicalProcessors}, " + + $"{WmicCpuInfoKeyNames.MaxClockSpeed}"; + string wmicPath = File.Exists(DefaultWmicPath) ? DefaultWmicPath : "wmic"; + string wmicOutput = ProcessHelper.RunAndReadOutput(wmicPath, $"cpu get {argList} /Format:List"); + return WmicCpuInfoParser.Parse(wmicOutput); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoKeyNames.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoKeyNames.cs new file mode 100644 index 0000000000..65d8ecf7d8 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoKeyNames.cs @@ -0,0 +1,9 @@ +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +internal static class WmicCpuInfoKeyNames +{ + internal const string NumberOfLogicalProcessors = "NumberOfLogicalProcessors"; + internal const string NumberOfCores = "NumberOfCores"; + internal const string Name = "Name"; + internal const string MaxClockSpeed = "MaxClockSpeed"; +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoParser.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoParser.cs new file mode 100644 index 0000000000..af6ed8c12e --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoParser.cs @@ -0,0 +1,65 @@ +using System.Collections.Generic; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using Perfolizer.Horology; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +internal static class WmicCpuInfoParser +{ + /// + /// Parses wmic output and returns + /// + /// Output of `wmic cpu get Name, NumberOfCores, NumberOfLogicalProcessors /Format:List` + internal static PhdCpu Parse(string? wmicOutput) + { + var processorModelNames = new HashSet(); + int physicalCoreCount = 0; + int logicalCoreCount = 0; + int processorsCount = 0; + var sumMaxFrequency = Frequency.Zero; + + var processors = SectionsHelper.ParseSections(wmicOutput, '='); + foreach (var processor in processors) + { + if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfCores, out string numberOfCoresValue) && + int.TryParse(numberOfCoresValue, out int numberOfCores) && + numberOfCores > 0) + physicalCoreCount += numberOfCores; + + if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfLogicalProcessors, out string numberOfLogicalValue) && + int.TryParse(numberOfLogicalValue, out int numberOfLogical) && + numberOfLogical > 0) + logicalCoreCount += numberOfLogical; + + if (processor.TryGetValue(WmicCpuInfoKeyNames.Name, out string name)) + { + processorModelNames.Add(name); + processorsCount++; + } + + if (processor.TryGetValue(WmicCpuInfoKeyNames.MaxClockSpeed, out string frequencyValue) + && int.TryParse(frequencyValue, out int frequency) + && frequency > 0) + { + sumMaxFrequency += frequency; + } + } + + string? processorName = processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null; + Frequency? maxFrequency = sumMaxFrequency > 0 && processorsCount > 0 + ? Frequency.FromMHz(sumMaxFrequency / processorsCount) + : null; + + return new PhdCpu + { + ProcessorName = processorName, + PhysicalProcessorCount = processorsCount > 0 ? processorsCount : null, + PhysicalCoreCount = physicalCoreCount > 0 ? physicalCoreCount : null, + LogicalCoreCount = logicalCoreCount > 0 ? logicalCoreCount : null, + NominalFrequencyHz = maxFrequency?.Hertz.RoundToLong(), + MaxFrequencyHz = maxFrequency?.Hertz.RoundToLong() + }; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/macOS/MacOsCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/macOS/MacOsCpuDetector.cs new file mode 100644 index 0000000000..54bf602020 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/macOS/MacOsCpuDetector.cs @@ -0,0 +1,22 @@ +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Portability; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Detectors.Cpu.macOS; + +/// +/// CPU information from output of the `sysctl -a` command. +/// MacOSX only. +/// +internal class MacOsCpuDetector : ICpuDetector +{ + public bool IsApplicable() => OsDetector.IsMacOS(); + + public PhdCpu? Detect() + { + if (!IsApplicable()) return null; + + string sysctlOutput = ProcessHelper.RunAndReadOutput("sysctl", "-a"); + return SysctlCpuInfoParser.Parse(sysctlOutput); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/macOS/SysctlCpuInfoParser.cs b/src/BenchmarkDotNet/Detectors/Cpu/macOS/SysctlCpuInfoParser.cs new file mode 100644 index 0000000000..9a96a5b677 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/macOS/SysctlCpuInfoParser.cs @@ -0,0 +1,60 @@ +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Detectors.Cpu.macOS; + +internal static class SysctlCpuInfoParser +{ + private static class Sysctl + { + internal const string ProcessorName = "machdep.cpu.brand_string"; + internal const string PhysicalProcessorCount = "hw.packages"; + internal const string PhysicalCoreCount = "hw.physicalcpu"; + internal const string LogicalCoreCount = "hw.logicalcpu"; + internal const string NominalFrequency = "hw.cpufrequency"; + internal const string MaxFrequency = "hw.cpufrequency_max"; + } + + /// Output of `sysctl -a` + [SuppressMessage("ReSharper", "StringLiteralTypo")] + internal static PhdCpu Parse(string? sysctlOutput) + { + var sysctl = SectionsHelper.ParseSection(sysctlOutput, ':'); + string processorName = sysctl.GetValueOrDefault(Sysctl.ProcessorName); + int? physicalProcessorCount = PositiveIntValue(sysctl, Sysctl.PhysicalProcessorCount); + int? physicalCoreCount = PositiveIntValue(sysctl, Sysctl.PhysicalCoreCount); + int? logicalCoreCount = PositiveIntValue(sysctl, Sysctl.LogicalCoreCount); + long? nominalFrequency = PositiveLongValue(sysctl, Sysctl.NominalFrequency); + long? maxFrequency = PositiveLongValue(sysctl, Sysctl.MaxFrequency); + return new PhdCpu + { + ProcessorName = processorName, + PhysicalProcessorCount = physicalProcessorCount, + PhysicalCoreCount = physicalCoreCount, + LogicalCoreCount = logicalCoreCount, + NominalFrequencyHz = nominalFrequency, + MaxFrequencyHz = maxFrequency + }; + } + + private static int? PositiveIntValue(Dictionary sysctl, string keyName) + { + if (sysctl.TryGetValue(keyName, out string value) && + int.TryParse(value, out int result) && + result > 0) + return result; + return null; + } + + private static long? PositiveLongValue(Dictionary sysctl, string keyName) + { + if (sysctl.TryGetValue(keyName, out string value) && + long.TryParse(value, out long result) && + result > 0) + return result; + return null; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/CpuDetector.cs b/src/BenchmarkDotNet/Detectors/CpuDetector.cs new file mode 100644 index 0000000000..4cf183430d --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/CpuDetector.cs @@ -0,0 +1,29 @@ +using System; +using System.Linq; +using BenchmarkDotNet.Detectors.Cpu; +using BenchmarkDotNet.Detectors.Cpu.Linux; +using BenchmarkDotNet.Detectors.Cpu.macOS; +using BenchmarkDotNet.Detectors.Cpu.Windows; +using BenchmarkDotNet.Extensions; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Detectors; + +public class CpuDetector(params ICpuDetector[] detectors) : ICpuDetector +{ + public static CpuDetector CrossPlatform => new ( + new WindowsCpuDetector(), + new LinuxCpuDetector(), + new MacOsCpuDetector()); + + private static readonly Lazy LazyCpu = new (() => CrossPlatform.Detect()); + public static PhdCpu? Cpu => LazyCpu.Value; + + public bool IsApplicable() => detectors.Any(loader => loader.IsApplicable()); + + public PhdCpu? Detect() => detectors + .Where(loader => loader.IsApplicable()) + .Select(loader => loader.Detect()) + .WhereNotNull() + .FirstOrDefault(); +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/OsDetector.cs b/src/BenchmarkDotNet/Detectors/OsDetector.cs new file mode 100644 index 0000000000..2afd862f2f --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/OsDetector.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using System.IO; +using BenchmarkDotNet.Helpers; +using Microsoft.Win32; +using Perfolizer.Phd.Dto; +using System.Runtime.InteropServices; +using BenchmarkDotNet.Extensions; +using static System.Runtime.InteropServices.RuntimeInformation; +using RuntimeEnvironment = Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment; + +namespace BenchmarkDotNet.Detectors; + +public class OsDetector +{ + public static readonly OsDetector Instance = new (); + private OsDetector() { } + + internal static string ExecutableExtension => IsWindows() ? ".exe" : string.Empty; + internal static string ScriptFileExtension => IsWindows() ? ".bat" : ".sh"; + + private readonly Lazy os = new (ResolveOs); + public static PhdOs GetOs() => Instance.os.Value; + + private static PhdOs ResolveOs() + { + if (IsMacOS()) + { + string systemVersion = ExternalToolsHelper.MacSystemProfilerData.Value.GetValueOrDefault("System Version") ?? ""; + string kernelVersion = ExternalToolsHelper.MacSystemProfilerData.Value.GetValueOrDefault("Kernel Version") ?? ""; + return new PhdOs + { + Name = "macOS", + Version = systemVersion, + KernelVersion = kernelVersion + }; + } + + if (IsLinux()) + { + try + { + string version = LinuxOsReleaseHelper.GetNameByOsRelease(File.ReadAllLines("/etc/os-release")); + bool wsl = IsUnderWsl(); + return new PhdOs + { + Name = "Linux", + Version = version, + Container = wsl ? "WSL" : null + }; + } + catch (Exception) + { + // Ignore + } + } + + string operatingSystem = RuntimeEnvironment.OperatingSystem; + string operatingSystemVersion = RuntimeEnvironment.OperatingSystemVersion; + if (IsWindows()) + { + int? ubr = GetWindowsUbr(); + if (ubr != null) + operatingSystemVersion += $".{ubr}"; + } + return new PhdOs + { + Name = operatingSystem, + Version = operatingSystemVersion + }; + } + + private static bool IsUnderWsl() + { + if (!IsLinux()) + return false; + try + { + return File.Exists("/proc/sys/fs/binfmt_misc/WSLInterop"); // https://superuser.com/a/1749811 + } + catch (Exception) + { + return false; + } + } + + // TODO: Introduce a common util API for registry calls, use it also in BenchmarkDotNet.Toolchains.CsProj.GetCurrentVersionBasedOnWindowsRegistry + /// + /// On Windows, this method returns UBR (Update Build Revision) based on Registry. + /// Returns null if the value is not available + /// + /// + private static int? GetWindowsUbr() + { + if (IsWindows()) + { + try + { + using (var baseKey = RegistryKey.OpenBaseKey(RegistryHive.LocalMachine, RegistryView.Registry32)) + using (var ndpKey = baseKey.OpenSubKey(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion")) + { + if (ndpKey == null) + return null; + + return Convert.ToInt32(ndpKey.GetValue("UBR")); + } + } + catch (Exception) + { + return null; + } + } + return null; + } + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("windows")] +#endif + internal static bool IsWindows() => +#if NET6_0_OR_GREATER + OperatingSystem.IsWindows(); // prefer linker-friendly OperatingSystem APIs +#else + IsOSPlatform(OSPlatform.Windows); +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("linux")] +#endif + internal static bool IsLinux() => +#if NET6_0_OR_GREATER + OperatingSystem.IsLinux(); +#else + IsOSPlatform(OSPlatform.Linux); +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("macos")] +#endif + // ReSharper disable once InconsistentNaming + internal static bool IsMacOS() => +#if NET6_0_OR_GREATER + OperatingSystem.IsMacOS(); +#else + IsOSPlatform(OSPlatform.OSX); +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("android")] +#endif + internal static bool IsAndroid() => +#if NET6_0_OR_GREATER + OperatingSystem.IsAndroid(); +#else + Type.GetType("Java.Lang.Object, Mono.Android") != null; +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("ios")] +#endif + // ReSharper disable once InconsistentNaming + internal static bool IsIOS() => +#if NET6_0_OR_GREATER + OperatingSystem.IsIOS(); +#else + Type.GetType("Foundation.NSObject, Xamarin.iOS") != null; +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("tvos")] +#endif + // ReSharper disable once InconsistentNaming + internal static bool IsTvOS() => +#if NET6_0_OR_GREATER + OperatingSystem.IsTvOS(); +#else + IsOSPlatform(OSPlatform.Create("TVOS")); +#endif +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/AllocatedMemoryMetricDescriptor.cs b/src/BenchmarkDotNet/Diagnosers/AllocatedMemoryMetricDescriptor.cs index dc6cac925d..634e795680 100644 --- a/src/BenchmarkDotNet/Diagnosers/AllocatedMemoryMetricDescriptor.cs +++ b/src/BenchmarkDotNet/Diagnosers/AllocatedMemoryMetricDescriptor.cs @@ -1,6 +1,7 @@ using System; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Reports; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Diagnosers { @@ -13,7 +14,7 @@ internal class AllocatedMemoryMetricDescriptor : IMetricDescriptor public string Legend => "Allocated memory per single operation (managed only, inclusive, 1KB = 1024B)"; public string NumberFormat => "0.##"; public UnitType UnitType => UnitType.Size; - public string Unit => SizeUnit.B.Name; + public string Unit => SizeUnit.B.Abbreviation; public bool TheGreaterTheBetter => false; public int PriorityInCategory => GC.MaxGeneration + 1; public bool GetIsAvailable(Metric metric) => true; diff --git a/src/BenchmarkDotNet/Diagnosers/AllocatedNativeMemoryDescriptor.cs b/src/BenchmarkDotNet/Diagnosers/AllocatedNativeMemoryDescriptor.cs index 8781333f80..c4cb964355 100644 --- a/src/BenchmarkDotNet/Diagnosers/AllocatedNativeMemoryDescriptor.cs +++ b/src/BenchmarkDotNet/Diagnosers/AllocatedNativeMemoryDescriptor.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet.Columns; using BenchmarkDotNet.Reports; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Diagnosers { @@ -12,7 +13,7 @@ internal class AllocatedNativeMemoryDescriptor : IMetricDescriptor public string Legend => $"Allocated native memory per single operation"; public string NumberFormat => "N0"; public UnitType UnitType => UnitType.Size; - public string Unit => SizeUnit.B.Name; + public string Unit => SizeUnit.B.Abbreviation; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; public bool GetIsAvailable(Metric metric) => true; @@ -27,7 +28,7 @@ internal class NativeMemoryLeakDescriptor : IMetricDescriptor public string Legend => $"Native memory leak size in byte."; public string NumberFormat => "N0"; public UnitType UnitType => UnitType.Size; - public string Unit => SizeUnit.B.Name; + public string Unit => SizeUnit.B.Abbreviation; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; public bool GetIsAvailable(Metric metric) => true; diff --git a/src/BenchmarkDotNet/Diagnosers/DiagnoserActionParameters.cs b/src/BenchmarkDotNet/Diagnosers/DiagnoserActionParameters.cs index a403e170f5..c7d957b385 100644 --- a/src/BenchmarkDotNet/Diagnosers/DiagnoserActionParameters.cs +++ b/src/BenchmarkDotNet/Diagnosers/DiagnoserActionParameters.cs @@ -6,7 +6,7 @@ namespace BenchmarkDotNet.Diagnosers { public class DiagnoserActionParameters { - public DiagnoserActionParameters(Process process, BenchmarkCase benchmarkCase, BenchmarkId benchmarkId) + public DiagnoserActionParameters(Process? process, BenchmarkCase benchmarkCase, BenchmarkId benchmarkId) { Process = process; BenchmarkCase = benchmarkCase; diff --git a/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs b/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs index 1410aad8bc..40e8f231e7 100644 --- a/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs +++ b/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs @@ -5,6 +5,7 @@ using System.Reflection; using System.Threading; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; @@ -40,11 +41,11 @@ private static IEnumerable LoadDiagnosers() { yield return EventPipeProfiler.Default; - if (RuntimeInformation.IsLinux()) + if (OsDetector.IsLinux()) yield return PerfCollectProfiler.Default; } - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) yield break; foreach (var windowsDiagnoser in LoadWindowsDiagnosers()) @@ -80,6 +81,11 @@ private static IDiagnoser[] LoadWindowsDiagnosers() CreateDiagnoser(diagnosticsAssembly, "BenchmarkDotNet.Diagnostics.Windows.NativeMemoryProfiler") }; } + catch (Exception ex) when (ex is FileNotFoundException || ex is BadImageFormatException) + { + // Return an array of UnresolvedDiagnoser objects when the assembly does not contain the requested diagnoser + return new[] { GetUnresolvedDiagnoser() }; + } catch (Exception ex) // we're loading a plug-in, better to be safe rather than sorry { ConsoleLogger.Default.WriteLineError($"Error loading {WindowsDiagnosticAssemblyFileName}: {ex.GetType().Name} - {ex.Message}"); diff --git a/src/BenchmarkDotNet/Diagnosers/EventPipeProfiler.cs b/src/BenchmarkDotNet/Diagnosers/EventPipeProfiler.cs index 211d4a06e2..c1c76e8c9e 100644 --- a/src/BenchmarkDotNet/Diagnosers/EventPipeProfiler.cs +++ b/src/BenchmarkDotNet/Diagnosers/EventPipeProfiler.cs @@ -40,7 +40,7 @@ public EventPipeProfiler() :this(profile: EventPipeProfile.CpuSampling, performE /// A named pre-defined set of provider configurations that allows common tracing scenarios to be specified succinctly. /// A list of EventPipe providers to be enabled. /// if set to true, benchmarks will be executed one more time with the profiler attached. If set to false, there will be no extra run but the results will contain overhead. True by default. - public EventPipeProfiler(EventPipeProfile profile = EventPipeProfile.CpuSampling, IReadOnlyCollection providers = null, bool performExtraBenchmarksRun = true) + public EventPipeProfiler(EventPipeProfile profile = EventPipeProfile.CpuSampling, IReadOnlyCollection? providers = null, bool performExtraBenchmarksRun = true) { this.performExtraBenchmarksRun = performExtraBenchmarksRun; eventPipeProviders = MapToProviders(profile, providers); diff --git a/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs index 782e895d3e..574d1e54f1 100644 --- a/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs @@ -1,4 +1,5 @@ using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; @@ -14,9 +15,11 @@ namespace BenchmarkDotNet.Diagnosers { public class ExceptionDiagnoser : IDiagnoser { - public static readonly ExceptionDiagnoser Default = new ExceptionDiagnoser(); + public static readonly ExceptionDiagnoser Default = new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: true)); - private ExceptionDiagnoser() { } + public ExceptionDiagnoser(ExceptionDiagnoserConfig config) => Config = config; + + public ExceptionDiagnoserConfig Config { get; } public IEnumerable Ids => new[] { nameof(ExceptionDiagnoser) }; @@ -32,14 +35,18 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } public IEnumerable ProcessResults(DiagnoserResults results) { - yield return new Metric(ExceptionsFrequencyMetricDescriptor.Instance, results.ExceptionFrequency); + yield return new Metric(new ExceptionsFrequencyMetricDescriptor(Config), results.ExceptionFrequency); } public IEnumerable Validate(ValidationParameters validationParameters) => Enumerable.Empty(); - private class ExceptionsFrequencyMetricDescriptor : IMetricDescriptor + internal class ExceptionsFrequencyMetricDescriptor : IMetricDescriptor { - internal static readonly IMetricDescriptor Instance = new ExceptionsFrequencyMetricDescriptor(); + public ExceptionDiagnoserConfig Config { get; } + public ExceptionsFrequencyMetricDescriptor(ExceptionDiagnoserConfig config = null) + { + Config = config; + } public string Id => "ExceptionFrequency"; public string DisplayName => Column.Exceptions; @@ -49,7 +56,13 @@ private class ExceptionsFrequencyMetricDescriptor : IMetricDescriptor public string Unit => "Count"; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; - public bool GetIsAvailable(Metric metric) => true; + public bool GetIsAvailable(Metric metric) + { + if (Config == null) + return metric.Value > 0; + else + return Config.DisplayExceptionsIfZeroValue || metric.Value > 0; + } } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs index 6dd5bdd2e9..f1b584ce09 100644 --- a/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs +++ b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Runtime.InteropServices; using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; @@ -53,7 +54,7 @@ public class PerfCollectProfiler : IProfiler public IEnumerable Validate(ValidationParameters validationParameters) { - if (!RuntimeInformation.IsLinux()) + if (!OsDetector.IsLinux()) { yield return new ValidationError(true, "The PerfCollectProfiler works only on Linux!"); yield break; diff --git a/src/BenchmarkDotNet/Diagnosers/SnapshotProfilerBase.cs b/src/BenchmarkDotNet/Diagnosers/SnapshotProfilerBase.cs new file mode 100644 index 0000000000..3105ceeeb9 --- /dev/null +++ b/src/BenchmarkDotNet/Diagnosers/SnapshotProfilerBase.cs @@ -0,0 +1,195 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.IO; +using System.Linq; +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Validators; + +namespace BenchmarkDotNet.Diagnosers; + +public abstract class SnapshotProfilerBase : IProfiler +{ + public abstract string ShortName { get; } + + protected abstract void InitTool(Progress progress); + protected abstract void AttachToCurrentProcess(string snapshotFile); + protected abstract void AttachToProcessByPid(int pid, string snapshotFile); + protected abstract void TakeSnapshot(); + protected abstract void Detach(); + + protected abstract string CreateSnapshotFilePath(DiagnoserActionParameters parameters); + protected abstract string GetRunnerPath(); + internal abstract bool IsSupported(RuntimeMoniker runtimeMoniker); + + private readonly List snapshotFilePaths = []; + + public IEnumerable Ids => [ShortName]; + public IEnumerable Exporters => []; + public IEnumerable Analysers => []; + + public RunMode GetRunMode(BenchmarkCase benchmarkCase) => + IsSupported(benchmarkCase.Job.Environment.GetRuntime().RuntimeMoniker) ? RunMode.ExtraRun : RunMode.None; + + public void Handle(HostSignal signal, DiagnoserActionParameters parameters) + { + var logger = parameters.Config.GetCompositeLogger(); + var job = parameters.BenchmarkCase.Job; + + var runtimeMoniker = job.Environment.GetRuntime().RuntimeMoniker; + if (!IsSupported(runtimeMoniker)) + { + logger.WriteLineError($"Runtime '{runtimeMoniker}' is not supported by dotMemory"); + return; + } + + switch (signal) + { + case HostSignal.BeforeAnythingElse: + Init(logger); + break; + case HostSignal.BeforeActualRun: + string snapshotFilePath = Start(logger, parameters); + snapshotFilePaths.Add(snapshotFilePath); + break; + case HostSignal.AfterActualRun: + Stop(logger); + break; + } + } + + public IEnumerable Validate(ValidationParameters validationParameters) + { + var runtimeMonikers = validationParameters.Benchmarks.Select(b => b.Job.Environment.GetRuntime().RuntimeMoniker).Distinct(); + foreach (var runtimeMoniker in runtimeMonikers) + if (!IsSupported(runtimeMoniker)) + yield return new ValidationError(true, $"Runtime '{runtimeMoniker}' is not supported by dotMemory"); + } + + public IEnumerable ProcessResults(DiagnoserResults results) => ImmutableArray.Empty; + + public void DisplayResults(ILogger logger) + { + if (snapshotFilePaths.Count != 0) + { + logger.WriteLineInfo($"The following {ShortName} snapshots were generated:"); + foreach (string snapshotFilePath in snapshotFilePaths) + logger.WriteLineInfo($"* {snapshotFilePath}"); + } + } + + private void Init(ILogger logger) + { + try + { + logger.WriteLineInfo($"Ensuring that {ShortName} prerequisite is installed..."); + var progress = new Progress(logger, $"Installing {ShortName}"); + InitTool(progress); + logger.WriteLineInfo($"{ShortName} prerequisite is installed"); + logger.WriteLineInfo($"{ShortName} runner path: {GetRunnerPath()}"); + } + catch (Exception e) + { + logger.WriteLineError(e.ToString()); + } + } + + private string Start(ILogger logger, DiagnoserActionParameters parameters) + { + string snapshotFilePath = CreateSnapshotFilePath(parameters); + string? snapshotDirectory = Path.GetDirectoryName(snapshotFilePath); + logger.WriteLineInfo($"Target snapshot file: {snapshotFilePath}"); + if (!Directory.Exists(snapshotDirectory) && snapshotDirectory != null) + { + try + { + Directory.CreateDirectory(snapshotDirectory); + } + catch (Exception e) + { + logger.WriteLineError($"Failed to create directory: {snapshotDirectory}"); + logger.WriteLineError(e.ToString()); + } + } + + try + { + logger.WriteLineInfo($"Attaching {ShortName} to the process..."); + Attach(parameters, snapshotFilePath); + logger.WriteLineInfo($"{ShortName} is successfully attached"); + } + catch (Exception e) + { + logger.WriteLineError(e.ToString()); + return snapshotFilePath; + } + + return snapshotFilePath; + } + + private void Stop(ILogger logger) + { + try + { + logger.WriteLineInfo($"Taking {ShortName} snapshot..."); + TakeSnapshot(); + logger.WriteLineInfo($"{ShortName} snapshot is successfully taken"); + } + catch (Exception e) + { + logger.WriteLineError(e.ToString()); + } + + try + { + logger.WriteLineInfo($"Detaching {ShortName} from the process..."); + Detach(); + logger.WriteLineInfo($"{ShortName} is successfully detached"); + } + catch (Exception e) + { + logger.WriteLineError(e.ToString()); + } + } + + + private void Attach(DiagnoserActionParameters parameters, string snapshotFile) + { + int pid = parameters.Process.Id; + int currentPid = Process.GetCurrentProcess().Id; + if (pid != currentPid) + AttachToProcessByPid(pid, snapshotFile); + else + AttachToCurrentProcess(snapshotFile); + } + + protected class Progress(ILogger logger, string title) : IProgress + { + private static readonly TimeSpan ReportInterval = TimeSpan.FromSeconds(0.1); + + private int lastProgress; + private Stopwatch? stopwatch; + + public void Report(double value) + { + int progress = (int)Math.Floor(value); + bool needToReport = stopwatch == null || + (stopwatch != null && stopwatch?.Elapsed > ReportInterval) || + progress == 100; + + if (lastProgress != progress && needToReport) + { + logger.WriteLineInfo($"{title}: {progress}%"); + lastProgress = progress; + stopwatch = Stopwatch.StartNew(); + } + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs index 057a0bb624..36b2f942c3 100644 --- a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs @@ -15,9 +15,10 @@ namespace BenchmarkDotNet.Diagnosers { public class ThreadingDiagnoser : IDiagnoser { - public static readonly ThreadingDiagnoser Default = new ThreadingDiagnoser(); + public static readonly ThreadingDiagnoser Default = new ThreadingDiagnoser(new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: true, displayLockContentionWhenZero: true)); - private ThreadingDiagnoser() { } + public ThreadingDiagnoser(ThreadingDiagnoserConfig config) => Config = config; + public ThreadingDiagnoserConfig Config { get; } public IEnumerable Ids => new[] { nameof(ThreadingDiagnoser) }; @@ -33,8 +34,9 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } public IEnumerable ProcessResults(DiagnoserResults results) { - yield return new Metric(CompletedWorkItemCountMetricDescriptor.Instance, results.ThreadingStats.CompletedWorkItemCount / (double)results.ThreadingStats.TotalOperations); - yield return new Metric(LockContentionCountMetricDescriptor.Instance, results.ThreadingStats.LockContentionCount / (double)results.ThreadingStats.TotalOperations); + + yield return new Metric(new CompletedWorkItemCountMetricDescriptor(Config), results.ThreadingStats.CompletedWorkItemCount / (double)results.ThreadingStats.TotalOperations); + yield return new Metric(new LockContentionCountMetricDescriptor(Config), results.ThreadingStats.LockContentionCount / (double)results.ThreadingStats.TotalOperations); } public IEnumerable Validate(ValidationParameters validationParameters) @@ -50,10 +52,15 @@ public IEnumerable Validate(ValidationParameters validationPara } } - private class CompletedWorkItemCountMetricDescriptor : IMetricDescriptor + internal class CompletedWorkItemCountMetricDescriptor : IMetricDescriptor { internal static readonly IMetricDescriptor Instance = new CompletedWorkItemCountMetricDescriptor(); + private ThreadingDiagnoserConfig Config { get; } + public CompletedWorkItemCountMetricDescriptor(ThreadingDiagnoserConfig config = null) + { + Config = config; + } public string Id => "CompletedWorkItemCount"; public string DisplayName => Column.CompletedWorkItems; public string Legend => "The number of work items that have been processed in ThreadPool (per single operation)"; @@ -62,13 +69,26 @@ private class CompletedWorkItemCountMetricDescriptor : IMetricDescriptor public string Unit => "Count"; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; - public bool GetIsAvailable(Metric metric) => true; + public bool GetIsAvailable(Metric metric) + { + if (Config == null) + return metric.Value > 0; + else + return Config.DisplayCompletedWorkItemCountWhenZero || metric.Value > 0; + } } - private class LockContentionCountMetricDescriptor : IMetricDescriptor + internal class LockContentionCountMetricDescriptor : IMetricDescriptor { internal static readonly IMetricDescriptor Instance = new LockContentionCountMetricDescriptor(); + private ThreadingDiagnoserConfig Config { get; } + + public LockContentionCountMetricDescriptor(ThreadingDiagnoserConfig config = null) + { + Config = config; + } + public string Id => "LockContentionCount"; public string DisplayName => Column.LockContentions; public string Legend => "The number of times there was contention upon trying to take a Monitor's lock (per single operation)"; @@ -77,7 +97,13 @@ private class LockContentionCountMetricDescriptor : IMetricDescriptor public string Unit => "Count"; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; - public bool GetIsAvailable(Metric metric) => true; + public bool GetIsAvailable(Metric metric) + { + if (Config == null) + return metric.Value > 0; + else + return Config.DisplayLockContentionWhenZero || metric.Value > 0; + } } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoserConfig.cs b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoserConfig.cs new file mode 100644 index 0000000000..6af3da25ac --- /dev/null +++ b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoserConfig.cs @@ -0,0 +1,23 @@ +using JetBrains.Annotations; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BenchmarkDotNet.Diagnosers +{ + public class ThreadingDiagnoserConfig + { + /// Display configuration for 'LockContentionCount' when it is empty. True (displayed) by default. + /// Display configuration for 'CompletedWorkItemCount' when it is empty. True (displayed) by default. + + [PublicAPI] + public ThreadingDiagnoserConfig(bool displayLockContentionWhenZero = true, bool displayCompletedWorkItemCountWhenZero = true) + { + DisplayLockContentionWhenZero = displayLockContentionWhenZero; + DisplayCompletedWorkItemCountWhenZero = displayCompletedWorkItemCountWhenZero; + } + + public bool DisplayLockContentionWhenZero { get; } + public bool DisplayCompletedWorkItemCountWhenZero { get; } + } +} diff --git a/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs index f00bb4ed67..43cc910559 100644 --- a/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Loggers; @@ -31,10 +32,10 @@ public IEnumerable Validate(ValidationParameters validationPara => new[] { new ValidationError(false, GetErrorMessage()) }; private string GetErrorMessage() => $@"Unable to resolve {unresolved.Name} diagnoser using dynamic assembly loading. - {(RuntimeInformation.IsFullFramework || RuntimeInformation.IsWindows() + {(RuntimeInformation.IsFullFramework || OsDetector.IsWindows() ? "Please make sure that you have installed the latest BenchmarkDotNet.Diagnostics.Windows package. " + Environment.NewLine + "If you are using `dotnet build` you also need to consume one of its public types to make sure that MSBuild copies it to the output directory. " + "The alternative is to use `true` in your project file." - : $"Please make sure that it's supported on {RuntimeInformation.GetOsVersion()}")}"; + : $"Please make sure that it's supported on {OsDetector.GetOs()}")}"; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs b/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs index 5b15a1e6ef..ce5176c4ca 100644 --- a/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs @@ -139,7 +139,7 @@ public void Feed(Arm64Instruction instruction) public Arm64RegisterId RegisterId { get { return _registerId; } } } - internal class Arm64Disassembler : ClrMdV2Disassembler + internal class Arm64Disassembler : ClrMdV3Disassembler { internal sealed class RuntimeSpecificData { @@ -247,7 +247,7 @@ protected override IEnumerable Decode(byte[] code, ulong startAddress, Stat } } } - TryTranslateAddressToName(address, isPrestubMD, state, isIndirect, depth, currentMethod); + TryTranslateAddressToName(address, isPrestubMD, state, depth, currentMethod); } accumulator.Feed(instruction); diff --git a/src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs b/src/BenchmarkDotNet/Disassemblers/ClrMdV3Disassembler.cs similarity index 67% rename from src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs rename to src/BenchmarkDotNet/Disassemblers/ClrMdV3Disassembler.cs index 0087b2b9a5..6bd01426d8 100644 --- a/src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/ClrMdV3Disassembler.cs @@ -7,17 +7,41 @@ using System.IO; using System.Linq; using System.Text.RegularExpressions; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Disassemblers { - // This Disassembler uses ClrMd v2x. Please keep it in sync with ClrMdV1Disassembler (if possible). - internal abstract class ClrMdV2Disassembler + // This Disassembler uses ClrMd v3x. Please keep it in sync with ClrMdV1Disassembler (if possible). + internal abstract class ClrMdV3Disassembler + { - // Translating an address to a method can cause AV and a process crash (https://github.com/dotnet/BenchmarkDotNet/issues/2070). - // It was fixed in https://github.com/dotnet/runtime/pull/79846, - // and most likely will be backported to 7.0.2 very soon (https://github.com/dotnet/runtime/pull/79862). - protected static readonly bool IsVulnerableToAvInDac = !RuntimeInformation.IsWindows() && Environment.Version < new Version(7, 0, 2); + private static readonly ulong MinValidAddress = GetMinValidAddress(); + + private static ulong GetMinValidAddress() + { + // https://github.com/dotnet/BenchmarkDotNet/pull/2413#issuecomment-1688100117 + if (OsDetector.IsWindows()) + return ushort.MaxValue + 1; + if (OsDetector.IsLinux()) + return (ulong) Environment.SystemPageSize; + if (OsDetector.IsMacOS()) + return RuntimeInformation.GetCurrentPlatform() switch + { + Environments.Platform.X86 or Environments.Platform.X64 => 4096, + Environments.Platform.Arm64 => 0x100000000, + _ => throw new NotSupportedException($"{RuntimeInformation.GetCurrentPlatform()} is not supported") + }; + throw new NotSupportedException($"{System.Runtime.InteropServices.RuntimeInformation.OSDescription} is not supported"); + } + + private static bool IsValidAddress(ulong address) + // -1 (ulong.MaxValue) address is invalid, and will crash the runtime in older runtimes. https://github.com/dotnet/runtime/pull/90794 + // 0 is NULL and therefore never valid. + // Addresses less than the minimum virtual address are also invalid. + => address != ulong.MaxValue + && address != 0 + && address >= MinValidAddress; internal DisassemblyResult AttachAndDisassemble(Settings settings) { @@ -42,7 +66,7 @@ internal DisassemblyResult AttachAndDisassemble(Settings settings) state.Todo.Enqueue( new MethodInfo( // the Disassembler Entry Method is always parameterless, so check by name is enough - typeWithBenchmark.Methods.Single(method => method.IsPublic && method.Name == settings.MethodName), + typeWithBenchmark.Methods.Single(method => method.Attributes.HasFlag(System.Reflection.MethodAttributes.Public) && method.Name == settings.MethodName), 0)); } @@ -104,6 +128,7 @@ private DisassembledMethod[] Disassemble(Settings settings, State state) var result = new List(); DisassemblySyntax syntax = (DisassemblySyntax)Enum.Parse(typeof(DisassemblySyntax), settings.Syntax); + using var sourceCodeProvider = new SourceCodeProvider(); while (state.Todo.Count != 0) { var methodInfo = state.Todo.Dequeue(); @@ -112,7 +137,7 @@ private DisassembledMethod[] Disassemble(Settings settings, State state) continue; // already handled if (settings.MaxDepth >= methodInfo.Depth) - result.Add(DisassembleMethod(methodInfo, state, settings, syntax)); + result.Add(DisassembleMethod(methodInfo, state, settings, syntax, sourceCodeProvider)); } return result.ToArray(); @@ -120,15 +145,16 @@ private DisassembledMethod[] Disassemble(Settings settings, State state) private static bool CanBeDisassembled(ClrMethod method) => method.ILOffsetMap.Length > 0 && method.NativeCode > 0; - private DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, Settings settings, DisassemblySyntax syntax) + private DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, Settings settings, DisassemblySyntax syntax, SourceCodeProvider sourceCodeProvider) { var method = methodInfo.Method; if (!CanBeDisassembled(method)) { - if (method.IsPInvoke) + if (method.Attributes.HasFlag(System.Reflection.MethodAttributes.PinvokeImpl)) return CreateEmpty(method, "PInvoke method"); - if (method.IL is null || method.IL.Length == 0) + var ilInfo = method.GetILInfo(); + if (ilInfo is null || ilInfo.Length == 0) return CreateEmpty(method, "Extern method"); if (method.CompilationType == MethodCompilationType.None) return CreateEmpty(method, "Method was not JITted yet."); @@ -143,7 +169,7 @@ private DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, var uniqueSourceCodeLines = new HashSet(new SharpComparer()); // for getting C# code we always use the original ILOffsetMap foreach (var map in method.ILOffsetMap.Where(map => map.StartAddress < map.EndAddress && map.ILOffset >= 0).OrderBy(map => map.StartAddress)) - foreach (var sharp in SourceCodeProvider.GetSource(method, map)) + foreach (var sharp in sourceCodeProvider.GetSource(method, map)) uniqueSourceCodeLines.Add(sharp); codes.AddRange(uniqueSourceCodeLines); @@ -191,67 +217,37 @@ private IEnumerable Decode(ILToNativeMap map, State state, int depth, ClrMe private static ILToNativeMap[] GetCompleteNativeMap(ClrMethod method, ClrRuntime runtime) { - if (!TryReadNativeCodeAddresses(runtime, method, out ulong startAddress, out ulong endAddress)) + // it's better to use one single map rather than few small ones + // it's simply easier to get next instruction when decoding ;) + + var hotColdInfo = method.HotColdInfo; + if (hotColdInfo.HotSize > 0 && hotColdInfo.HotStart > 0) { - startAddress = method.NativeCode; - endAddress = ulong.MaxValue; + return hotColdInfo.ColdSize <= 0 + ? new[] { new ILToNativeMap() { StartAddress = hotColdInfo.HotStart, EndAddress = hotColdInfo.HotStart + hotColdInfo.HotSize, ILOffset = -1 } } + : new[] + { + new ILToNativeMap() { StartAddress = hotColdInfo.HotStart, EndAddress = hotColdInfo.HotStart + hotColdInfo.HotSize, ILOffset = -1 }, + new ILToNativeMap() { StartAddress = hotColdInfo.ColdStart, EndAddress = hotColdInfo.ColdStart + hotColdInfo.ColdSize, ILOffset = -1 } + }; } - ILToNativeMap[] sortedMaps = method.ILOffsetMap // CanBeDisassembled ensures that there is at least one map in ILOffsetMap - .Where(map => map.StartAddress >= startAddress && map.StartAddress < endAddress) // can be false for Tier 0 maps, EndAddress is not checked on purpose here - .Where(map => map.StartAddress < map.EndAddress) // some maps have 0 length (they don't have corresponding assembly code?) + return method.ILOffsetMap + .Where(map => map.StartAddress < map.EndAddress) // some maps have 0 length? .OrderBy(map => map.StartAddress) // we need to print in the machine code order, not IL! #536 - .Select(map => new ILToNativeMap() - { - StartAddress = map.StartAddress, - // some maps have EndAddress > codeHeaderData.MethodStart + codeHeaderData.MethodSize and contain garbage (#2074). They need to be fixed! - EndAddress = Math.Min(map.EndAddress, endAddress), - ILOffset = map.ILOffset - }) .ToArray(); - - if (sortedMaps.Length == 0) - { - // In such situation ILOffsetMap most likely describes Tier 0, while CodeHeaderData Tier 1. - // Since we care about Tier 1 (if it's present), we "fake" a Tier 1 map. - return new[] { new ILToNativeMap() { StartAddress = startAddress, EndAddress = endAddress } }; - } - else if (sortedMaps[0].StartAddress != startAddress || (sortedMaps[sortedMaps.Length - 1].EndAddress != endAddress && endAddress != ulong.MaxValue)) - { - // In such situation ILOffsetMap most likely is missing few bytes. We just "extend" it to avoid producing "bad" instructions. - return new[] { new ILToNativeMap() { StartAddress = startAddress, EndAddress = endAddress } }; - } - - return sortedMaps; } private static DisassembledMethod CreateEmpty(ClrMethod method, string reason) => DisassembledMethod.Empty(method.Signature, method.NativeCode, reason); - protected static bool TryReadNativeCodeAddresses(ClrRuntime runtime, ClrMethod method, out ulong startAddress, out ulong endAddress) + protected void TryTranslateAddressToName(ulong address, bool isAddressPrecodeMD, State state, int depth, ClrMethod currentMethod) { - if (method is not null - && runtime.DacLibrary.SOSDacInterface.GetCodeHeaderData(method.NativeCode, out var codeHeaderData) == HResult.S_OK - && codeHeaderData.MethodSize > 0) // false for extern methods! - { - // HotSize can be missing or be invalid (https://github.com/microsoft/clrmd/issues/1036). - // So we fetch the method size on our own. - startAddress = codeHeaderData.MethodStart; - endAddress = codeHeaderData.MethodStart + codeHeaderData.MethodSize; - return true; - } - - startAddress = endAddress = 0; - return false; - } + if (!IsValidAddress(address) || state.AddressToNameMapping.ContainsKey(address)) + return; - protected void TryTranslateAddressToName(ulong address, bool isAddressPrecodeMD, State state, bool isIndirectCallOrJump, int depth, ClrMethod currentMethod) - { var runtime = state.Runtime; - if (state.AddressToNameMapping.ContainsKey(address)) - return; - var jitHelperFunctionName = runtime.GetJitHelperFunctionName(address); if (!string.IsNullOrEmpty(jitHelperFunctionName)) { @@ -260,47 +256,32 @@ protected void TryTranslateAddressToName(ulong address, bool isAddressPrecodeMD, } var method = runtime.GetMethodByInstructionPointer(address); - if (method is null && (address & ((uint)runtime.DataTarget.DataReader.PointerSize - 1)) == 0) - { - if (runtime.DataTarget.DataReader.ReadPointer(address, out ulong newAddress) && newAddress > ushort.MaxValue) - { - method = runtime.GetMethodByInstructionPointer(newAddress); - - method = WorkaroundGetMethodByInstructionPointerBug(runtime, method, newAddress); - } - } - else + if (method is null && (address & ((uint) runtime.DataTarget.DataReader.PointerSize - 1)) == 0 + && runtime.DataTarget.DataReader.ReadPointer(address, out ulong newAddress) && IsValidAddress(newAddress)) { - method = WorkaroundGetMethodByInstructionPointerBug(runtime, method, address); + method = runtime.GetMethodByInstructionPointer(newAddress); } if (method is null) { - if (isAddressPrecodeMD || !IsVulnerableToAvInDac) + var methodDescriptor = runtime.GetMethodByHandle(address); + if (methodDescriptor is not null) { - var methodDescriptor = runtime.GetMethodByHandle(address); - if (!(methodDescriptor is null)) + if (isAddressPrecodeMD) { - if (isAddressPrecodeMD) - { - state.AddressToNameMapping.Add(address, $"Precode of {methodDescriptor.Signature}"); - } - else - { - state.AddressToNameMapping.Add(address, $"MD_{methodDescriptor.Signature}"); - } - return; + state.AddressToNameMapping.Add(address, $"Precode of {methodDescriptor.Signature}"); } + else + { + state.AddressToNameMapping.Add(address, $"MD_{methodDescriptor.Signature}"); + } + return; } - if (!IsVulnerableToAvInDac) + var methodTableName = runtime.GetTypeByMethodTable(address)?.Name; + if (!string.IsNullOrEmpty(methodTableName)) { - var methodTableName = runtime.DacLibrary.SOSDacInterface.GetMethodTableName(address); - if (!string.IsNullOrEmpty(methodTableName)) - { - state.AddressToNameMapping.Add(address, $"MT_{methodTableName}"); - return; - } + state.AddressToNameMapping.Add(address, $"MT_{methodTableName}"); } return; } @@ -319,7 +300,7 @@ protected void TryTranslateAddressToName(ulong address, bool isAddressPrecodeMD, protected void FlushCachedDataIfNeeded(IDataReader dataTargetDataReader, ulong address, byte[] buffer) { - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { if (dataTargetDataReader.Read(address, buffer) <= 0) { @@ -333,13 +314,6 @@ protected void FlushCachedDataIfNeeded(IDataReader dataTargetDataReader, ulong a } } - // GetMethodByInstructionPointer sometimes returns wrong methods. - // In case given address does not belong to the methods range, null is returned. - private static ClrMethod WorkaroundGetMethodByInstructionPointerBug(ClrRuntime runtime, ClrMethod method, ulong newAddress) - => TryReadNativeCodeAddresses(runtime, method, out ulong startAddress, out ulong endAddress) && !(startAddress >= newAddress && newAddress <= endAddress) - ? null - : method; - private class SharpComparer : IEqualityComparer { public bool Equals(Sharp x, Sharp y) diff --git a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs index 2c6eb62f79..7d0f95326c 100644 --- a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs +++ b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs @@ -3,6 +3,7 @@ using System.Linq; using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Disassemblers; using BenchmarkDotNet.Disassemblers.Exporters; using BenchmarkDotNet.Engines; @@ -16,6 +17,7 @@ using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.InProcess.NoEmit; using BenchmarkDotNet.Validators; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Diagnosers { @@ -74,7 +76,7 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) case HostSignal.AfterAll when ShouldUseSameArchitectureDisassembler(benchmark, parameters): results.Add(benchmark, sameArchitectureDisassembler.Disassemble(parameters)); break; - case HostSignal.AfterAll when RuntimeInformation.IsWindows() && !ShouldUseMonoDisassembler(benchmark): + case HostSignal.AfterAll when OsDetector.IsWindows() && !ShouldUseMonoDisassembler(benchmark): results.Add(benchmark, windowsDifferentArchitectureDisassembler.Disassemble(parameters)); break; case HostSignal.SeparateLogic when ShouldUseMonoDisassembler(benchmark): @@ -109,24 +111,31 @@ public IEnumerable Validate(ValidationParameters validationPara yield return new ValidationError(true, "Currently NativeAOT has no DisassemblyDiagnoser support", benchmark); } - if (RuntimeInformation.IsLinux() && ShouldUseClrMdDisassembler(benchmark)) + if (ShouldUseClrMdDisassembler(benchmark)) { - var runtime = benchmark.Job.ResolveValue(EnvironmentMode.RuntimeCharacteristic, EnvironmentResolver.Instance); - - if (runtime.RuntimeMoniker < RuntimeMoniker.NetCoreApp30) - { - yield return new ValidationError(true, $"{nameof(DisassemblyDiagnoser)} supports only .NET Core 3.0+", benchmark); - } - - if (ptrace_scope.Value == "2") + if (OsDetector.IsLinux()) { - yield return new ValidationError(false, $"ptrace_scope is set to 2, {nameof(DisassemblyDiagnoser)} is going to work only if you run as sudo"); - } - else if (ptrace_scope.Value == "3") - { - yield return new ValidationError(true, $"ptrace_scope is set to 3, {nameof(DisassemblyDiagnoser)} is not going to work"); + var runtime = benchmark.Job.ResolveValue(EnvironmentMode.RuntimeCharacteristic, EnvironmentResolver.Instance); + + if (runtime.RuntimeMoniker < RuntimeMoniker.NetCoreApp30) + { + yield return new ValidationError(true, $"{nameof(DisassemblyDiagnoser)} supports only .NET Core 3.0+", benchmark); + } + + if (ptrace_scope.Value == "2") + { + yield return new ValidationError(false, $"ptrace_scope is set to 2, {nameof(DisassemblyDiagnoser)} is going to work only if you run as sudo"); + } + else if (ptrace_scope.Value == "3") + { + yield return new ValidationError(true, $"ptrace_scope is set to 3, {nameof(DisassemblyDiagnoser)} is not going to work"); + } } } + else if (!ShouldUseMonoDisassembler(benchmark)) + { + yield return new ValidationError(true, $"Only Windows and Linux are supported in DisassemblyDiagnoser without Mono. Current OS is {System.Runtime.InteropServices.RuntimeInformation.OSDescription}"); + } } } @@ -135,13 +144,13 @@ private static bool ShouldUseMonoDisassembler(BenchmarkCase benchmarkCase) // when we add macOS support, RuntimeInformation.IsMacOS() needs to be added here private static bool ShouldUseClrMdDisassembler(BenchmarkCase benchmarkCase) - => !ShouldUseMonoDisassembler(benchmarkCase) && (RuntimeInformation.IsWindows() || RuntimeInformation.IsLinux()); + => !ShouldUseMonoDisassembler(benchmarkCase) && (OsDetector.IsWindows() || OsDetector.IsLinux()); private static bool ShouldUseSameArchitectureDisassembler(BenchmarkCase benchmarkCase, DiagnoserActionParameters parameters) { if (ShouldUseClrMdDisassembler(benchmarkCase)) { - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { return WindowsDisassembler.GetDisassemblerArchitecture(parameters.Process, benchmarkCase.Job.Environment.Platform) == RuntimeInformation.GetCurrentPlatform(); @@ -187,7 +196,7 @@ private class NativeCodeSizeMetricDescriptor : IMetricDescriptor public string Legend => "Native code size of the disassembled method(s)"; public string NumberFormat => "N0"; public UnitType UnitType => UnitType.CodeSize; - public string Unit => SizeUnit.B.Name; + public string Unit => SizeUnit.B.Abbreviation; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; public bool GetIsAvailable(Metric metric) => true; diff --git a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoserConfig.cs b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoserConfig.cs index 7c917c80b0..feff76ef99 100644 --- a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoserConfig.cs +++ b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoserConfig.cs @@ -22,8 +22,8 @@ public class DisassemblyDiagnoserConfig public DisassemblyDiagnoserConfig( int maxDepth = 1, DisassemblySyntax syntax = DisassemblySyntax.Masm, - string[] filters = null, - FormatterOptions formatterOptions = null, + string[]? filters = null, + FormatterOptions? formatterOptions = null, bool printSource = false, bool printInstructionAddresses = false, bool exportGithubMarkdown = true, diff --git a/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs index 2c024f587d..3fd541528a 100644 --- a/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs @@ -7,7 +7,7 @@ namespace BenchmarkDotNet.Disassemblers { - internal class IntelDisassembler : ClrMdV2Disassembler + internal class IntelDisassembler : ClrMdV3Disassembler { internal sealed class RuntimeSpecificData { @@ -109,14 +109,7 @@ protected override IEnumerable Decode(byte[] code, ulong startAddress, Stat } } } - - if (address > ushort.MaxValue) - { - if (!IsVulnerableToAvInDac || IsCallOrJump(instruction)) - { - TryTranslateAddressToName(address, isPrestubMD, state, isIndirect, depth, currentMethod); - } - } + TryTranslateAddressToName(address, isPrestubMD, state, depth, currentMethod); } yield return new IntelAsm @@ -130,17 +123,6 @@ protected override IEnumerable Decode(byte[] code, ulong startAddress, Stat } } - private static bool IsCallOrJump(Instruction instruction) - => instruction.FlowControl switch - { - FlowControl.Call => true, - FlowControl.IndirectCall => true, - FlowControl.ConditionalBranch => true, - FlowControl.IndirectBranch => true, - FlowControl.UnconditionalBranch => true, - _ => false - }; - private static bool TryGetReferencedAddress(Instruction instruction, uint pointerSize, out ulong referencedAddress) { for (int i = 0; i < instruction.OpCount; i++) diff --git a/src/BenchmarkDotNet/Disassemblers/MonoDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/MonoDisassembler.cs index e8bae27870..de0e35b6dd 100644 --- a/src/BenchmarkDotNet/Disassemblers/MonoDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/MonoDisassembler.cs @@ -128,7 +128,7 @@ private static DisassemblyResult CreateErrorResult(IReadOnlyList input, //line example 2: 0000000000000000 subq $0x28, %rsp private static readonly Regex InstructionRegex = new Regex(@"\s*(?
[0-9a-f]+)(\:\s+([0-9a-f]{2}\s+)+)?\s+(?.*)\s*", RegexOptions.Compiled); - private static bool TryParseInstruction(string line, out MonoCode instruction) + private static bool TryParseInstruction(string line, out MonoCode? instruction) { instruction = null; var match = InstructionRegex.Match(line); diff --git a/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs index cf810cdd86..8ce026a5dd 100644 --- a/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs @@ -8,16 +8,16 @@ namespace BenchmarkDotNet.Disassemblers internal class SameArchitectureDisassembler { private readonly DisassemblyDiagnoserConfig config; - private ClrMdV2Disassembler? clrMdV2Disassembler; + private ClrMdV3Disassembler? clrMdV3Disassembler; internal SameArchitectureDisassembler(DisassemblyDiagnoserConfig config) => this.config = config; internal DisassemblyResult Disassemble(DiagnoserActionParameters parameters) // delay the creation to avoid exceptions at DisassemblyDiagnoser ctor - => (clrMdV2Disassembler ??= CreateDisassemblerForCurrentArchitecture()) + => (clrMdV3Disassembler ??= CreateDisassemblerForCurrentArchitecture()) .AttachAndDisassemble(BuildDisassemblerSettings(parameters)); - private static ClrMdV2Disassembler CreateDisassemblerForCurrentArchitecture() + private static ClrMdV3Disassembler CreateDisassemblerForCurrentArchitecture() => RuntimeInformation.GetCurrentPlatform() switch { Platform.X86 or Platform.X64 => new IntelDisassembler(), diff --git a/src/BenchmarkDotNet/Disassemblers/SourceCodeProvider.cs b/src/BenchmarkDotNet/Disassemblers/SourceCodeProvider.cs index 55e31cf8b8..beaeebfaa7 100644 --- a/src/BenchmarkDotNet/Disassemblers/SourceCodeProvider.cs +++ b/src/BenchmarkDotNet/Disassemblers/SourceCodeProvider.cs @@ -7,14 +7,21 @@ namespace BenchmarkDotNet.Disassemblers { - internal static class SourceCodeProvider + internal class SourceCodeProvider : IDisposable { - private static readonly Dictionary SourceFileCache = new Dictionary(); - private static readonly Dictionary SourceFilePathsCache = new Dictionary(); + private readonly Dictionary sourceFileCache = new Dictionary(); + private readonly Dictionary sourceFilePathsCache = new Dictionary(); + private readonly Dictionary pdbReaders = new Dictionary(); + private readonly SymbolReader symbolReader = new SymbolReader(TextWriter.Null) { SymbolPath = SymbolPath.MicrosoftSymbolServerPath }; - internal static IEnumerable GetSource(ClrMethod method, ILToNativeMap map) + public void Dispose() { - var sourceLocation = method.GetSourceLocation(map.ILOffset); + symbolReader.Dispose(); + } + + internal IEnumerable GetSource(ClrMethod method, ILToNativeMap map) + { + var sourceLocation = GetSourceLocation(method, map.ILOffset); if (sourceLocation == null) yield break; @@ -39,12 +46,12 @@ internal static IEnumerable GetSource(ClrMethod method, ILToNativeMap map } } - private static string GetFilePath(SourceFile sourceFile) - => SourceFilePathsCache.TryGetValue(sourceFile, out string filePath) ? filePath : sourceFile.Url; + private string GetFilePath(SourceFile sourceFile) + => sourceFilePathsCache.TryGetValue(sourceFile, out string filePath) ? filePath : sourceFile.Url; - private static string ReadSourceLine(SourceFile file, int line) + private string ReadSourceLine(SourceFile file, int line) { - if (!SourceFileCache.TryGetValue(file, out string[] contents)) + if (!sourceFileCache.TryGetValue(file, out string[] contents)) { // GetSourceFile method returns path when file is stored on the same machine // otherwise it downloads it from the Symbol Server and returns the source code ;) @@ -56,14 +63,14 @@ private static string ReadSourceLine(SourceFile file, int line) if (File.Exists(wholeFileOrJustPath)) { contents = File.ReadAllLines(wholeFileOrJustPath); - SourceFilePathsCache.Add(file, wholeFileOrJustPath); + sourceFilePathsCache.Add(file, wholeFileOrJustPath); } else { contents = wholeFileOrJustPath.Split(new string[] { Environment.NewLine }, StringSplitOptions.None); } - SourceFileCache.Add(file, contents); + sourceFileCache.Add(file, contents); } return line - 1 < contents.Length @@ -99,17 +106,8 @@ private static string GetSmartPointer(string sourceLine, int? start, int? end) return new string(prefix); } - } - - internal static class ClrSourceExtensions - { - // TODO Not sure we want this to be a shared dictionary, especially without - // any synchronization. Probably want to put this hanging off the Context - // somewhere, or inside SymbolCache. - private static readonly Dictionary s_pdbReaders = new Dictionary(); - private static readonly SymbolReader symbolReader = new SymbolReader(TextWriter.Null) { SymbolPath = SymbolPath.MicrosoftSymbolServerPath }; - internal static SourceLocation GetSourceLocation(this ClrMethod method, int ilOffset) + internal SourceLocation GetSourceLocation(ClrMethod method, int ilOffset) { var reader = GetReaderForMethod(method); if (reader == null) @@ -118,7 +116,7 @@ internal static SourceLocation GetSourceLocation(this ClrMethod method, int ilOf return reader.SourceLocationForManagedCode((uint)method.MetadataToken, ilOffset); } - internal static SourceLocation GetSourceLocation(this ClrStackFrame frame) + internal SourceLocation GetSourceLocation(ClrStackFrame frame) { var reader = GetReaderForMethod(frame.Method); if (reader == null) @@ -145,15 +143,15 @@ private static int FindIlOffset(ClrStackFrame frame) return last; } - private static ManagedSymbolModule GetReaderForMethod(ClrMethod method) + private ManagedSymbolModule GetReaderForMethod(ClrMethod method) { ClrModule module = method?.Type?.Module; PdbInfo info = module?.Pdb; - ManagedSymbolModule reader = null; + ManagedSymbolModule? reader = null; if (info != null) { - if (!s_pdbReaders.TryGetValue(info, out reader)) + if (!pdbReaders.TryGetValue(info, out reader)) { string pdbPath = info.Path; if (pdbPath != null) @@ -173,7 +171,7 @@ private static ManagedSymbolModule GetReaderForMethod(ClrMethod method) } } - s_pdbReaders[info] = reader; + pdbReaders[info] = reader; } } diff --git a/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs index 6bec70162a..8051ba3463 100644 --- a/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs @@ -7,6 +7,7 @@ using System.Text; using System.Xml; using System.Xml.Serialization; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; @@ -165,7 +166,7 @@ public static bool Is64Bit(Process process) if (Environment.GetEnvironmentVariable("PROCESSOR_ARCHITECTURE") == "x86") return false; - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { IsWow64Process(process.Handle, out bool isWow64); diff --git a/src/BenchmarkDotNet/Engines/AnonymousPipesHost.cs b/src/BenchmarkDotNet/Engines/AnonymousPipesHost.cs index 6add899105..760ebb4057 100644 --- a/src/BenchmarkDotNet/Engines/AnonymousPipesHost.cs +++ b/src/BenchmarkDotNet/Engines/AnonymousPipesHost.cs @@ -63,7 +63,7 @@ public void SendSignal(HostSignal hostSignal) public void ReportResults(RunResults runResults) => runResults.Print(outWriter); [PublicAPI] // called from generated code - public static bool TryGetFileHandles(string[] args, out string writeHandle, out string readHandle) + public static bool TryGetFileHandles(string[] args, out string? writeHandle, out string? readHandle) { for (int i = 0; i < args.Length; i++) { diff --git a/src/BenchmarkDotNet/Engines/Consumer.cs b/src/BenchmarkDotNet/Engines/Consumer.cs index 1abedec0d6..55f8e3b040 100644 --- a/src/BenchmarkDotNet/Engines/Consumer.cs +++ b/src/BenchmarkDotNet/Engines/Consumer.cs @@ -31,7 +31,7 @@ private static readonly HashSet SupportedTypes private double doubleHolder; private long longHolder; private ulong ulongHolder; - private volatile object objectHolder; + private volatile object? objectHolder; private volatile IntPtr ptrHolder; private volatile UIntPtr uptrHolder; #pragma warning restore IDE0052 // Remove unread private members @@ -157,7 +157,7 @@ public void Consume(in T value) internal static bool IsConsumable(Type type) => SupportedTypes.Contains(type) || type.GetTypeInfo().IsClass || type.GetTypeInfo().IsInterface; - internal static bool HasConsumableField(Type type, out FieldInfo consumableField) + internal static bool HasConsumableField(Type type, out FieldInfo? consumableField) { var typeInfo = type.GetTypeInfo(); diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index 29d2e0ddd7..9dc8cc616e 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -166,12 +166,9 @@ public Measurement RunIteration(IterationData data) if (EngineEventSource.Log.IsEnabled()) EngineEventSource.Log.IterationStart(data.IterationMode, data.IterationStage, totalOperations); - Span stackMemory = randomizeMemory ? stackalloc byte[random.Next(32)] : Span.Empty; - - // Measure - var clock = Clock.Start(); - action(invokeCount / unrollFactor); - var clockSpan = clock.GetElapsed(); + var clockSpan = randomizeMemory + ? MeasureWithRandomMemory(action, invokeCount / unrollFactor) + : Measure(action, invokeCount / unrollFactor); if (EngineEventSource.Log.IsEnabled()) EngineEventSource.Log.IterationStop(data.IterationMode, data.IterationStage, totalOperations); @@ -190,9 +187,29 @@ public Measurement RunIteration(IterationData data) if (measurement.IterationStage == IterationStage.Jitting) jittingMeasurements.Add(measurement); + return measurement; + } + + // This is in a separate method, because stackalloc can affect code alignment, + // resulting in unexpected measurements on some AMD cpus, + // even if the stackalloc branch isn't executed. (#2366) + [MethodImpl(MethodImplOptions.NoInlining)] + private unsafe ClockSpan MeasureWithRandomMemory(Action action, long invokeCount) + { + byte* stackMemory = stackalloc byte[random.Next(32)]; + var clockSpan = Measure(action, invokeCount); Consume(stackMemory); + return clockSpan; + } - return measurement; + [MethodImpl(MethodImplOptions.NoInlining)] + private unsafe void Consume(byte* _) { } + + private ClockSpan Measure(Action action, long invokeCount) + { + var clock = Clock.Start(); + action(invokeCount); + return clock.GetElapsed(); } private (GcStats, ThreadingStats, double) GetExtraStats(IterationData data) @@ -224,9 +241,6 @@ public Measurement RunIteration(IterationData data) return (gcStats, threadingStats, exceptionsStats.ExceptionsCount / (double)totalOperationsCount); } - [MethodImpl(MethodImplOptions.NoInlining)] - private void Consume(in Span _) { } - private void RandomizeManagedHeapMemory() { // invoke global cleanup before global setup diff --git a/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs b/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs index cecd779899..6394db3d1e 100644 --- a/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs @@ -58,7 +58,7 @@ private List RunAuto(long invokeCount, IterationMode iterationMode, measurementsForStatistics.Add(measurement); var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, outlierMode); - double actualError = statistics.ConfidenceInterval.Margin; + double actualError = statistics.LegacyConfidenceInterval.Margin; double maxError1 = effectiveMaxRelativeError * statistics.Mean; double maxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue; diff --git a/src/BenchmarkDotNet/Engines/GcStats.cs b/src/BenchmarkDotNet/Engines/GcStats.cs index a156c88161..ca60b0deea 100644 --- a/src/BenchmarkDotNet/Engines/GcStats.cs +++ b/src/BenchmarkDotNet/Engines/GcStats.cs @@ -14,11 +14,6 @@ public struct GcStats : IEquatable public static readonly long AllocationQuantum = CalculateAllocationQuantumSize(); -#if !NET6_0_OR_GREATER - private static readonly Func GetAllocatedBytesForCurrentThreadDelegate = CreateGetAllocatedBytesForCurrentThreadDelegate(); - private static readonly Func GetTotalAllocatedBytesDelegate = CreateGetTotalAllocatedBytesDelegate(); -#endif - public static readonly GcStats Empty = default; private GcStats(int gen0Collections, int gen1Collections, int gen2Collections, long? allocatedBytes, long totalOperations) @@ -143,9 +138,6 @@ public static GcStats FromForced(int forcedFullGarbageCollections) private static long? GetAllocatedBytes() { - if (RuntimeInformation.IsOldMono) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes- - return null; - // we have no tests for WASM and don't want to risk introducing a new bug (https://github.com/dotnet/BenchmarkDotNet/issues/2226) if (RuntimeInformation.IsWasm) return null; @@ -155,36 +147,20 @@ public static GcStats FromForced(int forcedFullGarbageCollections) // so we enforce GC.Collect here just to make sure we get accurate results GC.Collect(); - if (RuntimeInformation.IsFullFramework) // it can be a .NET app consuming our .NET Standard package - return AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize; - #if NET6_0_OR_GREATER return GC.GetTotalAllocatedBytes(precise: true); #else - if (GetTotalAllocatedBytesDelegate != null) // it's .NET Core 3.0 with the new API available - return GetTotalAllocatedBytesDelegate.Invoke(true); // true for the "precise" argument + if (GcHelpers.GetTotalAllocatedBytesDelegate != null) // it's .NET Core 3.0 with the new API available + return GcHelpers.GetTotalAllocatedBytesDelegate.Invoke(true); // true for the "precise" argument - // https://apisof.net/catalog/System.GC.GetAllocatedBytesForCurrentThread() is not part of the .NET Standard, so we use reflection to call it.. - return GetAllocatedBytesForCurrentThreadDelegate.Invoke(); -#endif - } - - private static Func CreateGetAllocatedBytesForCurrentThreadDelegate() - { - // this method is not a part of .NET Standard so we need to use reflection - var method = typeof(GC).GetTypeInfo().GetMethod("GetAllocatedBytesForCurrentThread", BindingFlags.Public | BindingFlags.Static); - - // we create delegate to avoid boxing, IMPORTANT! - return method != null ? (Func)method.CreateDelegate(typeof(Func)) : null; - } + if (GcHelpers.CanUseMonitoringTotalAllocatedMemorySize) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes- + return AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize; - private static Func CreateGetTotalAllocatedBytesDelegate() - { - // this method is not a part of .NET Standard so we need to use reflection - var method = typeof(GC).GetTypeInfo().GetMethod("GetTotalAllocatedBytes", BindingFlags.Public | BindingFlags.Static); + if (GcHelpers.GetAllocatedBytesForCurrentThreadDelegate != null) + return GcHelpers.GetAllocatedBytesForCurrentThreadDelegate.Invoke(); - // we create delegate to avoid boxing, IMPORTANT! - return method != null ? (Func)method.CreateDelegate(typeof(Func)) : null; + return null; +#endif } public string ToOutputLine() @@ -260,5 +236,81 @@ private static long CalculateAllocationQuantumSize() public override bool Equals(object obj) => obj is GcStats other && Equals(other); public override int GetHashCode() => HashCode.Combine(Gen0Collections, Gen1Collections, Gen2Collections, AllocatedBytes, TotalOperations); + +#if !NET6_0_OR_GREATER + // Separate class to have the cctor run lazily, to avoid enabling monitoring before the benchmarks are ran. + private static class GcHelpers + { + // do not reorder these, CheckMonitoringTotalAllocatedMemorySize relies on GetTotalAllocatedBytesDelegate being initialized first + public static readonly Func GetTotalAllocatedBytesDelegate = CreateGetTotalAllocatedBytesDelegate(); + public static readonly Func GetAllocatedBytesForCurrentThreadDelegate = CreateGetAllocatedBytesForCurrentThreadDelegate(); + public static readonly bool CanUseMonitoringTotalAllocatedMemorySize = CheckMonitoringTotalAllocatedMemorySize(); + + private static Func CreateGetTotalAllocatedBytesDelegate() + { + try + { + // this method is not a part of .NET Standard so we need to use reflection + var method = typeof(GC).GetTypeInfo().GetMethod("GetTotalAllocatedBytes", BindingFlags.Public | BindingFlags.Static); + + if (method == null) + return null; + + // we create delegate to avoid boxing, IMPORTANT! + var del = (Func)method.CreateDelegate(typeof(Func)); + + // verify the api works + return del.Invoke(true) >= 0 ? del : null; + } + catch + { + return null; + } + } + + private static Func CreateGetAllocatedBytesForCurrentThreadDelegate() + { + try + { + // this method is not a part of .NET Standard so we need to use reflection + var method = typeof(GC).GetTypeInfo().GetMethod("GetAllocatedBytesForCurrentThread", BindingFlags.Public | BindingFlags.Static); + + if (method == null) + return null; + + // we create delegate to avoid boxing, IMPORTANT! + var del = (Func)method.CreateDelegate(typeof(Func)); + + // verify the api works + return del.Invoke() >= 0 ? del : null; + } + catch + { + return null; + } + } + + private static bool CheckMonitoringTotalAllocatedMemorySize() + { + try + { + // we potentially don't want to enable monitoring if we don't need it + if (GetTotalAllocatedBytesDelegate != null) + return false; + + // check if monitoring is enabled + if (!AppDomain.MonitoringIsEnabled) + AppDomain.MonitoringIsEnabled = true; + + // verify the api works + return AppDomain.MonitoringIsEnabled && AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize >= 0; + } + catch + { + return false; + } + } + } +#endif } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Engines/HostSignal.cs b/src/BenchmarkDotNet/Engines/HostSignal.cs index f4a2f75a22..cf423d2268 100644 --- a/src/BenchmarkDotNet/Engines/HostSignal.cs +++ b/src/BenchmarkDotNet/Engines/HostSignal.cs @@ -7,6 +7,11 @@ public enum HostSignal ///
BeforeProcessStart, + /// + /// right after we start the benchmarking process + /// + AfterProcessStart, + /// /// before jitting, warmup /// diff --git a/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs b/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs index e13d50b650..f2e2f34e26 100644 --- a/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs +++ b/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs @@ -2,10 +2,10 @@ using System.Diagnostics; using System.Linq; using System.Runtime; +using BenchmarkDotNet.Detectors.Cpu; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Portability.Cpu; using BenchmarkDotNet.Properties; using BenchmarkDotNet.Validators; using JetBrains.Annotations; diff --git a/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs b/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs index 3b0da64fa3..78e1624f4a 100644 --- a/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs +++ b/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs @@ -2,15 +2,20 @@ using System.Collections.Generic; using System.Linq; using System.Text; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Phd; using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Portability.Cpu; using BenchmarkDotNet.Properties; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Toolchains.DotNetCli; using JetBrains.Annotations; +using Perfolizer.Helpers; using Perfolizer.Horology; +using Perfolizer.Metrology; +using Perfolizer.Phd; +using Perfolizer.Phd.Dto; namespace BenchmarkDotNet.Environments { @@ -25,21 +30,18 @@ public class HostEnvironmentInfo : BenchmarkEnvironmentInfo /// /// Logger to use when there's no config available. /// - public static ILogger FallbackLogger { get; } = ConsoleLogger.Default; + public static ILogger FallbackLogger => ConsoleLogger.Default; - private static HostEnvironmentInfo current; + private static HostEnvironmentInfo? current; public string BenchmarkDotNetVersion { get; protected set; } - /// - /// Could be expensive - /// - public Lazy OsVersion { get; protected set; } - /// /// is expensive to call (1s) /// - public Lazy CpuInfo { get; protected set; } + public Lazy Cpu { get; protected set; } + + public Lazy Os { get; protected set; } /// /// .NET Core SDK version @@ -57,48 +59,54 @@ public class HostEnvironmentInfo : BenchmarkEnvironmentInfo /// The frequency of the timer as the number of ticks per second. /// [PublicAPI] public Frequency ChronometerFrequency { get; protected set; } + [PublicAPI] public TimeInterval ChronometerResolution => ChronometerFrequency.ToResolution(); public HardwareTimerKind HardwareTimerKind { get; protected set; } public Lazy> AntivirusProducts { get; } - public Lazy VirtualMachineHypervisor { get; protected set; } + // TODO: Join with PhdOs + public Lazy VirtualMachineHypervisor { get; protected set; } protected HostEnvironmentInfo() { BenchmarkDotNetVersion = BenchmarkDotNetInfo.Instance.BrandVersion; - OsVersion = new Lazy(RuntimeInformation.GetOsVersion); - CpuInfo = new Lazy(RuntimeInformation.GetCpuInfo); ChronometerFrequency = Chronometer.Frequency; HardwareTimerKind = Chronometer.HardwareTimerKind; DotNetSdkVersion = new Lazy(DotNetCliCommandExecutor.GetDotNetSdkVersion); IsMonoInstalled = new Lazy(() => !string.IsNullOrEmpty(ProcessHelper.RunAndReadOutput("mono", "--version"))); AntivirusProducts = new Lazy>(RuntimeInformation.GetAntivirusProducts); VirtualMachineHypervisor = new Lazy(RuntimeInformation.GetVirtualMachineHypervisor); + Os = new Lazy(OsDetector.GetOs); + Cpu = new Lazy(CpuDetector.CrossPlatform.Detect); } - public new static HostEnvironmentInfo GetCurrent() => current ?? (current = new HostEnvironmentInfo()); + public new static HostEnvironmentInfo GetCurrent() => current ??= new HostEnvironmentInfo(); public override IEnumerable ToFormattedString() { - string vmName = VirtualMachineHypervisor.Value?.Name; + string? vmName = VirtualMachineHypervisor.Value?.Name; if (!string.IsNullOrEmpty(vmName)) - yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {OsVersion.Value} ({vmName})"; + yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {Os.Value.ToBrandString()} ({vmName})"; else if (RuntimeInformation.IsRunningInContainer) - yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {OsVersion.Value} (container)"; + yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {Os.Value.ToBrandString()} (container)"; else - yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {OsVersion.Value}"; + yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {Os.Value.ToBrandString()}"; - yield return CpuInfoFormatter.Format(CpuInfo.Value); - var cultureInfo = DefaultCultureInfo.Instance; + yield return Cpu.Value.ToFullBrandName(); if (HardwareTimerKind != HardwareTimerKind.Unknown) - yield return $"Frequency: {ChronometerFrequency}, Resolution: {ChronometerResolution.ToString(cultureInfo)}, Timer: {HardwareTimerKind.ToString().ToUpper()}"; + { + string frequency = ChronometerFrequency.ToString(FrequencyUnit.Hz, unitPresentation: UnitHelper.DefaultPresentation); + string resolution = ChronometerResolution.ToString(UnitHelper.DefaultPresentation); + string timer = HardwareTimerKind.ToString().ToUpper(); + yield return $"Frequency: {frequency}, Resolution: {resolution}, Timer: {timer}"; + } if (RuntimeInformation.IsNetCore && IsDotNetCliInstalled()) { - // this wonderful version number contains words like "preview" and ... 5 segments so it can not be parsed by Version.Parse. Example: "5.0.100-preview.8.20362.3" + // this wonderful version number contains words like "preview" and ... 5 segments, so it can not be parsed by Version.Parse. Example: "5.0.100-preview.8.20362.3" if (int.TryParse(new string(DotNetSdkVersion.Value.TrimStart().TakeWhile(char.IsDigit).ToArray()), out int major) && major >= 5) yield return $".NET SDK {DotNetSdkVersion.Value}"; else @@ -125,5 +133,18 @@ public static string GetInformation() sb.AppendLine(Summary.BuildAllRuntimes(hostEnvironmentInfo, Array.Empty())); return sb.ToString(); } + + public BdnHost ToPhd() => new () + { + Cpu = Cpu.Value, + Os = Os.Value, + RuntimeVersion = RuntimeVersion, + HasAttachedDebugger = HasAttachedDebugger, + HasRyuJit = HasRyuJit, + Configuration = Configuration, + DotNetSdkVersion = DotNetSdkVersion.Value, + ChronometerFrequency = ChronometerFrequency.Hertz, + HardwareTimerKind = HardwareTimerKind.ToString() + }; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/OsBrandStringHelper.cs b/src/BenchmarkDotNet/Environments/OsBrandStringHelper.cs deleted file mode 100644 index b82575f35c..0000000000 --- a/src/BenchmarkDotNet/Environments/OsBrandStringHelper.cs +++ /dev/null @@ -1,276 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using BenchmarkDotNet.Extensions; - -namespace BenchmarkDotNet.Environments -{ - [SuppressMessage("ReSharper", "ClassNeverInstantiated.Global")] - public class OsBrandStringHelper - { - // See https://en.wikipedia.org/wiki/Ver_(command) - // See https://docs.microsoft.com/en-us/windows/release-health/release-information - // See https://docs.microsoft.com/en-us/windows/release-health/windows11-release-information - private static readonly Dictionary WindowsBrandVersions = new Dictionary - { - { "1.04", "1.0" }, - { "2.11", "2.0" }, - { "3", "3.0" }, - { "3.10.528", "NT 3.1" }, - { "3.11", "for Workgroups 3.11" }, - { "3.50.807", "NT 3.5" }, - { "3.51.1057", "NT 3.51" }, - { "4.00.950", "95" }, - { "4.00.1111", "95 OSR2" }, - { "4.03.1212-1214", "95 OSR2.1" }, - { "4.03.1214", "95 OSR2.5" }, - { "4.00.1381", "NT 4.0" }, - { "4.10.1998", "98" }, - { "4.10.2222", "98 SE" }, - { "4.90.2380.2", "ME Beta" }, - { "4.90.2419", "ME Beta 2" }, - { "4.90.3000", "ME" }, - { "5.00.1515", "NT 5.0 Beta" }, - { "5.00.2031", "2000 Beta 3" }, - { "5.00.2128", "2000 RC2" }, - { "5.00.2183", "2000 RC3" }, - { "5.00.2195", "2000" }, - { "5.0.2195", "2000 Professional" }, - { "5.1.2505", "XP RC1" }, - { "5.1.2600", "XP" }, - { "5.1.2600.1105-1106", "XP SP1" }, - { "5.1.2600.2180", "XP SP2" }, - { "5.2.3541", ".NET Server interim" }, - { "5.2.3590", ".NET Server Beta 3" }, - { "5.2.3660", ".NET Server RC1" }, - { "5.2.3718", ".NET Server 2003 RC2" }, - { "5.2.3763", "Server 2003 Beta" }, - { "5.2.3790", "XP Professional x64 Edition" }, - { "5.2.3790.1180", "Server 2003 SP1" }, - { "5.2.3790.1218", "Server 2003" }, - { "6.0.5048", "Longhorn" }, - { "6.0.5112", "Vista Beta 1" }, - { "6.0.5219", "Vista CTP" }, - { "6.0.5259", "Vista TAP Preview" }, - { "6.0.5270", "Vista CTP December" }, - { "6.0.5308", "Vista CTP February" }, - { "6.0.5342", "Vista CTP Refresh" }, - { "6.0.5365", "Vista April EWD" }, - { "6.0.5381", "Vista Beta 2 Preview" }, - { "6.0.5384", "Vista Beta 2" }, - { "6.0.5456", "Vista Pre-RC1 Build 5456" }, - { "6.0.5472", "Vista Pre-RC1 Build 5472" }, - { "6.0.5536", "Vista Pre-RC1 Build 5536" }, - { "6.0.5600.16384", "Vista RC1" }, - { "6.0.5700", "Vista Pre-RC2" }, - { "6.0.5728", "Vista Pre-RC2 Build 5728" }, - { "6.0.5744.16384", "Vista RC2" }, - { "6.0.5808", "Vista Pre-RTM Build 5808" }, - { "6.0.5824", "Vista Pre-RTM Build 5824" }, - { "6.0.5840", "Vista Pre-RTM Build 5840" }, - { "6.0.6000", "Vista" }, - { "6.0.6000.16386", "Vista RTM" }, - { "6.0.6001", "Vista SP1" }, - { "6.0.6002", "Vista SP2" }, - { "6.1.7600", "7" }, - { "6.1.7600.16385", "7" }, - { "6.1.7601", "7 SP1" }, - { "6.1.8400", "Home Server 2011" }, - { "6.2.8102", "8 Developer Preview" }, - { "6.2.9200", "8" }, - { "6.2.9200.16384", "8 RTM" }, - { "6.2.10211", "Phone 8" }, - { "6.3.9600", "8.1" }, - { "6.4.9841", "10 Technical Preview 1" }, - { "6.4.9860", "10 Technical Preview 2" }, - { "6.4.9879", "10 Technical Preview 3" }, - { "10.0.9926", "10 Technical Preview 4" }, - { "10.0.10041", "10 Technical Preview 5" }, - { "10.0.10049", "10 Technical Preview 6" }, - { "10.0.10240", "10 Threshold 1 [1507, RTM]" }, - { "10.0.10586", "10 Threshold 2 [1511, November Update]" }, - { "10.0.14393", "10 Redstone 1 [1607, Anniversary Update]" }, - { "10.0.15063", "10 Redstone 2 [1703, Creators Update]" }, - { "10.0.16299", "10 Redstone 3 [1709, Fall Creators Update]" }, - { "10.0.17134", "10 Redstone 4 [1803, April 2018 Update]" }, - { "10.0.17763", "10 Redstone 5 [1809, October 2018 Update]" }, - { "10.0.18362", "10 19H1 [1903, May 2019 Update]" }, - { "10.0.18363", "10 19H2 [1909, November 2019 Update]" }, - { "10.0.19041", "10 20H1 [2004, May 2020 Update]" }, - { "10.0.19042", "10 20H2 [20H2, October 2020 Update]" }, - { "10.0.19043", "10 21H1 [21H1, May 2021 Update]" }, - { "10.0.19044", "10 21H2 [21H2, November 2021 Update]" }, - { "10.0.19045", "10 22H2 [22H2, 2022 Update]" }, - { "10.0.22000", "11 21H2 [21H2, Sun Valley]" }, - { "10.0.22621", "11 22H2 [22H2, Sun Valley 2]" }, - }; - - private class Windows1XVersion - { - private string? CodeVersion { get; } - private string? CodeName { get; } - private string? MarketingName { get; } - private int BuildNumber { get; } - - private string MarketingNumber => BuildNumber >= 22000 ? "11" : "10"; - private string? ShortifiedCodeName => CodeName?.Replace(" ", ""); - private string? ShortifiedMarketingName => MarketingName?.Replace(" ", ""); - - private Windows1XVersion(string? codeVersion, string? codeName, string? marketingName, int buildNumber) - { - CodeVersion = codeVersion; - CodeName = codeName; - MarketingName = marketingName; - BuildNumber = buildNumber; - } - - private string ToFullVersion(int? ubr = null) - => ubr == null ? $"10.0.{BuildNumber}" : $"10.0.{BuildNumber}.{ubr}"; - - private static string Collapse(params string[] values) => string.Join("/", values.Where(v => !string.IsNullOrEmpty(v))); - - // The line with OsBrandString is one of the longest lines in the summary. - // When people past in on GitHub, it can be a reason of an ugly horizontal scrollbar. - // To avoid this, we are trying to minimize this line and use the minimum possible number of characters. - public string ToPrettifiedString(int? ubr) - => CodeVersion == ShortifiedCodeName - ? $"{MarketingNumber} ({Collapse(ToFullVersion(ubr), CodeVersion, ShortifiedMarketingName)})" - : $"{MarketingNumber} ({Collapse(ToFullVersion(ubr), CodeVersion, ShortifiedMarketingName, ShortifiedCodeName)})"; - - // See https://en.wikipedia.org/wiki/Windows_10_version_history - // See https://en.wikipedia.org/wiki/Windows_11_version_history - private static readonly List WellKnownVersions = new () - { - // Windows 10 - new Windows1XVersion("1507", "Threshold 1", "RTM", 10240), - new Windows1XVersion("1511", "Threshold 2", "November Update", 10586), - new Windows1XVersion("1607", "Redstone 1", "Anniversary Update", 14393), - new Windows1XVersion("1703", "Redstone 2", "Creators Update", 15063), - new Windows1XVersion("1709", "Redstone 3", "Fall Creators Update", 16299), - new Windows1XVersion("1803", "Redstone 4", "April 2018 Update", 17134), - new Windows1XVersion("1809", "Redstone 5", "October 2018 Update", 17763), - new Windows1XVersion("1903", "19H1", "May 2019 Update", 18362), - new Windows1XVersion("1909", "19H2", "November 2019 Update", 18363), - new Windows1XVersion("2004", "20H1", "May 2020 Update", 19041), - new Windows1XVersion("20H2", "20H2", "October 2020 Update", 19042), - new Windows1XVersion("21H1", "21H1", "May 2021 Update", 19043), - new Windows1XVersion("21H2", "21H2", "November 2021 Update", 19044), - new Windows1XVersion("22H2", "22H2", "2022 Update", 19045), - // Windows 11 - new Windows1XVersion("21H2", "Sun Valley", null, 22000), - new Windows1XVersion("22H2", "Sun Valley 2", "2022 Update", 22621), - }; - - public static Windows1XVersion? Resolve(string osVersionString) - { - var windows1XVersion = WellKnownVersions.FirstOrDefault(v => osVersionString == $"10.0.{v.BuildNumber}"); - if (windows1XVersion != null) - return windows1XVersion; - if (Version.TryParse(osVersionString, out var osVersion)) - { - if (osVersion.Major == 10 && osVersion.Minor == 0) - return new Windows1XVersion(null, null, null, osVersion.Build); - } - return null; - } - } - - /// - /// Transform an operation system name and version to a nice form for summary. - /// - /// Original operation system name - /// Original operation system version - /// UBR (Update Build Revision), the revision number of Windows version (if available) - /// Prettified operation system title - public static string Prettify(string osName, string osVersion, int? windowsUbr = null) - { - if (osName == "Windows") - return PrettifyWindows(osVersion, windowsUbr); - return $"{osName} {osVersion}"; - } - - private static string PrettifyWindows(string osVersion, int? windowsUbr) - { - var windows1XVersion = Windows1XVersion.Resolve(osVersion); - if (windows1XVersion != null) - return "Windows " + windows1XVersion.ToPrettifiedString(windowsUbr); - - string brandVersion = WindowsBrandVersions.GetValueOrDefault(osVersion); - string completeOsVersion = windowsUbr != null && osVersion.Count(c => c == '.') == 2 - ? osVersion + "." + windowsUbr - : osVersion; - string fullVersion = brandVersion == null ? osVersion : brandVersion + " (" + completeOsVersion + ")"; - return "Windows " + fullVersion; - } - - private class MacOSXVersion - { - private int DarwinVersion { get; } - private string CodeName { get; } - - private MacOSXVersion(int darwinVersion, string codeName) - { - DarwinVersion = darwinVersion; - CodeName = codeName; - } - - private static readonly List WellKnownVersions = new List - { - new MacOSXVersion(6, "Jaguar"), - new MacOSXVersion(7, "Panther"), - new MacOSXVersion(8, "Tiger"), - new MacOSXVersion(9, "Leopard"), - new MacOSXVersion(10, "Snow Leopard"), - new MacOSXVersion(11, "Lion"), - new MacOSXVersion(12, "Mountain Lion"), - new MacOSXVersion(13, "Mavericks"), - new MacOSXVersion(14, "Yosemite"), - new MacOSXVersion(15, "El Capitan"), - new MacOSXVersion(16, "Sierra"), - new MacOSXVersion(17, "High Sierra"), - new MacOSXVersion(18, "Mojave"), - new MacOSXVersion(19, "Catalina"), - new MacOSXVersion(20, "Big Sur"), - new MacOSXVersion(21, "Monterey"), - new MacOSXVersion(22, "Ventura"), - new MacOSXVersion(23, "Sonoma"), - }; - - public static string? ResolveCodeName(string kernelVersion) - { - if (string.IsNullOrWhiteSpace(kernelVersion)) - return null; - - kernelVersion = kernelVersion.ToLowerInvariant().Trim(); - if (kernelVersion.StartsWith("darwin")) - kernelVersion = kernelVersion.Substring(6).Trim(); - var numbers = kernelVersion.Split('.'); - if (numbers.Length == 0) - return null; - - string majorVersionStr = numbers[0]; - if (int.TryParse(majorVersionStr, out int majorVersion)) - return WellKnownVersions.FirstOrDefault(v => v.DarwinVersion == majorVersion)?.CodeName; - return null; - } - } - - public static string PrettifyMacOSX(string systemVersion, string kernelVersion) - { - string codeName = MacOSXVersion.ResolveCodeName(kernelVersion); - if (codeName != null) - { - int firstDigitIndex = systemVersion.IndexOfAny("0123456789".ToCharArray()); - if (firstDigitIndex == -1) - return $"{systemVersion} {codeName} [{kernelVersion}]"; - - string systemVersionTitle = systemVersion.Substring(0, firstDigitIndex).Trim(); - string systemVersionNumbers = systemVersion.Substring(firstDigitIndex).Trim(); - return $"{systemVersionTitle} {codeName} {systemVersionNumbers} [{kernelVersion}]"; - } - - return $"{systemVersion} [{kernelVersion}]"; - } - } -} diff --git a/src/BenchmarkDotNet/Environments/Platform.cs b/src/BenchmarkDotNet/Environments/Platform.cs index bc8f1ad26b..434a84ff98 100644 --- a/src/BenchmarkDotNet/Environments/Platform.cs +++ b/src/BenchmarkDotNet/Environments/Platform.cs @@ -51,5 +51,10 @@ public enum Platform /// A PowerPC 64-bit (little-endian) processor architecture. /// Ppc64le, + + /// + /// A RiscV 64-bit processor architecture. + /// + RiscV64, } } diff --git a/src/BenchmarkDotNet/Environments/ProcessorBrandStringHelper.cs b/src/BenchmarkDotNet/Environments/ProcessorBrandStringHelper.cs deleted file mode 100644 index 17f79e3202..0000000000 --- a/src/BenchmarkDotNet/Environments/ProcessorBrandStringHelper.cs +++ /dev/null @@ -1,155 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Text.RegularExpressions; -using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Environments -{ - public static class ProcessorBrandStringHelper - { - /// - /// Transform a processor brand string to a nice form for summary. - /// - /// The CPU information - /// Whether to include determined max frequency information - /// Prettified version - public static string Prettify(CpuInfo cpuInfo, bool includeMaxFrequency = false) - { - if (cpuInfo == null || string.IsNullOrEmpty(cpuInfo.ProcessorName)) - { - return "Unknown processor"; - } - - // Remove parts which don't provide any useful information for user - var processorName = cpuInfo.ProcessorName.Replace("@", "").Replace("(R)", "").Replace("(TM)", ""); - - // If we have found physical core(s), we can safely assume we can drop extra info from brand - if (cpuInfo.PhysicalCoreCount.HasValue && cpuInfo.PhysicalCoreCount.Value > 0) - processorName = Regex.Replace(processorName, @"(\w+?-Core Processor)", "").Trim(); - - string frequencyString = GetBrandStyledActualFrequency(cpuInfo.NominalFrequency); - if (includeMaxFrequency && frequencyString != null && !processorName.Contains(frequencyString)) - { - // show Max only if there's already a frequency name to differentiate the two - string maxFrequency = processorName.Contains("Hz") ? $"(Max: {frequencyString})" : frequencyString; - processorName = $"{processorName} {maxFrequency}"; - } - - // Remove double spaces - processorName = Regex.Replace(processorName.Trim(), @"\s+", " "); - - // Add microarchitecture name if known - string microarchitecture = ParseMicroarchitecture(processorName); - if (microarchitecture != null) - processorName = $"{processorName} ({microarchitecture})"; - - return processorName; - } - - /// - /// Presents actual processor's frequency into brand string format - /// - /// - private static string GetBrandStyledActualFrequency(Frequency? frequency) - { - if (frequency == null) - return null; - return $"{frequency.Value.ToGHz().ToString("N2", DefaultCultureInfo.Instance)}GHz"; - } - - /// - /// Parse a processor name and tries to return a microarchitecture name. - /// Works only for well-known microarchitectures. - /// - private static string? ParseMicroarchitecture(string processorName) - { - if (processorName.StartsWith("Intel Core")) - { - string model = processorName.Substring("Intel Core".Length).Trim(); - - // Core i3/5/7/9 - if ( - model.Length > 4 && - model[0] == 'i' && - (model[1] == '3' || model[1] == '5' || model[1] == '7' || model[1] == '9') && - (model[2] == '-' || model[2] == ' ')) - { - string modelNumber = model.Substring(3); - if (modelNumber.StartsWith("CPU")) - modelNumber = modelNumber.Substring(3).Trim(); - if (modelNumber.Contains("CPU")) - modelNumber = modelNumber.Substring(0, modelNumber.IndexOf("CPU", StringComparison.Ordinal)).Trim(); - return ParseIntelCoreMicroarchitecture(modelNumber); - } - } - - return null; - } - - private static readonly Lazy> KnownMicroarchitectures = new Lazy>(() => - { - var data = ResourceHelper.LoadResource("BenchmarkDotNet.Environments.microarchitectures.txt").Split('\r', '\n'); - var dictionary = new Dictionary(); - string currentMicroarchitecture = null; - foreach (string line in data) - { - if (line.StartsWith("//") || string.IsNullOrWhiteSpace(line)) - continue; - if (line.StartsWith("#")) - { - currentMicroarchitecture = line.Substring(1).Trim(); - continue; - } - - string modelNumber = line.Trim(); - if (dictionary.ContainsKey(modelNumber)) - throw new Exception($"{modelNumber} is defined twice in microarchitectures.txt"); - if (currentMicroarchitecture == null) - throw new Exception($"{modelNumber} doesn't have defined microarchitecture in microarchitectures.txt"); - dictionary[modelNumber] = currentMicroarchitecture; - } - - return dictionary; - }); - - // see http://www.intel.com/content/www/us/en/processors/processor-numbers.html - [SuppressMessage("ReSharper", "StringLiteralTypo")] - internal static string? ParseIntelCoreMicroarchitecture(string modelNumber) - { - if (KnownMicroarchitectures.Value.TryGetValue(modelNumber, out string? microarchitecture)) - return microarchitecture; - - if (modelNumber.Length >= 3 && modelNumber.Substring(0, 3).All(char.IsDigit) && - (modelNumber.Length == 3 || !char.IsDigit(modelNumber[3]))) - return "Nehalem"; - if (modelNumber.Length >= 4 && modelNumber.Substring(0, 4).All(char.IsDigit)) - { - char generation = modelNumber[0]; - switch (generation) - { - case '2': - return "Sandy Bridge"; - case '3': - return "Ivy Bridge"; - case '4': - return "Haswell"; - case '5': - return "Broadwell"; - case '6': - return "Skylake"; - case '7': - return "Kaby Lake"; - case '8': - return "Coffee Lake"; - default: - return null; - } - } - return null; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs index 35460ce853..21bb2821df 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs @@ -1,4 +1,5 @@ using System; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -17,7 +18,7 @@ public class ClrRuntime : Runtime, IEquatable public string Version { get; } - private ClrRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName, string version = null) + private ClrRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName, string? version = null) : base(runtimeMoniker, msBuildMoniker, displayName) { Version = version; @@ -44,7 +45,7 @@ public static ClrRuntime CreateForLocalFullNetFrameworkBuild(string version) internal static ClrRuntime GetCurrentVersion() { - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { throw new NotSupportedException(".NET Framework supports Windows OS only."); } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs index bd6923d95d..53476adbc8 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs @@ -20,8 +20,10 @@ public class CoreRuntime : Runtime public static readonly CoreRuntime Core60 = new (RuntimeMoniker.Net60, "net6.0", ".NET 6.0"); public static readonly CoreRuntime Core70 = new (RuntimeMoniker.Net70, "net7.0", ".NET 7.0"); public static readonly CoreRuntime Core80 = new (RuntimeMoniker.Net80, "net8.0", ".NET 8.0"); + public static readonly CoreRuntime Core90 = new (RuntimeMoniker.Net90, "net9.0", ".NET 9.0"); + public static readonly CoreRuntime Core10_0 = new (RuntimeMoniker.Net10_0, "net10.0", ".NET 10.0"); - public static CoreRuntime Latest => Core80; // when dotnet/runtime branches for 9.0, this will need to get updated + public static CoreRuntime Latest => Core10_0; // when dotnet/runtime branches for 11.0, this will need to get updated private CoreRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName) : base(runtimeMoniker, msBuildMoniker, displayName) @@ -72,12 +74,14 @@ internal static CoreRuntime FromVersion(Version version) case Version v when v.Major == 6 && v.Minor == 0: return GetPlatformSpecific(Core60); case Version v when v.Major == 7 && v.Minor == 0: return GetPlatformSpecific(Core70); case Version v when v.Major == 8 && v.Minor == 0: return GetPlatformSpecific(Core80); + case Version v when v.Major == 9 && v.Minor == 0: return GetPlatformSpecific(Core90); + case Version v when v.Major == 10 && v.Minor == 0: return GetPlatformSpecific(Core10_0); default: return CreateForNewVersion($"net{version.Major}.{version.Minor}", $".NET {version.Major}.{version.Minor}"); } } - internal static bool TryGetVersion(out Version version) + internal static bool TryGetVersion(out Version? version) { // we can't just use System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription // because it can be null and it reports versions like 4.6.* for .NET Core 2.* @@ -124,7 +128,7 @@ internal static bool TryGetVersion(out Version version) // sample input: // for dotnet run: C:\Program Files\dotnet\shared\Microsoft.NETCore.App\2.1.12\ // for dotnet publish: C:\Users\adsitnik\source\repos\ConsoleApp25\ConsoleApp25\bin\Release\netcoreapp2.0\win-x64\publish\ - internal static bool TryGetVersionFromRuntimeDirectory(string runtimeDirectory, out Version version) + internal static bool TryGetVersionFromRuntimeDirectory(string runtimeDirectory, out Version? version) { if (!string.IsNullOrEmpty(runtimeDirectory) && Version.TryParse(GetParsableVersionPart(new DirectoryInfo(runtimeDirectory).Name), out version)) { @@ -141,7 +145,7 @@ internal static bool TryGetVersionFromRuntimeDirectory(string runtimeDirectory, // 2.2: 4.6.27817.03 @BuiltBy: dlab14-DDVSOWINAGE101 @Branch: release/2.2 @SrcCode: https://github.com/dotnet/coreclr/tree/ce1d090d33b400a25620c0145046471495067cc7, Microsoft .NET Framework // 3.0: 3.0.0-preview8.19379.2+ac25be694a5385a6a1496db40de932df0689b742, Microsoft .NET Core // 5.0: 5.0.0-alpha1.19413.7+0ecefa44c9d66adb8a997d5778dc6c246ad393a7, Microsoft .NET Core - internal static bool TryGetVersionFromProductInfo(string productVersion, string productName, out Version version) + internal static bool TryGetVersionFromProductInfo(string productVersion, string productName, out Version? version) { if (!string.IsNullOrEmpty(productVersion) && !string.IsNullOrEmpty(productName)) { @@ -175,7 +179,7 @@ internal static bool TryGetVersionFromProductInfo(string productVersion, string // sample input: // .NETCoreApp,Version=v2.0 // .NETCoreApp,Version=v2.1 - internal static bool TryGetVersionFromFrameworkName(string frameworkName, out Version version) + internal static bool TryGetVersionFromFrameworkName(string frameworkName, out Version? version) { const string versionPrefix = ".NETCoreApp,Version=v"; if (!string.IsNullOrEmpty(frameworkName) && frameworkName.StartsWith(versionPrefix)) @@ -190,7 +194,7 @@ internal static bool TryGetVersionFromFrameworkName(string frameworkName, out Ve } // Version.TryParse does not handle thing like 3.0.0-WORD - private static string GetParsableVersionPart(string fullVersionName) => new string(fullVersionName.TakeWhile(c => char.IsDigit(c) || c == '.').ToArray()); + internal static string GetParsableVersionPart(string fullVersionName) => new string(fullVersionName.TakeWhile(c => char.IsDigit(c) || c == '.').ToArray()); private static CoreRuntime GetPlatformSpecific(CoreRuntime fallback) { diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs index e81e64f6c6..0b5d2c159b 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs @@ -20,7 +20,7 @@ public class MonoAotLLVMRuntime : Runtime, IEquatable /// /// creates new instance of MonoAotLLVMRuntime /// - public MonoAotLLVMRuntime(FileInfo aotCompilerPath, MonoAotCompilerMode aotCompilerMode, string msBuildMoniker = "net6.0", string displayName = "MonoAOTLLVM") : base(RuntimeMoniker.MonoAOTLLVM, msBuildMoniker, displayName) + public MonoAotLLVMRuntime(FileInfo aotCompilerPath, MonoAotCompilerMode aotCompilerMode, string msBuildMoniker = "net6.0", string displayName = "MonoAOTLLVM", RuntimeMoniker moniker = RuntimeMoniker.MonoAOTLLVM) : base(moniker, msBuildMoniker, displayName) { if (aotCompilerPath == null) throw new ArgumentNullException(paramName: nameof(aotCompilerPath)); diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs index 597e18b90c..15b5a02669 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs @@ -9,6 +9,8 @@ public class MonoRuntime : Runtime, IEquatable public static readonly MonoRuntime Mono60 = new ("Mono with .NET 6.0", RuntimeMoniker.Mono60, "net6.0", isDotNetBuiltIn: true); public static readonly MonoRuntime Mono70 = new ("Mono with .NET 7.0", RuntimeMoniker.Mono70, "net7.0", isDotNetBuiltIn: true); public static readonly MonoRuntime Mono80 = new ("Mono with .NET 8.0", RuntimeMoniker.Mono80, "net8.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Mono90 = new ("Mono with .NET 9.0", RuntimeMoniker.Mono90, "net9.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Mono10_0 = new ("Mono with .NET 10.0", RuntimeMoniker.Mono10_0, "net10.0", isDotNetBuiltIn: true); public string CustomPath { get; } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs index 063210b37c..bff018558e 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs @@ -18,6 +18,14 @@ public class NativeAotRuntime : Runtime /// NativeAOT compiled as net8.0 /// public static readonly NativeAotRuntime Net80 = new NativeAotRuntime(RuntimeMoniker.NativeAot80, "net8.0", "NativeAOT 8.0"); + /// + /// NativeAOT compiled as net9.0 + /// + public static readonly NativeAotRuntime Net90 = new NativeAotRuntime(RuntimeMoniker.NativeAot90, "net9.0", "NativeAOT 9.0"); + /// + /// NativeAOT compiled as net10.0 + /// + public static readonly NativeAotRuntime Net10_0 = new NativeAotRuntime(RuntimeMoniker.NativeAot10_0, "net10.0", "NativeAOT 10.0"); public override bool IsAOT => true; @@ -42,6 +50,9 @@ public static NativeAotRuntime GetCurrentVersion() { case Version v when v.Major == 6 && v.Minor == 0: return Net60; case Version v when v.Major == 7 && v.Minor == 0: return Net70; + case Version v when v.Major == 8 && v.Minor == 0: return Net80; + case Version v when v.Major == 9 && v.Minor == 0: return Net90; + case Version v when v.Major == 10 && v.Minor == 0: return Net10_0; default: return new NativeAotRuntime(RuntimeMoniker.NotRecognized, $"net{version.Major}.{version.Minor}", $"NativeAOT {version.Major}.{version.Minor}"); } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs b/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs index b695de321c..4a46e6456d 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs @@ -18,7 +18,7 @@ public abstract class Runtime : IEquatable public RuntimeMoniker RuntimeMoniker { get; } /// - /// MsBuild Target Framework Moniker, example: net462, netcoreapp2.1 + /// MsBuild Target Framework Moniker, example: net462, net8.0 /// public string MsBuildMoniker { get; } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/WasmRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/WasmRuntime.cs index bb6c122700..3fcc07f26a 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/WasmRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/WasmRuntime.cs @@ -28,7 +28,7 @@ public class WasmRuntime : Runtime, IEquatable /// Specifies whether AOT or Interpreter (default) project should be generated. /// Specifies a wasm data directory surfaced as $(WasmDataDir) for the project /// Runtime moniker - public WasmRuntime(string msBuildMoniker = "net5.0", string displayName = "Wasm", string javaScriptEngine = "v8", string javaScriptEngineArguments = "--expose_wasm", bool aot = false, string wasmDataDir = null, RuntimeMoniker moniker = RuntimeMoniker.Wasm) : base(moniker, msBuildMoniker, displayName) + public WasmRuntime(string msBuildMoniker = "net5.0", string displayName = "Wasm", string javaScriptEngine = "v8", string javaScriptEngineArguments = "--expose_wasm", bool aot = false, string? wasmDataDir = null, RuntimeMoniker moniker = RuntimeMoniker.Wasm) : base(moniker, msBuildMoniker, displayName) { if (!string.IsNullOrEmpty(javaScriptEngine) && javaScriptEngine != "v8" && !File.Exists(javaScriptEngine)) throw new FileNotFoundException($"Provided {nameof(javaScriptEngine)} file: \"{javaScriptEngine}\" doest NOT exist"); diff --git a/src/BenchmarkDotNet/Environments/microarchitectures.txt b/src/BenchmarkDotNet/Environments/microarchitectures.txt deleted file mode 100644 index f06d295390..0000000000 --- a/src/BenchmarkDotNet/Environments/microarchitectures.txt +++ /dev/null @@ -1,183 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Kaby Lake -// See: https://en.wikipedia.org/wiki/Kaby_Lake -//////////////////////////////////////////////////////////////////////////////// - -# Kaby Lake -7Y30 -7Y32 -7Y54 -7Y57 -7Y75 -3865U -3965U -4410Y -4415U -7100 -7100H -7100T -7100U -7101E -7101TE -7130U -7167U -7200U -7260U -7267U -7287U -7300 -7300HQ -7300T -7300U -7320 -7350K -7360U -7400 -7400T -7440HQ -7500 -7500T -7500U -7560U -7567U -7600 -7600K -7600T -7600U -7640X -7660U -7700 -7700HQ -7700K -7700T -7740X -7820HK -7820HQ -7920HQ -8130U -E3-1220 v6 -E3-1225 v6 -E3-1230 v6 -E3-1240 v6 -E3-1245 v6 -E3-1270 v6 -E3-1275 v6 -E3-1280 v6 -E3-1285 v6 -E3-1505L v6 -E3-1505M v6 -E3-1535M v6 -G3930 -G3930T -G3950 -G4560 -G4560T -G4600 -G4600T -G4620 - -# Kaby Lake R -8250U -8350U -8550U -8650U - -# Kaby Lake G -8305G -8705G -8706G -8709G -8809G - -# Amber Lake Y -8100Y -8200Y -8210Y -8500Y - -//////////////////////////////////////////////////////////////////////////////// -// Coffee Lake -// See: https://en.wikipedia.org/wiki/Coffee_Lake -//////////////////////////////////////////////////////////////////////////////// - -# Coffee Lake -610 -2104G -2124 -2124G -2126G -2134 -2136 -2144G -2146G -2174G -2176G -2176M -2186G -2186M -8086K -8100 -8100H -8100T -8109U -8259U -8269U -8300 -8300H -8300T -8350K -8400 -8400B -8400H -8400T -8500 -8500B -8500T -8559U -8600 -8600K -8600T -8700 -8700B -8700K -8700T -8750H -8850H -8950HK -9350KF -9400 -9400F -9600K -9600KF -9700K -9700KF -9900K -9900KF -G4900 -G4900T -G4920 -G5400 -G5400T -G5500 -G5500T -G5600 - -//////////////////////////////////////////////////////////////////////////////// -// Cannon Lake -// See: https://en.wikipedia.org/wiki/Cannon_Lake_(microarchitecture) -//////////////////////////////////////////////////////////////////////////////// - -# Cannon Lake -8121U - -//////////////////////////////////////////////////////////////////////////////// -// Whiskey Lake -// See: https://en.wikipedia.org/wiki/Whiskey_Lake_(microarchitecture) -//////////////////////////////////////////////////////////////////////////////// - -# Whiskey Lake -8565U -8265U -8145U -5405U -4205U \ No newline at end of file diff --git a/src/BenchmarkDotNet/EventProcessors/CompositeEventProcessor.cs b/src/BenchmarkDotNet/EventProcessors/CompositeEventProcessor.cs new file mode 100644 index 0000000000..a73ea54636 --- /dev/null +++ b/src/BenchmarkDotNet/EventProcessors/CompositeEventProcessor.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.Results; +using BenchmarkDotNet.Validators; + +namespace BenchmarkDotNet.EventProcessors +{ + internal sealed class CompositeEventProcessor : EventProcessor + { + private readonly HashSet eventProcessors; + + public CompositeEventProcessor(BenchmarkRunInfo[] benchmarkRunInfos) + { + var eventProcessors = new HashSet(); + + foreach (var info in benchmarkRunInfos) + eventProcessors.AddRange(info.Config.GetEventProcessors()); + + this.eventProcessors = eventProcessors; + } + + public override void OnStartValidationStage() + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnStartValidationStage(); + } + + public override void OnValidationError(ValidationError validationError) + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnValidationError(validationError); + } + + public override void OnEndValidationStage() + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnEndValidationStage(); + } + + public override void OnStartBuildStage(IReadOnlyList partitions) + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnStartBuildStage(partitions); + } + + public override void OnBuildComplete(BuildPartition buildPartition, BuildResult buildResult) + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnBuildComplete(buildPartition, buildResult); + } + + public override void OnEndBuildStage() + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnEndBuildStage(); + } + + public override void OnStartRunStage() + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnStartRunStage(); + } + + public override void OnEndRunStage() + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnEndRunStage(); + } + + public override void OnStartRunBenchmarksInType(Type type, IReadOnlyList benchmarks) + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnStartRunBenchmarksInType(type, benchmarks); + } + + public override void OnEndRunBenchmarksInType(Type type, Summary summary) + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnEndRunBenchmarksInType(type, summary); + } + + public override void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkReport report) + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnEndRunBenchmark(benchmarkCase, report); + } + + public override void OnStartRunBenchmark(BenchmarkCase benchmarkCase) + { + foreach (var eventProcessor in eventProcessors) + eventProcessor.OnStartRunBenchmark(benchmarkCase); + } + } +} diff --git a/src/BenchmarkDotNet/EventProcessors/EventProcessor.cs b/src/BenchmarkDotNet/EventProcessors/EventProcessor.cs new file mode 100644 index 0000000000..339b98fa5e --- /dev/null +++ b/src/BenchmarkDotNet/EventProcessors/EventProcessor.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.Results; +using BenchmarkDotNet.Validators; + +namespace BenchmarkDotNet.EventProcessors +{ + public abstract class EventProcessor + { + public virtual void OnStartValidationStage() { } + public virtual void OnValidationError(ValidationError validationError) { } + public virtual void OnEndValidationStage() { } + public virtual void OnStartBuildStage(IReadOnlyList partitions) { } + public virtual void OnBuildComplete(BuildPartition partition, BuildResult buildResult) { } + public virtual void OnEndBuildStage() { } + public virtual void OnStartRunStage() { } + public virtual void OnStartRunBenchmarksInType(Type type, IReadOnlyList benchmarks) { } + public virtual void OnEndRunBenchmarksInType(Type type, Summary summary) { } + public virtual void OnStartRunBenchmark(BenchmarkCase benchmarkCase) { } + public virtual void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkReport report) { } + public virtual void OnEndRunStage() { } + } +} diff --git a/src/BenchmarkDotNet/Exporters/Csv/CsvMeasurementsExporter.cs b/src/BenchmarkDotNet/Exporters/Csv/CsvMeasurementsExporter.cs index 74f6130f1c..0d07c9058c 100644 --- a/src/BenchmarkDotNet/Exporters/Csv/CsvMeasurementsExporter.cs +++ b/src/BenchmarkDotNet/Exporters/Csv/CsvMeasurementsExporter.cs @@ -20,7 +20,7 @@ public class CsvMeasurementsExporter : ExporterBase private static readonly Lazy Columns = new Lazy(BuildColumns); private readonly CsvSeparator separator; - public CsvMeasurementsExporter(CsvSeparator separator, SummaryStyle style = null) + public CsvMeasurementsExporter(CsvSeparator separator, SummaryStyle? style = null) { this.separator = separator; Style = style ?? SummaryStyle.Default; diff --git a/src/BenchmarkDotNet/Exporters/Json/JsonExporterBase.cs b/src/BenchmarkDotNet/Exporters/Json/JsonExporterBase.cs index 8ce4472a5a..3727ecac80 100644 --- a/src/BenchmarkDotNet/Exporters/Json/JsonExporterBase.cs +++ b/src/BenchmarkDotNet/Exporters/Json/JsonExporterBase.cs @@ -3,8 +3,9 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; +using Perfolizer.Helpers; using Perfolizer.Horology; -using JsonSerializer = SimpleJson.SimpleJson; +using SimpleJson; namespace BenchmarkDotNet.Exporters.Json { @@ -23,8 +24,8 @@ protected JsonExporterBase(bool indentJson = false, bool excludeMeasurements = f public override void ExportToLog(Summary summary, ILogger logger) { - JsonSerializer.CurrentJsonSerializerStrategy.Indent = IndentJson; - logger.WriteLine(JsonSerializer.SerializeObject(GetDataToSerialize(summary))); + SimpleJsonSerializer.CurrentJsonSerializerStrategy.Indent = IndentJson; + logger.WriteLine(SimpleJsonSerializer.SerializeObject(GetDataToSerialize(summary))); } protected virtual IReadOnlyDictionary GetDataToSerialize(Summary summary) @@ -47,11 +48,11 @@ protected virtual IReadOnlyDictionary GetDataToSerialize(HostEnv { { nameof(HostEnvironmentInfo.BenchmarkDotNetCaption), HostEnvironmentInfo.BenchmarkDotNetCaption }, { nameof(environmentInfo.BenchmarkDotNetVersion), environmentInfo.BenchmarkDotNetVersion }, - { "OsVersion", environmentInfo.OsVersion.Value }, - { "ProcessorName", ProcessorBrandStringHelper.Prettify(environmentInfo.CpuInfo.Value) }, - { "PhysicalProcessorCount", environmentInfo.CpuInfo.Value?.PhysicalProcessorCount }, - { "PhysicalCoreCount", environmentInfo.CpuInfo.Value?.PhysicalCoreCount }, - { "LogicalCoreCount", environmentInfo.CpuInfo.Value?.LogicalCoreCount }, + { "OsVersion", environmentInfo.Os.Value.ToBrandString() }, + { "ProcessorName", environmentInfo.Cpu.Value.ToShortBrandName() }, + { "PhysicalProcessorCount", environmentInfo.Cpu.Value?.PhysicalProcessorCount }, + { "PhysicalCoreCount", environmentInfo.Cpu.Value?.PhysicalCoreCount }, + { "LogicalCoreCount", environmentInfo.Cpu.Value?.LogicalCoreCount }, { nameof(environmentInfo.RuntimeVersion), environmentInfo.RuntimeVersion }, { nameof(environmentInfo.Architecture), environmentInfo.Architecture }, { nameof(environmentInfo.HasAttachedDebugger), environmentInfo.HasAttachedDebugger }, @@ -83,40 +84,40 @@ protected virtual IReadOnlyDictionary GetDataToSerialize(Benchma { "Statistics", report.ResultStatistics } }; - // We show MemoryDiagnoser's results only if it is being used - if (report.BenchmarkCase.Config.HasMemoryDiagnoser()) + // We show MemoryDiagnoser's results only if it is being used + if (report.BenchmarkCase.Config.HasMemoryDiagnoser()) + { + benchmark.Add("Memory", new { - benchmark.Add("Memory", new + report.GcStats.Gen0Collections, + report.GcStats.Gen1Collections, + report.GcStats.Gen2Collections, + report.GcStats.TotalOperations, + BytesAllocatedPerOperation = report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) + }); + } + + if (ExcludeMeasurements == false) + { + // We construct Measurements manually, so that we can have the IterationMode enum as text, rather than an integer + benchmark.Add("Measurements", + report.AllMeasurements.Select(m => new { - report.GcStats.Gen0Collections, - report.GcStats.Gen1Collections, - report.GcStats.Gen2Collections, - report.GcStats.TotalOperations, - BytesAllocatedPerOperation = report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) - }); - } + IterationMode = m.IterationMode.ToString(), + IterationStage = m.IterationStage.ToString(), + m.LaunchIndex, + m.IterationIndex, + m.Operations, + m.Nanoseconds + })); - if (ExcludeMeasurements == false) + if (report.Metrics.Any()) { - // We construct Measurements manually, so that we can have the IterationMode enum as text, rather than an integer - benchmark.Add("Measurements", - report.AllMeasurements.Select(m => new - { - IterationMode = m.IterationMode.ToString(), - IterationStage = m.IterationStage.ToString(), - m.LaunchIndex, - m.IterationIndex, - m.Operations, - m.Nanoseconds - })); - - if (report.Metrics.Any()) - { - benchmark.Add("Metrics", report.Metrics.Values); - } + benchmark.Add("Metrics", report.Metrics.Values); } + } - return benchmark; + return benchmark; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Exporters/Json/SimpleJson.cs b/src/BenchmarkDotNet/Exporters/Json/SimpleJsonSerializer.cs similarity index 99% rename from src/BenchmarkDotNet/Exporters/Json/SimpleJson.cs rename to src/BenchmarkDotNet/Exporters/Json/SimpleJsonSerializer.cs index 5aa3ebe1c4..e461c8a5c6 100644 --- a/src/BenchmarkDotNet/Exporters/Json/SimpleJson.cs +++ b/src/BenchmarkDotNet/Exporters/Json/SimpleJsonSerializer.cs @@ -1,3 +1,4 @@ +#nullable disable // ReSharper disable All //----------------------------------------------------------------------- // @@ -104,7 +105,7 @@ public JsonArray(int capacity) : base(capacity) { } /// The json representation of the array. public override string ToString() { - return SimpleJson.SerializeObject(this) ?? string.Empty; + return SimpleJsonSerializer.SerializeObject(this) ?? string.Empty; } } @@ -343,7 +344,7 @@ IEnumerator IEnumerable.GetEnumerator() /// public override string ToString() { - return SimpleJson.SerializeObject(this); + return SimpleJsonSerializer.SerializeObject(this); } #if SIMPLE_JSON_DYNAMIC @@ -500,7 +501,7 @@ namespace SimpleJson #else public #endif - static class SimpleJson + static class SimpleJsonSerializer { private const int TOKEN_NONE = 0; private const int TOKEN_CURLY_OPEN = 1; @@ -521,7 +522,7 @@ static class SimpleJson private static readonly char[] EscapeCharacters = new char[] { '"', '\\', '\b', '\f', '\n', '\r', '\t' }; private static readonly string EscapeCharactersString = new string(EscapeCharacters); - static SimpleJson() + static SimpleJsonSerializer() { EscapeTable = new char[93]; EscapeTable['"'] = '"'; diff --git a/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs b/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs index 668396baa8..2a106e0132 100644 --- a/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs +++ b/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs @@ -164,9 +164,9 @@ private void PrintTable(SummaryTable table, ILogger logger) logger.WriteStatistic(ColumnsStartWithSeparator ? TableHeaderSeparator.TrimStart().TrimEnd() + "-" : "-"); logger.WriteLineStatistic(string.Join("", - table.Columns.Where(c => c.NeedToShow).Select(column => + table.Columns.Where(c => c.NeedToShow).Select((column, index) => new string('-', column.Width - 1) + GetHeaderSeparatorIndicator(column.OriginalColumn.IsNumeric) + - GetHeaderSeparatorColumnDivider(column.Index, table.ColumnCount)))); + GetHeaderSeparatorColumnDivider(index, table.Columns.Where(c => c.NeedToShow).Count())))); } int rowCounter = 0; diff --git a/src/BenchmarkDotNet/Exporters/PhdJsonExporter.cs b/src/BenchmarkDotNet/Exporters/PhdJsonExporter.cs new file mode 100644 index 0000000000..25f14af3f0 --- /dev/null +++ b/src/BenchmarkDotNet/Exporters/PhdJsonExporter.cs @@ -0,0 +1,18 @@ +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using Perfolizer.Json; + +namespace BenchmarkDotNet.Exporters; + +/// +/// IMPORTANT: Not fully implemented yet +/// +public class PhdJsonExporter(LightJsonSettings? jsonSettings = null) : ExporterBase +{ + protected override string FileExtension => "phd.json"; + + public override void ExportToLog(Summary summary, ILogger logger) + { + logger.WriteLine(LightJsonSerializer.Serialize(summary.ToPhd(), jsonSettings)); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Exporters/PhdMdExporter.cs b/src/BenchmarkDotNet/Exporters/PhdMdExporter.cs new file mode 100644 index 0000000000..bc1c1e4609 --- /dev/null +++ b/src/BenchmarkDotNet/Exporters/PhdMdExporter.cs @@ -0,0 +1,24 @@ +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using Perfolizer.Json; +using Perfolizer.Phd.Presenting; +using Perfolizer.Phd.Tables; +using Perfolizer.Presenting; + +namespace BenchmarkDotNet.Exporters; + +/// +/// IMPORTANT: Not fully implemented yet +/// +public class PhdMdExporter : ExporterBase +{ + protected override string FileExtension => "phd.md"; + + public override void ExportToLog(Summary summary, ILogger logger) + { + var table = new PhdTable(summary.ToPhd()); + var presenter = new StringPresenter(); + new PhdMarkdownTablePresenter(presenter).Present(table, new PhdTableStyle()); + logger.WriteLine(presenter.Dump()); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Exporters/RPlotExporter.cs b/src/BenchmarkDotNet/Exporters/RPlotExporter.cs index 5c23f3b598..c90338f490 100644 --- a/src/BenchmarkDotNet/Exporters/RPlotExporter.cs +++ b/src/BenchmarkDotNet/Exporters/RPlotExporter.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Exporters.Csv; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; @@ -82,7 +83,7 @@ public void ExportToLog(Summary summary, ILogger logger) private static bool TryFindRScript(ILogger consoleLogger, out string? rscriptPath) { - string rscriptExecutable = RuntimeInformation.IsWindows() ? "Rscript.exe" : "Rscript"; + string rscriptExecutable = OsDetector.IsWindows() ? "Rscript.exe" : "Rscript"; rscriptPath = null; string rHome = Environment.GetEnvironmentVariable("R_HOME"); @@ -101,7 +102,7 @@ private static bool TryFindRScript(ILogger consoleLogger, out string? rscriptPat if (rscriptPath != null) return true; - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { string programFiles = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles); string programFilesR = Path.Combine(programFiles, "R"); diff --git a/src/BenchmarkDotNet/Exporters/Xml/SummaryDto.cs b/src/BenchmarkDotNet/Exporters/Xml/SummaryDto.cs index 13c9a74619..3087250b52 100644 --- a/src/BenchmarkDotNet/Exporters/Xml/SummaryDto.cs +++ b/src/BenchmarkDotNet/Exporters/Xml/SummaryDto.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Reports; using JetBrains.Annotations; +using Perfolizer.Helpers; using Perfolizer.Horology; // ReSharper disable UnusedMember.Global @@ -34,11 +35,11 @@ internal class HostEnvironmentInfoDto { public string BenchmarkDotNetCaption => HostEnvironmentInfo.BenchmarkDotNetCaption; public string BenchmarkDotNetVersion => hei.BenchmarkDotNetVersion; - public string OsVersion => hei.OsVersion.Value; - public string ProcessorName => ProcessorBrandStringHelper.Prettify(hei.CpuInfo.Value); - public string PhysicalProcessorCount => hei.CpuInfo.Value?.PhysicalProcessorCount?.ToString(); - public string PhysicalCoreCount => hei.CpuInfo.Value?.PhysicalCoreCount?.ToString(); - public string LogicalCoreCount => hei.CpuInfo.Value?.LogicalCoreCount?.ToString(); + public string OsVersion => hei.Os.Value.ToBrandString(); + public string ProcessorName => hei.Cpu.Value.ToShortBrandName(); + public string PhysicalProcessorCount => hei.Cpu.Value?.PhysicalProcessorCount?.ToString(); + public string PhysicalCoreCount => hei.Cpu.Value?.PhysicalCoreCount?.ToString(); + public string LogicalCoreCount => hei.Cpu.Value?.LogicalCoreCount?.ToString(); public string RuntimeVersion => hei.RuntimeVersion; public string Architecture => hei.Architecture; public bool HasAttachedDebugger => hei.HasAttachedDebugger; @@ -71,6 +72,8 @@ internal class BenchmarkReportDto public string MethodTitle => report.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo; public string Parameters => report.BenchmarkCase.Parameters.PrintInfo; public Statistics Statistics => report.ResultStatistics; + public IEnumerable Metrics => report.Metrics.Values; + public GcStats Memory => new GcStats() { Gen0Collections = report.GcStats.Gen0Collections, @@ -79,6 +82,7 @@ internal class BenchmarkReportDto TotalOperations = report.GcStats.TotalOperations, BytesAllocatedPerOperation = report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) }; + [PublicAPI] public IEnumerable Measurements { get; } private readonly BenchmarkReport report; @@ -104,4 +108,4 @@ internal struct GcStats public long TotalOperations { get; set; } public long? BytesAllocatedPerOperation { get; set; } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Exporters/Xml/XmlSerializer.cs b/src/BenchmarkDotNet/Exporters/Xml/XmlSerializer.cs index 1f4160393c..9ccbdd2323 100644 --- a/src/BenchmarkDotNet/Exporters/Xml/XmlSerializer.cs +++ b/src/BenchmarkDotNet/Exporters/Xml/XmlSerializer.cs @@ -110,7 +110,7 @@ private void WriteCollectionProperty(object source, PropertyInfo property) writer.WriteStartElement(property.Name); - string itemName = null; + string? itemName = null; foreach (var item in collection) { diff --git a/src/BenchmarkDotNet/Extensions/CommonExtensions.cs b/src/BenchmarkDotNet/Extensions/CommonExtensions.cs index a753787c53..aa402ee5dd 100644 --- a/src/BenchmarkDotNet/Extensions/CommonExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/CommonExtensions.cs @@ -20,11 +20,11 @@ public static string GetColumnTitle(this IColumn column, SummaryStyle style) switch (column.UnitType) { case UnitType.CodeSize: - return $"{column.ColumnName} [{style.CodeSizeUnit.Name}]"; + return $"{column.ColumnName} [{style.CodeSizeUnit.Abbreviation}]"; case UnitType.Size: - return $"{column.ColumnName} [{style.SizeUnit.Name}]"; + return $"{column.ColumnName} [{style.SizeUnit.Abbreviation}]"; case UnitType.Time: - return $"{column.ColumnName} [{style.TimeUnit.Name}]"; + return $"{column.ColumnName} [{style.TimeUnit.Abbreviation}]"; case UnitType.Dimensionless: return column.ColumnName; default: @@ -32,10 +32,12 @@ public static string GetColumnTitle(this IColumn column, SummaryStyle style) } } - public static bool IsNullOrEmpty(this IReadOnlyCollection value) => value == null || value.Count == 0; + public static bool IsNullOrEmpty(this IReadOnlyCollection? value) => value == null || value.Count == 0; public static bool IsEmpty(this IReadOnlyCollection value) => value.Count == 0; public static bool IsEmpty(this IEnumerable value) => !value.Any(); + public static IEnumerable WhereNotNull(this IEnumerable values) => values.Where(value => value != null).Cast(); + public static void AddRange(this HashSet hashSet, IEnumerable collection) { foreach (var item in collection) @@ -43,7 +45,7 @@ public static void AddRange(this HashSet hashSet, IEnumerable collectio } #if NETSTANDARD2_0 - public static TValue GetValueOrDefault(this IDictionary dictionary, TKey key) + public static TValue? GetValueOrDefault(this IDictionary dictionary, TKey key) => dictionary.TryGetValue(key, out var value) ? value : default; #endif @@ -97,7 +99,9 @@ internal static string DeleteFileIfExists(this string filePath) internal static string EnsureFolderExists(this string filePath) { - string directoryPath = Path.GetDirectoryName(filePath); + string? directoryPath = Path.GetDirectoryName(filePath); + if (directoryPath == null) + throw new ArgumentException($"Can't get directory path from '{filePath}'"); if (!Directory.Exists(directoryPath)) Directory.CreateDirectory(directoryPath); @@ -105,7 +109,7 @@ internal static string EnsureFolderExists(this string filePath) return filePath; } - internal static bool IsNotNullButDoesNotExist(this FileSystemInfo fileInfo) + internal static bool IsNotNullButDoesNotExist(this FileSystemInfo? fileInfo) => fileInfo != null && !fileInfo.Exists; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Extensions/MathExtensions.cs b/src/BenchmarkDotNet/Extensions/MathExtensions.cs new file mode 100644 index 0000000000..86a17f93f5 --- /dev/null +++ b/src/BenchmarkDotNet/Extensions/MathExtensions.cs @@ -0,0 +1,9 @@ +using System; + +namespace BenchmarkDotNet.Extensions; + +internal static class MathExtensions +{ + public static int RoundToInt(this double x) => (int)Math.Round(x); + public static long RoundToLong(this double x) => (long)Math.Round(x); +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs index 607e8c33fb..4d0854ba03 100644 --- a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.IO; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; @@ -32,7 +33,7 @@ public static void EnsureHighPriority(this Process process, ILogger logger) } catch (Exception ex) { - logger.WriteLineError($"Failed to set up high priority. Make sure you have the right permissions. Message: {ex.Message}"); + logger.WriteLineInfo($"// Failed to set up high priority ({ex.Message}). In order to run benchmarks with high priority, make sure you have the right permissions."); } } @@ -86,7 +87,7 @@ public static bool TrySetAffinity( if (logger == null) throw new ArgumentNullException(nameof(logger)); - if (!RuntimeInformation.IsWindows() && !RuntimeInformation.IsLinux()) + if (!OsDetector.IsWindows() && !OsDetector.IsLinux()) return false; try @@ -108,7 +109,7 @@ public static bool TrySetAffinity( if (process == null) throw new ArgumentNullException(nameof(process)); - if (!RuntimeInformation.IsWindows() && !RuntimeInformation.IsLinux()) + if (!OsDetector.IsWindows() && !OsDetector.IsLinux()) return null; try @@ -124,7 +125,7 @@ public static bool TrySetAffinity( internal static void SetEnvironmentVariables(this ProcessStartInfo start, BenchmarkCase benchmarkCase, IResolver resolver) { if (benchmarkCase.Job.Environment.Runtime is ClrRuntime clrRuntime && !string.IsNullOrEmpty(clrRuntime.Version)) - start.EnvironmentVariables["COMPLUS_Version"] = clrRuntime.Version; + SetClrEnvironmentVariables(start, "Version", clrRuntime.Version); if (benchmarkCase.Job.Environment.Runtime is MonoRuntime monoRuntime && !string.IsNullOrEmpty(monoRuntime.MonoBclPath)) start.EnvironmentVariables["MONO_PATH"] = monoRuntime.MonoBclPath; @@ -132,10 +133,10 @@ internal static void SetEnvironmentVariables(this ProcessStartInfo start, Benchm if (benchmarkCase.Config.HasPerfCollectProfiler()) { // enable tracing configuration inside of CoreCLR (https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/linux-performance-tracing.md#collecting-a-trace) - start.EnvironmentVariables["COMPlus_PerfMapEnabled"] = "1"; - start.EnvironmentVariables["COMPlus_EnableEventLog"] = "1"; + SetClrEnvironmentVariables(start, "PerfMapEnabled", "1"); + SetClrEnvironmentVariables(start, "EnableEventLog", "1"); // enable BDN Event Source (https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/linux-performance-tracing.md#filtering) - start.EnvironmentVariables["COMPlus_EventSourceFilter"] = EngineEventSource.SourceName; + SetClrEnvironmentVariables(start, "EventSourceFilter", EngineEventSource.SourceName); // workaround for https://github.com/dotnet/runtime/issues/71786, will be solved by next perf version start.EnvironmentVariables["DOTNET_EnableWriteXorExecute"] = "0"; } @@ -163,7 +164,7 @@ internal static void SetEnvironmentVariables(this ProcessStartInfo start, Benchm public static void KillTree(this Process process, TimeSpan timeout) { - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { RunProcessAndIgnoreOutput("taskkill", $"/T /F /PID {process.Id}", timeout); } @@ -257,21 +258,27 @@ private static void SetCoreRunEnvironmentVariables(this ProcessStartInfo start, { var gcMode = benchmarkCase.Job.Environment.Gc; - start.EnvironmentVariables["COMPlus_gcServer"] = gcMode.ResolveValue(GcMode.ServerCharacteristic, resolver) ? "1" : "0"; - start.EnvironmentVariables["COMPlus_gcConcurrent"] = gcMode.ResolveValue(GcMode.ConcurrentCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "gcServer", gcMode.ResolveValue(GcMode.ServerCharacteristic, resolver) ? "1" : "0"); + SetClrEnvironmentVariables(start, "gcConcurrent", gcMode.ResolveValue(GcMode.ConcurrentCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.CpuGroupsCharacteristic)) - start.EnvironmentVariables["COMPlus_GCCpuGroup"] = gcMode.ResolveValue(GcMode.CpuGroupsCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "GCCpuGroup", gcMode.ResolveValue(GcMode.CpuGroupsCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.AllowVeryLargeObjectsCharacteristic)) - start.EnvironmentVariables["COMPlus_gcAllowVeryLargeObjects"] = gcMode.ResolveValue(GcMode.AllowVeryLargeObjectsCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "gcAllowVeryLargeObjects", gcMode.ResolveValue(GcMode.AllowVeryLargeObjectsCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.RetainVmCharacteristic)) - start.EnvironmentVariables["COMPlus_GCRetainVM"] = gcMode.ResolveValue(GcMode.RetainVmCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "GCRetainVM", gcMode.ResolveValue(GcMode.RetainVmCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.NoAffinitizeCharacteristic)) - start.EnvironmentVariables["COMPlus_GCNoAffinitize"] = gcMode.ResolveValue(GcMode.NoAffinitizeCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "GCNoAffinitize", gcMode.ResolveValue(GcMode.NoAffinitizeCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.HeapAffinitizeMaskCharacteristic)) - start.EnvironmentVariables["COMPlus_GCHeapAffinitizeMask"] = gcMode.HeapAffinitizeMask.ToString("X"); + SetClrEnvironmentVariables(start, "GCHeapAffinitizeMask", gcMode.HeapAffinitizeMask.ToString("X")); if (gcMode.HasValue(GcMode.HeapCountCharacteristic)) - start.EnvironmentVariables["COMPlus_GCHeapCount"] = gcMode.HeapCount.ToString("X"); + SetClrEnvironmentVariables(start, "GCHeapCount", gcMode.HeapCount.ToString("X")); + } + + private static void SetClrEnvironmentVariables(ProcessStartInfo start, string suffix, string value) + { + start.EnvironmentVariables[$"DOTNET_{suffix}"] = value; + start.EnvironmentVariables[$"COMPlus_{suffix}"] = value; } } } diff --git a/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs b/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs index cf1f71a166..0a1a7c22fc 100644 --- a/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs @@ -155,10 +155,7 @@ internal static bool ContainsRunnableBenchmarks(this Type type) { var typeInfo = type.GetTypeInfo(); - if (typeInfo.IsAbstract - || typeInfo.IsSealed - || typeInfo.IsNotPublic - || typeInfo.IsGenericType && !IsRunnableGenericType(typeInfo)) + if (typeInfo.IsAbstract || typeInfo.IsGenericType && !IsRunnableGenericType(typeInfo)) return false; return typeInfo.GetBenchmarks().Any(); diff --git a/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs b/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs index 60fa23df3e..49c26320b3 100644 --- a/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs @@ -45,6 +45,10 @@ internal static Runtime GetRuntime(this RuntimeMoniker runtimeMoniker) return CoreRuntime.Core70; case RuntimeMoniker.Net80: return CoreRuntime.Core80; + case RuntimeMoniker.Net90: + return CoreRuntime.Core90; + case RuntimeMoniker.Net10_0: + return CoreRuntime.Core10_0; case RuntimeMoniker.Mono: return MonoRuntime.Default; case RuntimeMoniker.NativeAot60: @@ -53,10 +57,20 @@ internal static Runtime GetRuntime(this RuntimeMoniker runtimeMoniker) return NativeAotRuntime.Net70; case RuntimeMoniker.NativeAot80: return NativeAotRuntime.Net80; + case RuntimeMoniker.NativeAot90: + return NativeAotRuntime.Net90; + case RuntimeMoniker.NativeAot10_0: + return NativeAotRuntime.Net10_0; case RuntimeMoniker.Mono60: return MonoRuntime.Mono60; case RuntimeMoniker.Mono70: return MonoRuntime.Mono70; + case RuntimeMoniker.Mono80: + return MonoRuntime.Mono80; + case RuntimeMoniker.Mono90: + return MonoRuntime.Mono90; + case RuntimeMoniker.Mono10_0: + return MonoRuntime.Mono10_0; default: throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "Runtime Moniker not supported"); } diff --git a/src/BenchmarkDotNet/Extensions/StatisticsExtensions.cs b/src/BenchmarkDotNet/Extensions/StatisticsExtensions.cs index 7abb00768b..922dc50d44 100644 --- a/src/BenchmarkDotNet/Extensions/StatisticsExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/StatisticsExtensions.cs @@ -1,6 +1,7 @@ using System; using System.Globalization; using System.Text; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Mathematics; using JetBrains.Annotations; using Perfolizer.Horology; @@ -16,11 +17,11 @@ public static class StatisticsExtensions public static Func CreateNanosecondFormatter(this Statistics s, CultureInfo cultureInfo, string format = "N3") { var timeUnit = TimeUnit.GetBestTimeUnit(s.Mean); - return x => TimeInterval.FromNanoseconds(x).ToString(timeUnit, cultureInfo, format); + return x => TimeInterval.FromNanoseconds(x).ToString(timeUnit, format, cultureInfo, UnitHelper.DefaultPresentation); } [PublicAPI] - public static string ToString(this Statistics s, CultureInfo cultureInfo, Func formatter, bool calcHistogram = false) + public static string ToString(this Statistics? s, CultureInfo cultureInfo, Func formatter, bool calcHistogram = false) { if (s == null) return NullSummaryMessage; @@ -29,9 +30,9 @@ public static string ToString(this Statistics s, CultureInfo cultureInfo, Func> InvalidFileNameCharactersMappings = new Lazy>(BuildInvalidPathCharactersMappings); @@ -73,7 +73,7 @@ private static Dictionary BuildInvalidPathCharactersMappings() return invalidFileNameChars.ToDictionary( character => character.ToString(), - character => $"char{(short) character}"); + character => $"char{(short)character}"); } /// @@ -150,5 +150,8 @@ internal static StringBuilder AppendArgument(this StringBuilder stringBuilder, s /// The string builder with the arguments added internal static StringBuilder AppendArgument(this StringBuilder stringBuilder, object argument) => argument == null ? stringBuilder : AppendArgument(stringBuilder, argument.ToString()); + + public static bool IsBlank(this string? value) => string.IsNullOrWhiteSpace(value); + public static bool IsNotBlank(this string? value) => !value.IsBlank(); } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs b/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs index 62b4d31d94..6873bb834b 100644 --- a/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs +++ b/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Linq; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; @@ -26,7 +27,7 @@ internal static string GetFilePath(DiagnoserActionParameters details, string? su // long paths can be enabled on Windows but it does not mean that everything is going to work fine.. // so we always use 260 as limit on Windows - int limit = RuntimeInformation.IsWindows() + int limit = OsDetector.IsWindows() ? WindowsOldPathLimit - reserve : CommonSenseLimit; @@ -68,7 +69,7 @@ private static string GetLimitedFilePath(DiagnoserActionParameters details, stri private static string GetFilePath(string fileName, DiagnoserActionParameters details, string? subfolder, DateTime? creationTime, string fileExtension) { - // if we run for more than one toolchain, the output file name should contain the name too so we can differ net462 vs netcoreapp2.1 etc + // if we run for more than one toolchain, the output file name should contain the name too so we can differ net462 vs net8.0 etc if (details.Config.GetJobs().Select(job => ToolchainExtensions.GetToolchain(job)).Distinct().Count() > 1) fileName += $"-{details.BenchmarkCase.Job.Environment.Runtime?.Name ?? details.BenchmarkCase.GetToolchain()?.Name ?? details.BenchmarkCase.Job.Id}"; diff --git a/src/BenchmarkDotNet/Helpers/Assertion.cs b/src/BenchmarkDotNet/Helpers/Assertion.cs new file mode 100644 index 0000000000..44e8b36e46 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/Assertion.cs @@ -0,0 +1,14 @@ +using System; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Helpers; + +internal static class Assertion +{ + [AssertionMethod] + public static void NotNull(string name, object? value) + { + if (value == null) + throw new ArgumentNullException(name, $"{name} can't be null"); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/AwaitHelper.cs b/src/BenchmarkDotNet/Helpers/AwaitHelper.cs new file mode 100644 index 0000000000..8d16fb716a --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/AwaitHelper.cs @@ -0,0 +1,108 @@ +using System; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace BenchmarkDotNet.Helpers +{ + public static class AwaitHelper + { + private class ValueTaskWaiter + { + // We use thread static field so that each thread uses its own individual callback and reset event. + [ThreadStatic] + private static ValueTaskWaiter ts_current; + internal static ValueTaskWaiter Current => ts_current ??= new ValueTaskWaiter(); + + // We cache the callback to prevent allocations for memory diagnoser. + private readonly Action awaiterCallback; + private readonly ManualResetEventSlim resetEvent; + + private ValueTaskWaiter() + { + resetEvent = new (); + awaiterCallback = resetEvent.Set; + } + + internal void Wait(TAwaiter awaiter) where TAwaiter : ICriticalNotifyCompletion + { + resetEvent.Reset(); + awaiter.UnsafeOnCompleted(awaiterCallback); + + // The fastest way to wait for completion is to spin a bit before waiting on the event. This is the same logic that Task.GetAwaiter().GetResult() uses. + var spinner = new SpinWait(); + while (!resetEvent.IsSet) + { + if (spinner.NextSpinWillYield) + { + resetEvent.Wait(); + return; + } + spinner.SpinOnce(); + } + } + } + + // we use GetAwaiter().GetResult() because it's fastest way to obtain the result in blocking way, + // and will eventually throw actual exception, not aggregated one + public static void GetResult(Task task) => task.GetAwaiter().GetResult(); + + public static T GetResult(Task task) => task.GetAwaiter().GetResult(); + + // ValueTask can be backed by an IValueTaskSource that only supports asynchronous awaits, + // so we have to hook up a callback instead of calling .GetAwaiter().GetResult() like we do for Task. + // The alternative is to convert it to Task using .AsTask(), but that causes allocations which we must avoid for memory diagnoser. + public static void GetResult(ValueTask task) + { + // Don't continue on the captured context, as that may result in a deadlock if the user runs this in-process. + var awaiter = task.ConfigureAwait(false).GetAwaiter(); + if (!awaiter.IsCompleted) + { + ValueTaskWaiter.Current.Wait(awaiter); + } + awaiter.GetResult(); + } + + public static T GetResult(ValueTask task) + { + // Don't continue on the captured context, as that may result in a deadlock if the user runs this in-process. + var awaiter = task.ConfigureAwait(false).GetAwaiter(); + if (!awaiter.IsCompleted) + { + ValueTaskWaiter.Current.Wait(awaiter); + } + return awaiter.GetResult(); + } + + internal static MethodInfo GetGetResultMethod(Type taskType) + { + if (!taskType.IsGenericType) + { + return typeof(AwaitHelper).GetMethod(nameof(AwaitHelper.GetResult), BindingFlags.Public | BindingFlags.Static, null, new Type[1] { taskType }, null); + } + + Type compareType = taskType.GetGenericTypeDefinition() == typeof(ValueTask<>) ? typeof(ValueTask<>) + : typeof(Task).IsAssignableFrom(taskType.GetGenericTypeDefinition()) ? typeof(Task<>) + : null; + if (compareType == null) + { + return null; + } + var resultType = taskType + .GetMethod(nameof(Task.GetAwaiter), BindingFlags.Public | BindingFlags.Instance) + .ReturnType + .GetMethod(nameof(TaskAwaiter.GetResult), BindingFlags.Public | BindingFlags.Instance) + .ReturnType; + return typeof(AwaitHelper).GetMethods(BindingFlags.Public | BindingFlags.Static) + .First(m => + { + if (m.Name != nameof(AwaitHelper.GetResult)) return false; + Type paramType = m.GetParameters().First().ParameterType; + return paramType.IsGenericType && paramType.GetGenericTypeDefinition() == compareType; + }) + .MakeGenericMethod(new[] { resultType }); + } + } +} diff --git a/src/BenchmarkDotNet/Helpers/DirtyAssemblyResolveHelper.cs b/src/BenchmarkDotNet/Helpers/DirtyAssemblyResolveHelper.cs index a16e15f653..0921b7d1d6 100644 --- a/src/BenchmarkDotNet/Helpers/DirtyAssemblyResolveHelper.cs +++ b/src/BenchmarkDotNet/Helpers/DirtyAssemblyResolveHelper.cs @@ -28,7 +28,7 @@ internal class DirtyAssemblyResolveHelper : IDisposable /// "the handler is invoked whenever the runtime fails to bind to an assembly by name." /// /// not null when we find it manually, null when can't help - private Assembly HelpTheFrameworkToResolveTheAssembly(object sender, ResolveEventArgs args) + private Assembly? HelpTheFrameworkToResolveTheAssembly(object sender, ResolveEventArgs args) { var fullName = new AssemblyName(args.Name); string simpleName = fullName.Name; diff --git a/src/BenchmarkDotNet/Helpers/DisposeAtProcessTermination.cs b/src/BenchmarkDotNet/Helpers/DisposeAtProcessTermination.cs new file mode 100644 index 0000000000..12aa89a6fa --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/DisposeAtProcessTermination.cs @@ -0,0 +1,57 @@ +using System; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Helpers +{ + /// + /// Ensures that explicit Dispose is called at termination of the Process. + /// + /// + /// + /// This class exists to help in reverting system state where C#'s using statement does not + /// suffice. I.e. when Benchmark's process is aborted via Ctrl-C, Ctrl-Break or via click on the + /// X in the upper right of Window. + /// + /// + /// Usage: Derive your clas that changes system state of this class. Revert system state in + /// override of implementation. + /// Use your class in C#'s using statement, to ensure system state is reverted in normal situations. + /// This class ensures your override is also called at process 'abort'. + /// + /// + /// Note: This class is explicitly not responsible for cleanup of Native resources. Of course, + /// derived classes can cleanup their Native resources (usually managed via + /// derived classes), by delegating explicit Disposal to their + /// fields. + /// + /// + public abstract class DisposeAtProcessTermination : IDisposable + { + public DisposeAtProcessTermination() + { + Console.CancelKeyPress += OnCancelKeyPress; + AppDomain.CurrentDomain.ProcessExit += OnProcessExit; + // It does not make sense to include a Finalizer. We do not manage any native resource and: + // as we are subscribed to static events, it would never be called. + } + + /// + /// Called when the user presses Ctrl-C or Ctrl-Break. + /// + private void OnCancelKeyPress(object? sender, ConsoleCancelEventArgs e) + { + if (!e.Cancel) { Dispose(); } + } + + /// + /// Called when the user clicks on the X in the upper right corner to close the Benchmark's Window. + /// + private void OnProcessExit(object? sender, EventArgs e) => Dispose(); + + public virtual void Dispose() + { + Console.CancelKeyPress -= OnCancelKeyPress; + AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; + } + } +} diff --git a/src/BenchmarkDotNet/Helpers/ExternalToolsHelper.cs b/src/BenchmarkDotNet/Helpers/ExternalToolsHelper.cs index 3802608047..bd5274c6b3 100644 --- a/src/BenchmarkDotNet/Helpers/ExternalToolsHelper.cs +++ b/src/BenchmarkDotNet/Helpers/ExternalToolsHelper.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Helpers @@ -11,7 +12,7 @@ public static class ExternalToolsHelper /// MacOSX only. /// public static readonly Lazy> MacSystemProfilerData = - LazyParse(RuntimeInformation.IsMacOS, "system_profiler", "SPSoftwareDataType", s => SectionsHelper.ParseSection(s, ':')); + LazyParse(OsDetector.IsMacOS, "system_profiler", "SPSoftwareDataType", s => SectionsHelper.ParseSection(s, ':')); private static Lazy LazyParse(Func isAvailable, string fileName, string arguments, Func parseFunc) { diff --git a/src/BenchmarkDotNet/Helpers/ProcessHelper.cs b/src/BenchmarkDotNet/Helpers/ProcessHelper.cs index 114ae287fb..caa3410b21 100644 --- a/src/BenchmarkDotNet/Helpers/ProcessHelper.cs +++ b/src/BenchmarkDotNet/Helpers/ProcessHelper.cs @@ -12,7 +12,8 @@ internal static class ProcessHelper /// Run external process and return the console output. /// In the case of any exception, null will be returned. /// - internal static string? RunAndReadOutput(string fileName, string arguments = "", ILogger logger = null) + internal static string? RunAndReadOutput(string fileName, string arguments = "", ILogger? logger = null, + Dictionary? environmentVariables = null) { var processStartInfo = new ProcessStartInfo { @@ -24,6 +25,9 @@ internal static class ProcessHelper RedirectStandardOutput = true, RedirectStandardError = true }; + if (environmentVariables != null) + foreach (var variable in environmentVariables) + processStartInfo.Environment[variable.Key] = variable.Value; using (var process = new Process { StartInfo = processStartInfo }) using (new ConsoleExitHandler(process, logger ?? NullLogger.Instance)) { @@ -42,7 +46,7 @@ internal static class ProcessHelper } internal static (int exitCode, ImmutableArray output) RunAndReadOutputLineByLine(string fileName, string arguments = "", string workingDirectory = "", - Dictionary environmentVariables = null, bool includeErrors = false, ILogger logger = null) + Dictionary? environmentVariables = null, bool includeErrors = false, ILogger? logger = null) { var processStartInfo = new ProcessStartInfo { diff --git a/src/BenchmarkDotNet/Helpers/Taskbar/TaskbarProgress.cs b/src/BenchmarkDotNet/Helpers/Taskbar/TaskbarProgress.cs deleted file mode 100644 index 19ad5f6ffe..0000000000 --- a/src/BenchmarkDotNet/Helpers/Taskbar/TaskbarProgress.cs +++ /dev/null @@ -1,197 +0,0 @@ -using System; -using System.Runtime.InteropServices; - -namespace BenchmarkDotNet.Helpers -{ - internal class TaskbarProgress : IDisposable - { - private static readonly bool OsVersionIsSupported = Portability.RuntimeInformation.IsWindows() - // Must be windows 7 or greater - && Environment.OSVersion.Version >= new Version(6, 1); - - private IntPtr consoleWindowHandle = IntPtr.Zero; - private IntPtr consoleHandle = IntPtr.Zero; - - [DllImport("kernel32.dll")] - private static extern IntPtr GetConsoleWindow(); - [DllImport("kernel32.dll", SetLastError = true)] - private static extern IntPtr GetStdHandle(int nStdHandle); - - private const int STD_OUTPUT_HANDLE = -11; - - internal TaskbarProgress() - { - if (OsVersionIsSupported) - { - consoleWindowHandle = GetConsoleWindow(); - consoleHandle = GetStdHandle(STD_OUTPUT_HANDLE); - Console.CancelKeyPress += OnConsoleCancelEvent; - } - } - - internal void SetState(TaskbarProgressState state) - { - if (OsVersionIsSupported) - { - TaskbarProgressCom.SetState(consoleWindowHandle, consoleHandle, state); - } - } - - internal void SetProgress(float progressValue) - { - if (OsVersionIsSupported) - { - TaskbarProgressCom.SetValue(consoleWindowHandle, consoleHandle, progressValue); - } - } - - private void OnConsoleCancelEvent(object sender, ConsoleCancelEventArgs e) - { - Dispose(); - } - - public void Dispose() - { - if (OsVersionIsSupported) - { - TaskbarProgressCom.SetState(consoleWindowHandle, consoleHandle, TaskbarProgressState.NoProgress); - consoleWindowHandle = IntPtr.Zero; - consoleHandle = IntPtr.Zero; - Console.CancelKeyPress -= OnConsoleCancelEvent; - } - } - } - - internal enum TaskbarProgressState - { - NoProgress = 0, - Indeterminate = 0x1, - Normal = 0x2, - Error = 0x4, - Paused = 0x8, - Warning = Paused - } - - internal static class TaskbarProgressCom - { - [DllImport("kernel32.dll", SetLastError = true)] - private static extern bool GetConsoleMode(IntPtr hConsoleHandle, out ConsoleModes lpMode); - [DllImport("kernel32.dll", SetLastError = true)] - private static extern bool SetConsoleMode(IntPtr hConsoleHandle, ConsoleModes dwMode); - - [Flags] - private enum ConsoleModes : uint - { - ENABLE_PROCESSED_INPUT = 0x0001, - ENABLE_LINE_INPUT = 0x0002, - ENABLE_ECHO_INPUT = 0x0004, - ENABLE_WINDOW_INPUT = 0x0008, - ENABLE_MOUSE_INPUT = 0x0010, - ENABLE_INSERT_MODE = 0x0020, - ENABLE_QUICK_EDIT_MODE = 0x0040, - ENABLE_EXTENDED_FLAGS = 0x0080, - ENABLE_AUTO_POSITION = 0x0100, - - ENABLE_PROCESSED_OUTPUT = 0x0001, - ENABLE_WRAP_AT_EOL_OUTPUT = 0x0002, - ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004, - DISABLE_NEWLINE_AUTO_RETURN = 0x0008, - ENABLE_LVB_GRID_WORLDWIDE = 0x0010 - } - - [ComImport] - [Guid("ea1afb91-9e28-4b86-90e9-9e9f8a5eefaf")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - private interface ITaskbarList3 - { - // ITaskbarList - [PreserveSig] - void HrInit(); - [PreserveSig] - void AddTab(IntPtr hwnd); - [PreserveSig] - void DeleteTab(IntPtr hwnd); - [PreserveSig] - void ActivateTab(IntPtr hwnd); - [PreserveSig] - void SetActiveAlt(IntPtr hwnd); - - // ITaskbarList2 - [PreserveSig] - void MarkFullscreenWindow(IntPtr hwnd, [MarshalAs(UnmanagedType.Bool)] bool fFullscreen); - - // ITaskbarList3 - [PreserveSig] - void SetProgressValue(IntPtr hwnd, ulong ullCompleted, ulong ullTotal); - [PreserveSig] - void SetProgressState(IntPtr hwnd, TaskbarProgressState state); - } - - [Guid("56FDF344-FD6D-11d0-958A-006097C9A090")] - [ClassInterface(ClassInterfaceType.None)] - [ComImport] - private class TaskbarInstance - { - } - - private static readonly ITaskbarList3 s_taskbarInstance = (ITaskbarList3) new TaskbarInstance(); - - internal static void SetState(IntPtr consoleWindowHandle, IntPtr consoleHandle, TaskbarProgressState taskbarState) - { - if (consoleWindowHandle != IntPtr.Zero) - { - s_taskbarInstance.SetProgressState(consoleWindowHandle, taskbarState); - } - - if (consoleHandle != IntPtr.Zero) - { - // Write progress state to console for Windows Terminal (https://github.com/microsoft/terminal/issues/6700). - GetConsoleMode(consoleHandle, out ConsoleModes previousConsoleMode); - SetConsoleMode(consoleHandle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT); - switch (taskbarState) - { - case TaskbarProgressState.NoProgress: - Console.Write("\x1b]9;4;0;0\x1b\\"); - break; - case TaskbarProgressState.Indeterminate: - Console.Write("\x1b]9;4;3;0\x1b\\"); - break; - case TaskbarProgressState.Normal: - // Do nothing, this is set automatically when SetValue is called (and WT has no documented way to set this). - break; - case TaskbarProgressState.Error: - Console.Write("\x1b]9;4;2;0\x1b\\"); - break; - case TaskbarProgressState.Warning: - Console.Write("\x1b]9;4;4;0\x1b\\"); - break; - } - SetConsoleMode(consoleHandle, previousConsoleMode); - } - } - - internal static void SetValue(IntPtr consoleWindowHandle, IntPtr consoleHandle, float progressValue) - { - bool isValidRange = progressValue >= 0 & progressValue <= 1; - if (!isValidRange) - { - throw new ArgumentOutOfRangeException(nameof(progressValue), "progressValue must be between 0 and 1 inclusive."); - } - uint value = (uint) (progressValue * 100); - - if (consoleWindowHandle != IntPtr.Zero) - { - s_taskbarInstance.SetProgressValue(consoleWindowHandle, value, 100); - } - - if (consoleHandle != IntPtr.Zero) - { - // Write progress sequence to console for Windows Terminal (https://github.com/microsoft/terminal/discussions/14268). - GetConsoleMode(consoleHandle, out ConsoleModes previousConsoleMode); - SetConsoleMode(consoleHandle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT); - Console.Write($"\x1b]9;4;1;{value}\x1b\\"); - SetConsoleMode(consoleHandle, previousConsoleMode); - } - } - } -} diff --git a/src/BenchmarkDotNet/Helpers/TaskbarProgress.cs b/src/BenchmarkDotNet/Helpers/TaskbarProgress.cs new file mode 100644 index 0000000000..e3910a0924 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/TaskbarProgress.cs @@ -0,0 +1,252 @@ +using System; +using System.Runtime.InteropServices; +using BenchmarkDotNet.Detectors; + +namespace BenchmarkDotNet.Helpers +{ + internal enum TaskbarProgressState + { + NoProgress = 0, + Indeterminate = 0x1, + Normal = 0x2, + Error = 0x4, + Paused = 0x8, + Warning = Paused + } + + internal class TaskbarProgress : DisposeAtProcessTermination + { + private static readonly bool OsVersionIsSupported = OsDetector.IsWindows() + // Must be windows 7 or greater + && Environment.OSVersion.Version >= new Version(6, 1); + + private Com? com; + private Terminal? terminal; + + private bool IsEnabled => com != null || terminal != null; + + internal TaskbarProgress(TaskbarProgressState initialTaskbarState) + { + if (OsVersionIsSupported) + { + com = Com.MaybeCreateInstanceAndSetInitialState(initialTaskbarState); + terminal = Terminal.MaybeCreateInstanceAndSetInitialState(initialTaskbarState); + } + } + + internal void SetState(TaskbarProgressState state) + { + com?.SetState(state); + terminal?.SetState(state); + } + + internal void SetProgress(float progressValue) + { + bool isValidRange = progressValue >= 0 & progressValue <= 1; + if (!isValidRange) + { + throw new ArgumentOutOfRangeException(nameof(progressValue), "progressValue must be between 0 and 1 inclusive."); + } + uint value = (uint)(progressValue * 100); + com?.SetValue(value); + terminal?.SetValue(value); + } + + public override void Dispose() + { + if (IsEnabled) + { + SetState(TaskbarProgressState.NoProgress); + com = null; + terminal = null; + } + base.Dispose(); + } + + private sealed class Com + { + [ComImport] + [Guid("ea1afb91-9e28-4b86-90e9-9e9f8a5eefaf")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + private interface ITaskbarList3 + { + // ITaskbarList + [PreserveSig] + void HrInit(); + [PreserveSig] + void AddTab(IntPtr hwnd); + [PreserveSig] + void DeleteTab(IntPtr hwnd); + [PreserveSig] + void ActivateTab(IntPtr hwnd); + [PreserveSig] + void SetActiveAlt(IntPtr hwnd); + + // ITaskbarList2 + [PreserveSig] + void MarkFullscreenWindow(IntPtr hwnd, [MarshalAs(UnmanagedType.Bool)] bool fFullscreen); + + // ITaskbarList3 + [PreserveSig] + void SetProgressValue(IntPtr hwnd, ulong ullCompleted, ulong ullTotal); + [PreserveSig] + void SetProgressState(IntPtr hwnd, TaskbarProgressState state); + } + + [Guid("56FDF344-FD6D-11d0-958A-006097C9A090")] + [ClassInterface(ClassInterfaceType.None)] + [ComImport] + private class TaskbarInstance { } + + [DllImport("kernel32.dll")] + private static extern IntPtr GetConsoleWindow(); + + private readonly ITaskbarList3 taskbarInstance; + private readonly IntPtr consoleWindowHandle; + + private Com(IntPtr handle) + { + taskbarInstance = (ITaskbarList3) new TaskbarInstance(); + consoleWindowHandle = handle; + } + + internal static Com? MaybeCreateInstanceAndSetInitialState(TaskbarProgressState initialTaskbarState) + { + try + { + IntPtr handle = GetConsoleWindow(); + if (handle == IntPtr.Zero) + { + return null; + } + var com = new Com(handle); + com.SetState(initialTaskbarState); + return com; + } + // COM may be disabled, in which case this will throw (#2253). + // It could be NotSupportedException or COMException, we just catch all. + catch + { + return null; + } + } + + internal void SetState(TaskbarProgressState taskbarState) + => taskbarInstance.SetProgressState(consoleWindowHandle, taskbarState); + + /// + /// Sets the progress value out of 100. + /// + internal void SetValue(uint progressValue) + => taskbarInstance.SetProgressValue(consoleWindowHandle, progressValue, 100); + } + + private sealed class Terminal + { + [Flags] + private enum ConsoleModes : uint + { + ENABLE_PROCESSED_INPUT = 0x0001, + ENABLE_LINE_INPUT = 0x0002, + ENABLE_ECHO_INPUT = 0x0004, + ENABLE_WINDOW_INPUT = 0x0008, + ENABLE_MOUSE_INPUT = 0x0010, + ENABLE_INSERT_MODE = 0x0020, + ENABLE_QUICK_EDIT_MODE = 0x0040, + ENABLE_EXTENDED_FLAGS = 0x0080, + ENABLE_AUTO_POSITION = 0x0100, + + ENABLE_PROCESSED_OUTPUT = 0x0001, + ENABLE_WRAP_AT_EOL_OUTPUT = 0x0002, + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004, + DISABLE_NEWLINE_AUTO_RETURN = 0x0008, + ENABLE_LVB_GRID_WORLDWIDE = 0x0010 + } + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool GetConsoleMode(IntPtr hConsoleHandle, out ConsoleModes lpMode); + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool SetConsoleMode(IntPtr hConsoleHandle, ConsoleModes dwMode); + [DllImport("kernel32.dll", SetLastError = true)] + private static extern IntPtr GetStdHandle(int nStdHandle); + private const int STD_OUTPUT_HANDLE = -11; + + private readonly IntPtr consoleHandle; + private uint currentProgress; + + private Terminal(IntPtr handle) + => consoleHandle = handle; + + internal static Terminal? MaybeCreateInstanceAndSetInitialState(TaskbarProgressState initialTaskbarState) + { + IntPtr handle = GetStdHandle(STD_OUTPUT_HANDLE); + if (handle == IntPtr.Zero) + { + return null; + } + if (!GetConsoleMode(handle, out ConsoleModes previousConsoleMode) + || !SetConsoleMode(handle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT)) + { + // If we failed to set virtual terminal processing mode, it is likely due to an older Windows version that does not support it, + // or legacy console. In either case the TaskbarProgressCom will take care of the progress. See https://stackoverflow.com/a/44574463/5703407. + // If we try to write without VT mode, the sequence will be printed for the user to see, which clutters the output. + return null; + } + var terminal = new Terminal(handle); + terminal.WriteStateSequence(initialTaskbarState); + SetConsoleMode(handle, previousConsoleMode); + return terminal; + } + + internal void SetState(TaskbarProgressState taskbarState) + { + GetConsoleMode(consoleHandle, out ConsoleModes previousConsoleMode); + SetConsoleMode(consoleHandle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT); + WriteStateSequence(taskbarState); + SetConsoleMode(consoleHandle, previousConsoleMode); + } + + private void WriteStateSequence(TaskbarProgressState taskbarState) + { + // Write progress state to console for Windows Terminal (https://github.com/microsoft/terminal/issues/6700). + switch (taskbarState) + { + case TaskbarProgressState.NoProgress: + currentProgress = 100; + Console.Write("\x1b]9;4;0;0\x1b\\"); + break; + case TaskbarProgressState.Indeterminate: + currentProgress = 100; + Console.Write("\x1b]9;4;3;0\x1b\\"); + break; + case TaskbarProgressState.Normal: + // Normal state is set when progress is set. + WriteProgressSequence(currentProgress); + break; + case TaskbarProgressState.Error: + Console.Write($"\x1b]9;4;2;{currentProgress}\x1b\\"); + break; + case TaskbarProgressState.Warning: + Console.Write($"\x1b]9;4;4;{currentProgress}\x1b\\"); + break; + } + } + + /// + /// Sets the progress value out of 100. + /// + internal void SetValue(uint progressValue) + { + currentProgress = progressValue; + // Write progress sequence to console for Windows Terminal (https://github.com/microsoft/terminal/discussions/14268). + GetConsoleMode(consoleHandle, out ConsoleModes previousConsoleMode); + SetConsoleMode(consoleHandle, ConsoleModes.ENABLE_VIRTUAL_TERMINAL_PROCESSING | ConsoleModes.ENABLE_PROCESSED_OUTPUT); + WriteProgressSequence(progressValue); + SetConsoleMode(consoleHandle, previousConsoleMode); + } + + private static void WriteProgressSequence(uint progressValue) + => Console.Write($"\x1b]9;4;1;{progressValue}\x1b\\"); + } + } +} diff --git a/src/BenchmarkDotNet/Helpers/UnitHelper.cs b/src/BenchmarkDotNet/Helpers/UnitHelper.cs new file mode 100644 index 0000000000..ae76faf101 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/UnitHelper.cs @@ -0,0 +1,11 @@ +using Perfolizer.Horology; +using Perfolizer.Metrology; + +namespace BenchmarkDotNet.Helpers; + +public static class UnitHelper +{ + public static readonly UnitPresentation DefaultPresentation = new (true, 0, gap: true); + + public static string ToDefaultString(this TimeInterval timeInterval, string? format = null) => timeInterval.ToString(format, null, DefaultPresentation); +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/UserInteractionHelper.cs b/src/BenchmarkDotNet/Helpers/UserInteractionHelper.cs index 7951d168dd..73e83d6ada 100644 --- a/src/BenchmarkDotNet/Helpers/UserInteractionHelper.cs +++ b/src/BenchmarkDotNet/Helpers/UserInteractionHelper.cs @@ -1,3 +1,4 @@ +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Helpers @@ -19,7 +20,7 @@ internal static class UserInteractionHelper /// public static string EscapeCommandExample(string input) { - return !RuntimeInformation.IsWindows() && input.IndexOf('*') >= 0 ? $"'{input}'" : input; + return !OsDetector.IsWindows() && input.IndexOf('*') >= 0 ? $"'{input}'" : input; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs b/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs index 8fe04e5764..b7b1d15394 100644 --- a/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs +++ b/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Phd; using BenchmarkDotNet.Portability; using JetBrains.Annotations; @@ -14,7 +16,10 @@ public sealed class EnvironmentMode : JobMode public static readonly Characteristic RuntimeCharacteristic = CreateCharacteristic(nameof(Runtime)); public static readonly Characteristic AffinityCharacteristic = CreateCharacteristic(nameof(Affinity)); public static readonly Characteristic GcCharacteristic = CreateCharacteristic(nameof(Gc)); - public static readonly Characteristic> EnvironmentVariablesCharacteristic = CreateCharacteristic>(nameof(EnvironmentVariables)); + + public static readonly Characteristic> EnvironmentVariablesCharacteristic = + CreateCharacteristic>(nameof(EnvironmentVariables)); + public static readonly Characteristic PowerPlanModeCharacteristic = CreateCharacteristic(nameof(PowerPlanMode)); public static readonly Characteristic LargeAddressAwareCharacteristic = CreateCharacteristic(nameof(LargeAddressAware)); @@ -60,7 +65,7 @@ public Jit Jit /// /// Runtime /// - public Runtime Runtime + public Runtime? Runtime { get { return RuntimeCharacteristic[this]; } set { RuntimeCharacteristic[this] = value; } @@ -107,7 +112,7 @@ public bool LargeAddressAware get => LargeAddressAwareCharacteristic[this]; set { - if (value && !RuntimeInformation.IsWindows()) + if (value && !OsDetector.IsWindows()) { throw new NotSupportedException("LargeAddressAware is a Windows-specific concept."); } @@ -133,5 +138,12 @@ public void SetEnvironmentVariable(EnvironmentVariable variable) } internal Runtime GetRuntime() => HasValue(RuntimeCharacteristic) ? Runtime : RuntimeInformation.GetCurrentRuntime(); + + public BdnEnvironment ToPhd() => new () + { + Jit = HasValue(JitCharacteristic) ? Jit : null, + Runtime = HasValue(RuntimeCharacteristic) ? Runtime?.RuntimeMoniker : null, + Affinity = HasValue(AffinityCharacteristic) ? (int)Affinity : null + }; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 8cbf7445d7..2c778df589 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -339,7 +339,7 @@ public static Job WithEnvironmentVariable(this Job job, string key, string value /// (optional)Indicate the URI of the NuGet package source to use during the restore operation. /// (optional)Allows prerelease packages to be installed. /// - public static Job WithNuGet(this Job job, string packageName, string packageVersion = null, Uri source = null, bool prerelease = false) => + public static Job WithNuGet(this Job job, string packageName, string? packageVersion = null, Uri? source = null, bool prerelease = false) => job.WithCore(j => j.Infrastructure.NuGetReferences = new NuGetReferenceList(j.Infrastructure.NuGetReferences ?? Array.Empty()) { @@ -436,5 +436,10 @@ private static Job WithCore(this Job job, Action updateCallback) updateCallback(newJob); return newJob; } + + internal static bool HasDynamicBuildCharacteristic(this Job job) => + job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic) + || job.HasValue(InfrastructureMode.BuildConfigurationCharacteristic) + || job.HasValue(InfrastructureMode.ArgumentsCharacteristic); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/NugetReference.cs b/src/BenchmarkDotNet/Jobs/NugetReference.cs index 3e5819c4ae..0db3a028de 100644 --- a/src/BenchmarkDotNet/Jobs/NugetReference.cs +++ b/src/BenchmarkDotNet/Jobs/NugetReference.cs @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Jobs { public class NuGetReference : IEquatable { - public NuGetReference(string packageName, string packageVersion, Uri source = null, bool prerelease = false) + public NuGetReference(string packageName, string packageVersion, Uri? source = null, bool prerelease = false) { if (string.IsNullOrWhiteSpace(packageName)) throw new ArgumentException("message", nameof(packageName)); diff --git a/src/BenchmarkDotNet/Jobs/RunMode.cs b/src/BenchmarkDotNet/Jobs/RunMode.cs index a76b5dc832..070c1753b0 100644 --- a/src/BenchmarkDotNet/Jobs/RunMode.cs +++ b/src/BenchmarkDotNet/Jobs/RunMode.cs @@ -2,6 +2,8 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Phd; using Perfolizer.Horology; namespace BenchmarkDotNet.Jobs @@ -9,7 +11,8 @@ namespace BenchmarkDotNet.Jobs [SuppressMessage("ReSharper", "UnusedMember.Global")] public sealed class RunMode : JobMode { - public static readonly Characteristic RunStrategyCharacteristic = Characteristic.Create(nameof(RunStrategy), RunStrategy.Throughput); + public static readonly Characteristic RunStrategyCharacteristic = + Characteristic.Create(nameof(RunStrategy), RunStrategy.Throughput); public static readonly Characteristic LaunchCountCharacteristic = CreateCharacteristic(nameof(LaunchCount)); public static readonly Characteristic InvocationCountCharacteristic = CreateCharacteristic(nameof(InvocationCount)); @@ -61,13 +64,9 @@ public sealed class RunMode : JobMode }.Freeze(); - public RunMode() : this(null) - { - } + public RunMode() : this(null) { } - private RunMode(string id) : base(id) - { - } + private RunMode(string id) : base(id) { } /// /// Available values: Throughput and ColdStart. @@ -191,5 +190,21 @@ public bool MemoryRandomization get => MemoryRandomizationCharacteristic[this]; set => MemoryRandomizationCharacteristic[this] = value; } + + public BdnExecution ToPhd() => new () + { + LaunchCount = HasValue(LaunchCountCharacteristic) ? LaunchCount : null, + WarmupCount = HasValue(WarmupCountCharacteristic) ? WarmupCount : null, + IterationCount = HasValue(IterationCountCharacteristic) ? IterationCount : null, + IterationTimeMs = HasValue(IterationTimeCharacteristic) ? IterationTime.ToMilliseconds().RoundToLong() : null, + InvocationCount = HasValue(InvocationCountCharacteristic) ? InvocationCount : null, + UnrollFactor = HasValue(UnrollFactorCharacteristic) ? UnrollFactor : null, + MinIterationCount = HasValue(MinIterationCountCharacteristic) ? MinIterationCount : null, + MaxIterationCount = HasValue(MaxIterationCountCharacteristic) ? MaxIterationCount : null, + MinWarmupIterationCount = HasValue(MinWarmupIterationCountCharacteristic) ? MinWarmupIterationCount : null, + MaxWarmupIterationCount = HasValue(MaxWarmupIterationCountCharacteristic) ? MaxWarmupIterationCount : null, + MemoryRandomization = HasValue(MemoryRandomizationCharacteristic) ? MemoryRandomization : null, + RunStrategy = HasValue(RunStrategyCharacteristic) ? RunStrategy : null, + }; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs b/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs index a503da8e76..d4a059153d 100644 --- a/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs +++ b/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs @@ -19,7 +19,7 @@ internal class AsyncProcessOutputReader : IDisposable private long status; - internal AsyncProcessOutputReader(Process process, bool logOutput = false, ILogger logger = null, bool readStandardError = true) + internal AsyncProcessOutputReader(Process process, bool logOutput = false, ILogger? logger = null, bool readStandardError = true) { if (!process.StartInfo.RedirectStandardOutput) throw new NotSupportedException("set RedirectStandardOutput to true first"); diff --git a/src/BenchmarkDotNet/Loggers/Broker.cs b/src/BenchmarkDotNet/Loggers/Broker.cs index 9055523340..dcec4c0d56 100644 --- a/src/BenchmarkDotNet/Loggers/Broker.cs +++ b/src/BenchmarkDotNet/Loggers/Broker.cs @@ -15,9 +15,7 @@ internal class Broker { private readonly ILogger logger; private readonly Process process; - private readonly IDiagnoser diagnoser; private readonly AnonymousPipeServerStream inputFromBenchmark, acknowledgments; - private readonly DiagnoserActionParameters diagnoserActionParameters; private readonly ManualResetEvent finished; public Broker(ILogger logger, Process process, IDiagnoser diagnoser, @@ -25,10 +23,10 @@ public Broker(ILogger logger, Process process, IDiagnoser diagnoser, { this.logger = logger; this.process = process; - this.diagnoser = diagnoser; + this.Diagnoser = diagnoser; this.inputFromBenchmark = inputFromBenchmark; this.acknowledgments = acknowledgments; - diagnoserActionParameters = new DiagnoserActionParameters(process, benchmarkCase, benchmarkId); + DiagnoserActionParameters = new DiagnoserActionParameters(process, benchmarkCase, benchmarkId); finished = new ManualResetEvent(false); Results = new List(); @@ -38,6 +36,10 @@ public Broker(ILogger logger, Process process, IDiagnoser diagnoser, process.Exited += OnProcessExited; } + internal IDiagnoser Diagnoser { get; } + + internal DiagnoserActionParameters DiagnoserActionParameters { get; } + internal List Results { get; } internal List PrefixedOutput { get; } @@ -78,9 +80,8 @@ private void ProcessDataBlocking() using StreamWriter writer = new (acknowledgments, AnonymousPipesHost.UTF8NoBOM, bufferSize: 1); // Flush the data to the Stream after each write, otherwise the client will wait for input endlessly! writer.AutoFlush = true; - string line = null; - while ((line = reader.ReadLine()) is not null) + while (reader.ReadLine() is { } line) { // TODO: implement Silent mode here logger.WriteLine(LogKind.Default, line); @@ -91,7 +92,7 @@ private void ProcessDataBlocking() } else if (Engine.Signals.TryGetSignal(line, out var signal)) { - diagnoser?.Handle(signal, diagnoserActionParameters); + Diagnoser?.Handle(signal, DiagnoserActionParameters); writer.WriteLine(Engine.Signals.Acknowledgment); diff --git a/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs b/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs index a54bf88edc..d201f5da75 100644 --- a/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs +++ b/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Portability; using JetBrains.Annotations; @@ -15,13 +16,13 @@ public sealed class ConsoleLogger : ILogger public static readonly ILogger Ascii = new ConsoleLogger(false); public static readonly ILogger Unicode = new ConsoleLogger(true); private static readonly bool ConsoleSupportsColors - = !(RuntimeInformation.IsAndroid() || RuntimeInformation.IsIOS() || RuntimeInformation.IsWasm || RuntimeInformation.IsTvOS()); + = !(OsDetector.IsAndroid() || OsDetector.IsIOS() || RuntimeInformation.IsWasm || OsDetector.IsTvOS()); private readonly bool unicodeSupport; private readonly Dictionary colorScheme; [PublicAPI] - public ConsoleLogger(bool unicodeSupport = false, Dictionary colorScheme = null) + public ConsoleLogger(bool unicodeSupport = false, Dictionary? colorScheme = null) { this.unicodeSupport = unicodeSupport; this.colorScheme = colorScheme ?? CreateColorfulScheme(); @@ -79,6 +80,7 @@ private static Dictionary CreateColorfulScheme() => { LogKind.Statistic, ConsoleColor.Cyan }, { LogKind.Info, ConsoleColor.DarkYellow }, { LogKind.Error, ConsoleColor.Red }, + { LogKind.Warning, ConsoleColor.Yellow }, { LogKind.Hint, ConsoleColor.DarkCyan } }; diff --git a/src/BenchmarkDotNet/Loggers/LinqPadLogger.cs b/src/BenchmarkDotNet/Loggers/LinqPadLogger.cs index 930d19d4ca..cbb023848f 100644 --- a/src/BenchmarkDotNet/Loggers/LinqPadLogger.cs +++ b/src/BenchmarkDotNet/Loggers/LinqPadLogger.cs @@ -16,7 +16,7 @@ public sealed class LinqPadLogger : ILogger public static readonly Lazy lazyInstance = new Lazy(() => { // Detect if being run from LINQPad; see https://github.com/dotnet/BenchmarkDotNet/issues/445#issuecomment-300723741 - MethodInfo withStyle = null; + MethodInfo? withStyle = null; if (AppDomain.CurrentDomain.FriendlyName.StartsWith("LINQPad", StringComparison.OrdinalIgnoreCase)) { try @@ -82,6 +82,7 @@ private static IReadOnlyDictionary CreateDarkScheme() => { LogKind.Statistic, "#00FFFF" }, { LogKind.Info, "#808000" }, { LogKind.Error, "#FF0000" }, + { LogKind.Warning, "#FFFF00" }, { LogKind.Hint, "#008080" } }; @@ -95,6 +96,7 @@ private static IReadOnlyDictionary CreateLightScheme() => { LogKind.Statistic, "#008080" }, { LogKind.Info, "#808000" }, { LogKind.Error, "#FF0000" }, + { LogKind.Warning, "#FFFF00" }, { LogKind.Hint, "#008080" } }; } diff --git a/src/BenchmarkDotNet/Loggers/LogKind.cs b/src/BenchmarkDotNet/Loggers/LogKind.cs index 8f3dff72a8..8fa4660aa8 100644 --- a/src/BenchmarkDotNet/Loggers/LogKind.cs +++ b/src/BenchmarkDotNet/Loggers/LogKind.cs @@ -2,6 +2,6 @@ { public enum LogKind { - Default, Help, Header, Result, Statistic, Info, Error, Hint + Default, Help, Header, Result, Statistic, Info, Error, Hint, Warning } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Loggers/LoggerExtensions.cs b/src/BenchmarkDotNet/Loggers/LoggerExtensions.cs index 866620c3fb..1d75341bb8 100644 --- a/src/BenchmarkDotNet/Loggers/LoggerExtensions.cs +++ b/src/BenchmarkDotNet/Loggers/LoggerExtensions.cs @@ -18,6 +18,8 @@ public static class LoggerExtensions public static void WriteLineError(this ILogger logger, string text) => logger.WriteLine(LogKind.Error, text); + public static void WriteLineWarning(this ILogger logger, string text) => logger.WriteLine(LogKind.Warning, text); + public static void WriteLineHint(this ILogger logger, string text) => logger.WriteLine(LogKind.Hint, text); public static void Write(this ILogger logger, string text) => logger.Write(LogKind.Default, text); @@ -36,6 +38,8 @@ public static class LoggerExtensions public static void WriteError(this ILogger logger, string text) => logger.Write(LogKind.Error, text); + public static void WriteWarning(this ILogger logger, string text) => logger.Write(LogKind.Warning, text); + [PublicAPI] public static void WriteHint(this ILogger logger, string text) => logger.Write(LogKind.Hint, text); } diff --git a/src/BenchmarkDotNet/Mathematics/LegacyConfidenceInterval.cs b/src/BenchmarkDotNet/Mathematics/LegacyConfidenceInterval.cs new file mode 100644 index 0000000000..f1fa247f21 --- /dev/null +++ b/src/BenchmarkDotNet/Mathematics/LegacyConfidenceInterval.cs @@ -0,0 +1,167 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using BenchmarkDotNet.Helpers; +using JetBrains.Annotations; +using Perfolizer.Mathematics.Common; + +namespace BenchmarkDotNet.Mathematics; + +public enum LegacyConfidenceLevel +{ + /// + /// 50.0% confidence interval + /// + [PublicAPI] L50, + + /// + /// 70.0% confidence interval + /// + [PublicAPI] L70, + + /// + /// 75.0% confidence interval + /// + [PublicAPI] L75, + + /// + /// 80.0% confidence interval + /// + [PublicAPI] L80, + + /// + /// 85.0% confidence interval + /// + [PublicAPI] L85, + + /// + /// 90.0% confidence interval + /// + [PublicAPI] L90, + + /// + /// 92.0% confidence interval + /// + [PublicAPI] L92, + + /// + /// 95.0% confidence interval + /// + [PublicAPI] L95, + + /// + /// 96.0% confidence interval + /// + [PublicAPI] L96, + + /// + /// 97.0% confidence interval + /// + [PublicAPI] L97, + + /// + /// 98.0% confidence interval + /// + [PublicAPI] L98, + + /// + /// 99.0% confidence interval + /// + [PublicAPI] L99, + + /// + /// 99.9% confidence interval + /// + [PublicAPI] L999 +} + +public static class ConfidenceLevelExtensions +{ + private static readonly Dictionary ConfidenceLevelDetails = CreateConfidenceLevelMapping(); + + /// + /// Calculates Z value (z-star) for confidence interval + /// + /// ConfidenceLevel for a confidence interval + /// Sample size (n >= 3) + public static double GetZValue(this LegacyConfidenceLevel level, int n) + { + return new ConfidenceIntervalEstimator(n, 0, 1).ZLevel(new ConfidenceLevel(level.ToPercent())); + } + + public static string ToPercentStr(this LegacyConfidenceLevel level) + { + string s = level.ToString().Substring(1); + if (s.Length > 2) + s = s.Substring(0, 2) + "." + s.Substring(2); + return s + "%"; + } + + [PublicAPI] public static double ToPercent(this LegacyConfidenceLevel level) + { + (int value, int digits) = ConfidenceLevelDetails[level]; + + return value / Math.Pow(10, digits); + } + + private static Dictionary CreateConfidenceLevelMapping() + => Enum.GetValues(typeof(LegacyConfidenceLevel)) + .Cast() + .ToDictionary( + confidenceLevel => confidenceLevel, + confidenceLevel => + { + string textRepresentation = confidenceLevel.ToString().Substring(1); + + return (int.Parse(textRepresentation), textRepresentation.Length); + }); +} + +public struct LegacyConfidenceInterval +{ + [PublicAPI] public int N { get; } + [PublicAPI] public double Mean { get; } + [PublicAPI] public double StandardError { get; } + + [PublicAPI] public LegacyConfidenceLevel Level { get; } + [PublicAPI] public double Margin { get; } + + [PublicAPI] public double Lower { get; } + [PublicAPI] public double Upper { get; } + + public LegacyConfidenceInterval(double mean, double standardError, int n, LegacyConfidenceLevel level = LegacyConfidenceLevel.L999) + { + N = n; + Mean = mean; + StandardError = standardError; + Level = level; + Margin = n <= 2 ? double.NaN : standardError * level.GetZValue(n); + Lower = mean - Margin; + Upper = mean + Margin; + } + + public bool Contains(double value) => Lower - 1e-9 < value && value < Upper + 1e-9; + + private string GetLevelHint(bool showLevel = true) => showLevel ? $" (CI {Level.ToPercentStr()})" : ""; + + public override string ToString() => ToString(DefaultCultureInfo.Instance); + + public string ToString(CultureInfo cultureInfo, string format = "0.##", bool showLevel = true) + { + return ToString(x => x.ToString(format, cultureInfo), showLevel); + } + + public string ToString(Func formatter, bool showLevel = true) + { + var builder = new StringBuilder(); + builder.Append('['); + builder.Append(formatter(Lower)); + builder.Append("; "); + builder.Append(formatter(Upper)); + builder.Append("]"); + builder.Append(GetLevelHint(showLevel)); + return builder.ToString(); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Mathematics/MathHelper.cs b/src/BenchmarkDotNet/Mathematics/MathHelper.cs index 01f5491763..d6250f1618 100644 --- a/src/BenchmarkDotNet/Mathematics/MathHelper.cs +++ b/src/BenchmarkDotNet/Mathematics/MathHelper.cs @@ -1,10 +1,13 @@ -using static System.Math; +using Perfolizer.Mathematics.Common; +using Perfolizer.Metrology; +using static System.Math; namespace BenchmarkDotNet.Mathematics { - public static class MathHelper + internal static class MathHelper { - + public static readonly Threshold DefaultThreshold = PercentValue.Of(2).ToThreshold(); + public static readonly SignificanceLevel DefaultSignificanceLevel = SignificanceLevel.P1E5; public static int Clamp(int value, int min, int max) => Min(Max(value, min), max); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Mathematics/MeasurementsStatistics.cs b/src/BenchmarkDotNet/Mathematics/MeasurementsStatistics.cs index 0ec336b101..41d76fe849 100644 --- a/src/BenchmarkDotNet/Mathematics/MeasurementsStatistics.cs +++ b/src/BenchmarkDotNet/Mathematics/MeasurementsStatistics.cs @@ -26,13 +26,13 @@ internal readonly ref struct MeasurementsStatistics /// /// 99.9% confidence interval in nanoseconds. /// - public ConfidenceInterval ConfidenceInterval { get; } + public ConfidenceInterval LegacyConfidenceInterval { get; } - private MeasurementsStatistics(double standardError, double mean, ConfidenceInterval confidenceInterval) + private MeasurementsStatistics(double standardError, double mean, ConfidenceInterval legacyConfidenceInterval) { StandardError = standardError; Mean = mean; - ConfidenceInterval = confidenceInterval; + LegacyConfidenceInterval = legacyConfidenceInterval; } public static MeasurementsStatistics Calculate(List measurements, OutlierMode outlierMode) @@ -47,7 +47,8 @@ public static MeasurementsStatistics Calculate(List measurements, O double variance = Variance(measurements, n, mean); double standardDeviation = Math.Sqrt(variance); double standardError = standardDeviation / Math.Sqrt(n); - var confidenceInterval = new ConfidenceInterval(mean, standardError, n); + var confidenceIntervalEstimator = new ConfidenceIntervalEstimator(n, mean, standardError); + var confidenceInterval = confidenceIntervalEstimator.ConfidenceInterval(ConfidenceLevel.L999); if (outlierMode == OutlierMode.DontRemove) // most simple scenario is done without allocations! but this is not the default case return new MeasurementsStatistics(standardError, mean, confidenceInterval); @@ -74,7 +75,8 @@ public static MeasurementsStatistics Calculate(List measurements, O variance = VarianceWithoutOutliers(outlierMode, measurements, n, mean, lowerFence, upperFence); standardDeviation = Math.Sqrt(variance); standardError = standardDeviation / Math.Sqrt(n); - confidenceInterval = new ConfidenceInterval(mean, standardError, n); + confidenceIntervalEstimator = new ConfidenceIntervalEstimator(n, mean, standardError); + confidenceInterval = confidenceIntervalEstimator.ConfidenceInterval(ConfidenceLevel.L999); return new MeasurementsStatistics(standardError, mean, confidenceInterval); } @@ -113,7 +115,8 @@ private static double Variance(List measurements, int n, double mea return variance; } - private static double VarianceWithoutOutliers(OutlierMode outlierMode, List measurements, int n, double mean, double lowerFence, double upperFence) + private static double VarianceWithoutOutliers(OutlierMode outlierMode, List measurements, int n, double mean, double lowerFence, + double upperFence) { if (n == 1) return 0; @@ -151,4 +154,4 @@ private static bool IsOutlier(OutlierMode outlierMode, double value, double lowe } } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Mathematics/PercentileValues.cs b/src/BenchmarkDotNet/Mathematics/PercentileValues.cs index 34abe47430..7241206517 100644 --- a/src/BenchmarkDotNet/Mathematics/PercentileValues.cs +++ b/src/BenchmarkDotNet/Mathematics/PercentileValues.cs @@ -4,42 +4,14 @@ using System.Text; using BenchmarkDotNet.Helpers; using JetBrains.Annotations; +using Perfolizer; using Perfolizer.Mathematics.QuantileEstimators; namespace BenchmarkDotNet.Mathematics { public class PercentileValues { - /// - /// Calculates the Nth percentile from the set of values - /// - /// - /// The implementation is expected to be consistent with the one from Excel. - /// It's a quite common to export bench output into .csv for further analysis - /// And it's a good idea to have same results from all tools being used. - /// - /// Sequence of the values to be calculated - /// Value in range 0..100 - /// Percentile from the set of values - // Based on: http://stackoverflow.com/a/8137526 - private static double Percentile(IReadOnlyList sortedValues, int percentile) - { - if (sortedValues == null) - throw new ArgumentNullException(nameof(sortedValues)); - if (percentile < 0 || percentile > 100) - { - throw new ArgumentOutOfRangeException( - nameof(percentile), percentile, - "The percentile arg should be in range of 0 - 100."); - } - - if (sortedValues.Count == 0) - return 0; - - return SimpleQuantileEstimator.Instance.GetQuantileFromSorted(sortedValues, percentile / 100.0); - } - - [PublicAPI] public double Percentile(int percentile) => Percentile(SortedValues, percentile); + [PublicAPI] public double Percentile(int percentile) => SimpleQuantileEstimator.Instance.Quantile(new Sample(SortedValues), percentile / 100.0); private IReadOnlyList SortedValues { get; } diff --git a/src/BenchmarkDotNet/Mathematics/RankHelper.cs b/src/BenchmarkDotNet/Mathematics/RankHelper.cs index 8f7587a9ea..526ad46a1d 100644 --- a/src/BenchmarkDotNet/Mathematics/RankHelper.cs +++ b/src/BenchmarkDotNet/Mathematics/RankHelper.cs @@ -1,7 +1,8 @@ using System; using System.Linq; +using Perfolizer.Mathematics.Common; using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; +using Perfolizer.Mathematics.SignificanceTesting.MannWhitney; namespace BenchmarkDotNet.Mathematics { @@ -9,10 +10,7 @@ internal static class RankHelper { public static int[] GetRanks(params Statistics[] stats) { - var values = stats. - Select((s, index) => new { Stats = s, Index = index }). - OrderBy(pair => pair.Stats.Mean). - ToArray(); + var values = stats.Select((s, index) => new { Stats = s, Index = index }).OrderBy(pair => pair.Stats.Mean).ToArray(); int n = values.Length; var ranks = new int[n]; @@ -33,14 +31,9 @@ public static int[] GetRanks(params Statistics[] stats) private static bool AreSame(Statistics x, Statistics y) { - if (x.N >= 3 && y.N >= 3) - { - var tost = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, x.SortedValues.ToArray(), y.SortedValues.ToArray(), RelativeThreshold.Default); - if (tost.Conclusion != EquivalenceTestConclusion.Unknown) - return tost.Conclusion == EquivalenceTestConclusion.Same; - } - - return Math.Abs(x.Mean - y.Mean) < Math.Abs(x.Mean + y.Mean) / 2 * 0.01; + var test = new SimpleEquivalenceTest(MannWhitneyTest.Instance); + var comparisonResult = test.Perform(x.Sample, y.Sample, MathHelper.DefaultThreshold, MathHelper.DefaultSignificanceLevel); + return comparisonResult == ComparisonResult.Indistinguishable; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Mathematics/Statistics.cs b/src/BenchmarkDotNet/Mathematics/Statistics.cs index 35743d009c..be6f47db6c 100644 --- a/src/BenchmarkDotNet/Mathematics/Statistics.cs +++ b/src/BenchmarkDotNet/Mathematics/Statistics.cs @@ -3,6 +3,8 @@ using System.Linq; using BenchmarkDotNet.Extensions; using JetBrains.Annotations; +using Perfolizer; +using Perfolizer.Horology; using Perfolizer.Mathematics.Common; using Perfolizer.Mathematics.OutlierDetection; using Perfolizer.Mathematics.QuantileEstimators; @@ -11,8 +13,8 @@ namespace BenchmarkDotNet.Mathematics { public class Statistics { + internal Sample Sample { get; } public IReadOnlyList OriginalValues { get; } - internal IReadOnlyList SortedValues { get; } public int N { get; } public double Min { get; } public double LowerFence { get; } @@ -31,26 +33,28 @@ public class Statistics public double StandardDeviation { get; } public double Skewness { get; } public double Kurtosis { get; } - public ConfidenceInterval ConfidenceInterval { get; } + private ConfidenceIntervalEstimator ConfidenceIntervalEstimator { get; } + internal ConfidenceInterval PerfolizerConfidenceInterval { get; } + public LegacyConfidenceInterval ConfidenceInterval { get; } public PercentileValues Percentiles { get; } - private OutlierDetector outlierDetector; + private readonly TukeyOutlierDetector outlierDetector; public Statistics(params double[] values) : this(values.ToList()) { } public Statistics(IEnumerable values) : - this(values.Select(value => (double) value)) { } + this(values.Select(value => (double)value)) { } - public Statistics(IEnumerable values) + public Statistics(IEnumerable values) : this(new Sample(values.ToArray(), TimeUnit.Nanosecond)) { } + + public Statistics(Sample sample) { - OriginalValues = values.Where(d => !double.IsNaN(d)).ToArray(); - SortedValues = OriginalValues.OrderBy(value => value).ToArray(); - N = SortedValues.Count; - if (N == 0) - throw new InvalidOperationException("Sequence of values contains no elements, Statistics can't be calculated"); + Sample = sample; + OriginalValues = sample.Values; + N = Sample.Size; - var quartiles = Quartiles.FromSorted(SortedValues); + var quartiles = Quartiles.Create(Sample); Min = quartiles.Min; Q1 = quartiles.Q1; Median = quartiles.Median; @@ -58,33 +62,35 @@ public Statistics(IEnumerable values) Max = quartiles.Max; InterquartileRange = quartiles.InterquartileRange; - var moments = Moments.Create(SortedValues); + var moments = Moments.Create(Sample); Mean = moments.Mean; StandardDeviation = moments.StandardDeviation; Variance = moments.Variance; Skewness = moments.Skewness; Kurtosis = moments.Kurtosis; - var tukey = TukeyOutlierDetector.FromQuartiles(quartiles); + var tukey = TukeyOutlierDetector.Create(Sample); LowerFence = tukey.LowerFence; UpperFence = tukey.UpperFence; - AllOutliers = SortedValues.Where(tukey.IsOutlier).ToArray(); - LowerOutliers = SortedValues.Where(tukey.IsLowerOutlier).ToArray(); - UpperOutliers = SortedValues.Where(tukey.IsUpperOutlier).ToArray(); + AllOutliers = Sample.SortedValues.Where(tukey.IsOutlier).ToArray(); + LowerOutliers = Sample.SortedValues.Where(tukey.IsLowerOutlier).ToArray(); + UpperOutliers = Sample.SortedValues.Where(tukey.IsUpperOutlier).ToArray(); outlierDetector = tukey; StandardError = StandardDeviation / Math.Sqrt(N); - ConfidenceInterval = new ConfidenceInterval(Mean, StandardError, N); - Percentiles = new PercentileValues(SortedValues); + ConfidenceIntervalEstimator = new ConfidenceIntervalEstimator(Sample.Size, Mean, StandardError); + PerfolizerConfidenceInterval = ConfidenceIntervalEstimator.ConfidenceInterval(ConfidenceLevel.L999); + ConfidenceInterval = new LegacyConfidenceInterval(PerfolizerConfidenceInterval.Estimation, StandardError, N, LegacyConfidenceLevel.L999); + Percentiles = new PercentileValues(Sample.SortedValues); } - [PublicAPI] public ConfidenceInterval GetConfidenceInterval(ConfidenceLevel level, int n) => new ConfidenceInterval(Mean, StandardError, n, level); + [PublicAPI] public ConfidenceInterval GetConfidenceInterval(ConfidenceLevel level) => ConfidenceIntervalEstimator.ConfidenceInterval(level); [PublicAPI] public bool IsLowerOutlier(double value) => outlierDetector.IsLowerOutlier(value); [PublicAPI] public bool IsUpperOutlier(double value) => outlierDetector.IsUpperOutlier(value); [PublicAPI] public bool IsOutlier(double value) => outlierDetector.IsOutlier(value); - [PublicAPI] public double[] WithoutOutliers() => SortedValues.Where(value => !IsOutlier(value)).ToArray(); + [PublicAPI] public double[] WithoutOutliers() => outlierDetector.WithoutAllOutliers(Sample.Values).ToArray(); - [PublicAPI] public double CalcCentralMoment(int k) => SortedValues.Average(x => (x - Mean).Pow(k)); + [PublicAPI] public double CalcCentralMoment(int k) => Sample.SortedValues.Average(x => (x - Mean).Pow(k)); public bool IsActualOutlier(double value, OutlierMode outlierMode) { @@ -121,18 +127,7 @@ public double[] GetActualOutliers(OutlierMode outlierMode) } } - public override string ToString() - { - switch (N) - { - case 1: - return $"{SortedValues[0]} (N = 1)"; - case 2: - return $"{SortedValues[0]},{SortedValues[1]} (N = 2)"; - default: - return $"{Mean} +- {ConfidenceInterval.Margin} (N = {N})"; - } - } + public override string ToString() => Sample.ToString(); /// /// Returns true, if this statistics can be inverted (see ). @@ -142,7 +137,7 @@ public override string ToString() /// /// Statistics for [1/X]. If Min is less then or equal to 0, returns null. /// - public Statistics Invert() => CanBeInverted() ? new Statistics(SortedValues.Select(x => 1 / x)) : null; + public Statistics Invert() => CanBeInverted() ? new Statistics(Sample.SortedValues.Select(x => 1 / x)) : null; /// /// Mean for [X*Y]. @@ -168,13 +163,15 @@ public static Statistics Divide(Statistics x, Statistics y) throw new ArgumentOutOfRangeException(nameof(x), "Argument doesn't contain any values"); if (y.N < 1) throw new ArgumentOutOfRangeException(nameof(y), "Argument doesn't contain any values"); - int n = Math.Min(x.N, y.N); - var z = new double[n]; - for (int i = 0; i < n; i++) + + double[]? z = new double[x.N * y.N]; + int k = 0; + for (int i = 0; i < x.N; i++) + for (int j = 0; j < y.N; j++) { - if (Math.Abs(y.OriginalValues[i]) < 1e-9) - throw new DivideByZeroException($"y[{i}] is {y.OriginalValues[i]}"); - z[i] = x.OriginalValues[i] / y.OriginalValues[i]; + if (Math.Abs(y.Sample.Values[j]) < 1e-9) + throw new DivideByZeroException($"y[{j}] is {y.Sample.Values[j]}"); + z[k++] = x.Sample.Values[i] / y.Sample.Values[j]; } return new Statistics(z); diff --git a/src/BenchmarkDotNet/Order/DefaultOrderer.cs b/src/BenchmarkDotNet/Order/DefaultOrderer.cs index 0bca3cf3ec..656de7ce98 100644 --- a/src/BenchmarkDotNet/Order/DefaultOrderer.cs +++ b/src/BenchmarkDotNet/Order/DefaultOrderer.cs @@ -37,7 +37,7 @@ public DefaultOrderer( [PublicAPI] public virtual IEnumerable GetExecutionOrder( ImmutableArray benchmarkCases, - IEnumerable order = null) + IEnumerable? order = null) { var benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, targetComparer, order); var list = benchmarkCases.ToList(); @@ -137,7 +137,7 @@ public string GetLogicalGroupKey(ImmutableArray allBenchmarksCase public virtual IEnumerable> GetLogicalGroupOrder( IEnumerable> logicalGroups, - IEnumerable order = null) + IEnumerable? order = null) { var benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, targetComparer, order); var logicalGroupComparer = new LogicalGroupComparer(benchmarkComparer); diff --git a/src/BenchmarkDotNet/Order/IOrderer.cs b/src/BenchmarkDotNet/Order/IOrderer.cs index dece54e519..bfc6ba5433 100644 --- a/src/BenchmarkDotNet/Order/IOrderer.cs +++ b/src/BenchmarkDotNet/Order/IOrderer.cs @@ -11,7 +11,7 @@ namespace BenchmarkDotNet.Order public interface IOrderer { [PublicAPI] - IEnumerable GetExecutionOrder(ImmutableArray benchmarksCase, IEnumerable order = null); + IEnumerable GetExecutionOrder(ImmutableArray benchmarksCase, IEnumerable? order = null); [PublicAPI] IEnumerable GetSummaryOrder(ImmutableArray benchmarksCases, Summary summary); @@ -24,7 +24,7 @@ public interface IOrderer [PublicAPI] IEnumerable> GetLogicalGroupOrder(IEnumerable> logicalGroups, - IEnumerable order = null); + IEnumerable? order = null); [PublicAPI] bool SeparateLogicalGroups { get; } diff --git a/src/BenchmarkDotNet/Phd/BdnBenchmark.cs b/src/BenchmarkDotNet/Phd/BdnBenchmark.cs new file mode 100644 index 0000000000..b52a84d30e --- /dev/null +++ b/src/BenchmarkDotNet/Phd/BdnBenchmark.cs @@ -0,0 +1,38 @@ +using System.Text; +using BenchmarkDotNet.Extensions; +using JetBrains.Annotations; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Phd; + +[PublicAPI] +public class BdnBenchmark : PhdBenchmark +{ + public string Namespace { get; set; } = ""; + public string Type { get; set; } = ""; + public string Method { get; set; } = ""; + public string Parameters { get; set; } = ""; + public string? HardwareIntrinsics { get; set; } = ""; + + // TODO: Improve + public override string? GetDisplay() + { + if (Display != null) return Display; + + var builder = new StringBuilder(); + builder.Append($"{Namespace}"); + if (Type.IsNotBlank()) + { + if (builder.Length > 0) + builder.Append('.'); + builder.Append(Type); + } + if (Method.IsNotBlank()) + { + if (builder.Length > 0) + builder.Append('.'); + builder.Append(Method); + } + return builder.ToString(); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Phd/BdnEnvironment.cs b/src/BenchmarkDotNet/Phd/BdnEnvironment.cs new file mode 100644 index 0000000000..c250c9faeb --- /dev/null +++ b/src/BenchmarkDotNet/Phd/BdnEnvironment.cs @@ -0,0 +1,12 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Phd; + +public class BdnEnvironment : PhdEnvironment +{ + public RuntimeMoniker? Runtime { get; set; } + public Jit? Jit { get; set; } + public int? Affinity { get; set; } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Phd/BdnExecution.cs b/src/BenchmarkDotNet/Phd/BdnExecution.cs new file mode 100644 index 0000000000..13cad143aa --- /dev/null +++ b/src/BenchmarkDotNet/Phd/BdnExecution.cs @@ -0,0 +1,84 @@ +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Engines; +using Perfolizer.Horology; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Phd; + +public class BdnExecution : PhdExecution +{ + /// + /// Available values: Throughput and ColdStart. + /// Throughput: default strategy which allows to get good precision level. + /// ColdStart: should be used only for measuring cold start of the application or testing purpose. + /// Monitoring: no overhead evaluating, with several target iterations. Perfect for macrobenchmarks without a steady state with high variance. + /// + public RunStrategy? RunStrategy { get; set; } + + /// + /// How many times we should launch process with target benchmark. + /// + public int? LaunchCount { get; set; } + + /// + /// How many warmup iterations should be performed. + /// + public int? WarmupCount { get; set; } + + /// + /// How many target iterations should be performed + /// If specified, will be ignored. + /// If specified, will be ignored. + /// + public int? IterationCount { get; set; } + + /// + /// Desired time of execution of an iteration. Used by Pilot stage to estimate the number of invocations per iteration. + /// The default value is 500 milliseconds. + /// + public long? IterationTimeMs { get; set; } + + /// + /// Invocation count in a single iteration. + /// If specified, will be ignored. + /// If specified, it must be a multiple of . + /// + public long? InvocationCount { get; set; } + + /// + /// How many times the benchmark method will be invoked per one iteration of a generated loop. + /// + public int? UnrollFactor { get; set; } + + /// + /// Minimum count of target iterations that should be performed + /// The default value is 15 + /// If you set this value to below 15, then is not going to work + /// + public int? MinIterationCount { get; set; } + + /// + /// Maximum count of target iterations that should be performed + /// The default value is 100 + /// If you set this value to below 15, then is not going to work + /// + public int? MaxIterationCount { get; set; } + + /// + /// Minimum count of warmup iterations that should be performed + /// The default value is 6 + /// + public int? MinWarmupIterationCount { get; set; } + + /// + /// Maximum count of warmup iterations that should be performed + /// The default value is 50 + /// + public int? MaxWarmupIterationCount { get; set; } + + /// + /// specifies whether Engine should allocate some random-sized memory between iterations + /// it makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration + /// + public bool? MemoryRandomization { get; set; } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Phd/BdnHost.cs b/src/BenchmarkDotNet/Phd/BdnHost.cs new file mode 100644 index 0000000000..3c66d3f5d2 --- /dev/null +++ b/src/BenchmarkDotNet/Phd/BdnHost.cs @@ -0,0 +1,17 @@ +using JetBrains.Annotations; +using Perfolizer.Phd; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Phd; + +[PublicAPI] +public class BdnHost : PhdHost +{ + public string RuntimeVersion { get; set; } = ""; + public bool HasAttachedDebugger { get; set; } + public bool HasRyuJit { get; set; } + public string Configuration { get; set; } = ""; + public string DotNetSdkVersion { get; set; } = ""; + public double ChronometerFrequency { get; set; } + public string HardwareTimerKind { get; set; } = ""; +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Phd/BdnInfo.cs b/src/BenchmarkDotNet/Phd/BdnInfo.cs new file mode 100644 index 0000000000..85c5956392 --- /dev/null +++ b/src/BenchmarkDotNet/Phd/BdnInfo.cs @@ -0,0 +1,11 @@ +using JetBrains.Annotations; +using Perfolizer.Phd; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Phd; + +[PublicAPI] +public class BdnInfo : PhdInfo +{ + +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Phd/BdnJob.cs b/src/BenchmarkDotNet/Phd/BdnJob.cs new file mode 100644 index 0000000000..0400faf3c0 --- /dev/null +++ b/src/BenchmarkDotNet/Phd/BdnJob.cs @@ -0,0 +1,8 @@ +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Phd; + +public class BdnJob : PhdJob +{ + +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Phd/BdnLifecycle.cs b/src/BenchmarkDotNet/Phd/BdnLifecycle.cs new file mode 100644 index 0000000000..dafc65a880 --- /dev/null +++ b/src/BenchmarkDotNet/Phd/BdnLifecycle.cs @@ -0,0 +1,57 @@ +using System; +using BenchmarkDotNet.Engines; +using JetBrains.Annotations; +using Perfolizer.Phd; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Phd; + +[PublicAPI] +public class BdnLifecycle : PhdLifecycle, IEquatable, IComparable +{ + public int LaunchIndex { get; set; } + public IterationStage IterationStage { get; set; } = IterationStage.Unknown; + public IterationMode IterationMode { get; set; } = IterationMode.Unknown; + + public bool Equals(BdnLifecycle? other) + { + if (ReferenceEquals(null, other)) + return false; + if (ReferenceEquals(this, other)) + return true; + return LaunchIndex == other.LaunchIndex && + IterationMode == other.IterationMode && + IterationStage == other.IterationStage; + } + + public override bool Equals(object? obj) + { + if (ReferenceEquals(null, obj)) + return false; + if (ReferenceEquals(this, obj)) + return true; + if (obj.GetType() != GetType()) + return false; + return Equals((BdnLifecycle)obj); + } + + public override int GetHashCode() => HashCode.Combine(LaunchIndex, IterationMode, IterationStage); + + public static bool operator ==(BdnLifecycle? left, BdnLifecycle? right) => Equals(left, right); + public static bool operator !=(BdnLifecycle? left, BdnLifecycle? right) => !Equals(left, right); + + public int CompareTo(BdnLifecycle? other) + { + if (ReferenceEquals(this, other)) + return 0; + if (ReferenceEquals(null, other)) + return 1; + int launchIndexComparison = LaunchIndex.CompareTo(other.LaunchIndex); + if (launchIndexComparison != 0) + return launchIndexComparison; + int iterationStageComparison = IterationStage.CompareTo(other.IterationStage); + if (iterationStageComparison != 0) + return iterationStageComparison; + return IterationMode.CompareTo(other.IterationMode); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Phd/BdnSchema.cs b/src/BenchmarkDotNet/Phd/BdnSchema.cs new file mode 100644 index 0000000000..dd63db3e3c --- /dev/null +++ b/src/BenchmarkDotNet/Phd/BdnSchema.cs @@ -0,0 +1,19 @@ +using Perfolizer.Phd.Base; + +namespace BenchmarkDotNet.Phd; + +public class BdnSchema : PhdSchema +{ + public static readonly BdnSchema Instance = new (); + + private BdnSchema() : base("bdn") + { + Add(); + Add(); + Add(); + Add(); + Add(); + Add(); + Add(); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/CpuInfo.cs b/src/BenchmarkDotNet/Portability/Cpu/CpuInfo.cs deleted file mode 100644 index 8c3227a25d..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/CpuInfo.cs +++ /dev/null @@ -1,37 +0,0 @@ -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - public class CpuInfo - { - public string ProcessorName { get; } - public int? PhysicalProcessorCount { get; } - public int? PhysicalCoreCount { get; } - public int? LogicalCoreCount { get; } - public Frequency? NominalFrequency { get; } - public Frequency? MinFrequency { get; } - public Frequency? MaxFrequency { get; } - - internal CpuInfo(string processorName, Frequency? nominalFrequency) - : this(processorName, null, null, null, nominalFrequency, null, null) - { - } - - public CpuInfo(string processorName, - int? physicalProcessorCount, - int? physicalCoreCount, - int? logicalCoreCount, - Frequency? nominalFrequency, - Frequency? minFrequency, - Frequency? maxFrequency) - { - ProcessorName = processorName; - PhysicalProcessorCount = physicalProcessorCount; - PhysicalCoreCount = physicalCoreCount; - LogicalCoreCount = logicalCoreCount; - NominalFrequency = nominalFrequency; - MinFrequency = minFrequency; - MaxFrequency = maxFrequency; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/CpuInfoFormatter.cs b/src/BenchmarkDotNet/Portability/Cpu/CpuInfoFormatter.cs deleted file mode 100644 index e016ad1a4b..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/CpuInfoFormatter.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Environments; - -namespace BenchmarkDotNet.Portability.Cpu -{ - public static class CpuInfoFormatter - { - public static string Format(CpuInfo cpuInfo) - { - if (cpuInfo == null) - { - return "Unknown processor"; - } - - var parts = new List - { - ProcessorBrandStringHelper.Prettify(cpuInfo, includeMaxFrequency: true) - }; - - if (cpuInfo.PhysicalProcessorCount > 0) - parts.Add($", {cpuInfo.PhysicalProcessorCount} CPU"); - - if (cpuInfo.LogicalCoreCount == 1) - parts.Add(", 1 logical core"); - - if (cpuInfo.LogicalCoreCount > 1) - parts.Add($", {cpuInfo.LogicalCoreCount} logical cores"); - - if (cpuInfo.LogicalCoreCount > 0 && cpuInfo.PhysicalCoreCount > 0) - parts.Add(" and "); - else if (cpuInfo.PhysicalCoreCount > 0) - parts.Add(", "); - - if (cpuInfo.PhysicalCoreCount == 1) - parts.Add("1 physical core"); - if (cpuInfo.PhysicalCoreCount > 1) - parts.Add($"{cpuInfo.PhysicalCoreCount} physical cores"); - - string result = string.Join("", parts); - // The line with ProcessorBrandString is one of the longest lines in the summary. - // When people past in on GitHub, it can be a reason of an ugly horizontal scrollbar. - // To avoid this, we are trying to minimize this line and use the minimum possible number of characters. - // Here we are removing the repetitive "cores" word. - if (result.Contains("logical cores") && result.Contains("physical cores")) - result = result.Replace("logical cores", "logical"); - - return result; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/MosCpuInfoProvider.cs b/src/BenchmarkDotNet/Portability/Cpu/MosCpuInfoProvider.cs deleted file mode 100644 index 2edc7cabcb..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/MosCpuInfoProvider.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Management; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class MosCpuInfoProvider - { -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatform("windows")] -#endif - internal static readonly Lazy MosCpuInfo = new Lazy(Load); - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatform("windows")] -#endif - private static CpuInfo Load() - { - var processorModelNames = new HashSet(); - uint physicalCoreCount = 0; - uint logicalCoreCount = 0; - int processorsCount = 0; - uint nominalClockSpeed = 0; - uint maxClockSpeed = 0; - uint minClockSpeed = 0; - - - using (var mosProcessor = new ManagementObjectSearcher("SELECT * FROM Win32_Processor")) - { - foreach (var moProcessor in mosProcessor.Get().Cast()) - { - string name = moProcessor[WmicCpuInfoKeyNames.Name]?.ToString(); - if (!string.IsNullOrEmpty(name)) - { - processorModelNames.Add(name); - processorsCount++; - physicalCoreCount += (uint) moProcessor[WmicCpuInfoKeyNames.NumberOfCores]; - logicalCoreCount += (uint) moProcessor[WmicCpuInfoKeyNames.NumberOfLogicalProcessors]; - maxClockSpeed = (uint) moProcessor[WmicCpuInfoKeyNames.MaxClockSpeed]; - } - } - } - - return new CpuInfo( - processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null, - processorsCount > 0 ? processorsCount : (int?) null, - physicalCoreCount > 0 ? (int?) physicalCoreCount : null, - logicalCoreCount > 0 ? (int?) logicalCoreCount : null, - nominalClockSpeed > 0 && logicalCoreCount > 0 ? Frequency.FromMHz(nominalClockSpeed) : (Frequency?) null, - minClockSpeed > 0 && logicalCoreCount > 0 ? Frequency.FromMHz(minClockSpeed) : (Frequency?) null, - maxClockSpeed > 0 && logicalCoreCount > 0 ? Frequency.FromMHz(maxClockSpeed) : (Frequency?) null); - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoKeyNames.cs b/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoKeyNames.cs deleted file mode 100644 index 60284c5363..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoKeyNames.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class ProcCpuInfoKeyNames - { - internal const string PhysicalId = "physical id"; - internal const string CpuCores = "cpu cores"; - internal const string ModelName = "model name"; - internal const string MaxFrequency = "max freq"; - internal const string MinFrequency = "min freq"; - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoParser.cs b/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoParser.cs deleted file mode 100644 index e3d0a14c46..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoParser.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; -using BenchmarkDotNet.Helpers; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class ProcCpuInfoParser - { - internal static CpuInfo ParseOutput(string? content) - { - var logicalCores = SectionsHelper.ParseSections(content, ':'); - var processorModelNames = new HashSet(); - var processorsToPhysicalCoreCount = new Dictionary(); - - int logicalCoreCount = 0; - var nominalFrequency = Frequency.Zero; - var minFrequency = Frequency.Zero; - var maxFrequency = Frequency.Zero; - - foreach (var logicalCore in logicalCores) - { - if (logicalCore.TryGetValue(ProcCpuInfoKeyNames.PhysicalId, out string physicalId) && - logicalCore.TryGetValue(ProcCpuInfoKeyNames.CpuCores, out string cpuCoresValue) && - int.TryParse(cpuCoresValue, out int cpuCoreCount) && - cpuCoreCount > 0) - processorsToPhysicalCoreCount[physicalId] = cpuCoreCount; - - if (logicalCore.TryGetValue(ProcCpuInfoKeyNames.ModelName, out string modelName)) - { - processorModelNames.Add(modelName); - nominalFrequency = ParseFrequencyFromBrandString(modelName); - logicalCoreCount++; - } - - if (logicalCore.TryGetValue(ProcCpuInfoKeyNames.MinFrequency, out string minCpuFreqValue) - && Frequency.TryParseMHz(minCpuFreqValue, out var minCpuFreq)) - { - minFrequency = minCpuFreq; - } - - if (logicalCore.TryGetValue(ProcCpuInfoKeyNames.MaxFrequency, out string maxCpuFreqValue) - && Frequency.TryParseMHz(maxCpuFreqValue, out var maxCpuFreq)) - { - maxFrequency = maxCpuFreq; - } - } - - return new CpuInfo( - processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null, - processorsToPhysicalCoreCount.Count > 0 ? processorsToPhysicalCoreCount.Count : (int?) null, - processorsToPhysicalCoreCount.Count > 0 ? processorsToPhysicalCoreCount.Values.Sum() : (int?) null, - logicalCoreCount > 0 ? logicalCoreCount : (int?) null, - nominalFrequency > 0 ? nominalFrequency : (Frequency?) null, - minFrequency > 0 ? minFrequency : (Frequency?) null, - maxFrequency > 0 ? maxFrequency : (Frequency?) null); - } - - internal static Frequency ParseFrequencyFromBrandString(string brandString) - { - const string pattern = "(\\d.\\d+)GHz"; - var matches = Regex.Matches(brandString, pattern, RegexOptions.IgnoreCase); - if (matches.Count > 0 && matches[0].Groups.Count > 1) - { - string match = Regex.Matches(brandString, pattern, RegexOptions.IgnoreCase)[0].Groups[1].ToString(); - return Frequency.TryParseGHz(match, out var result) ? result : Frequency.Zero; - } - - return 0d; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoProvider.cs b/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoProvider.cs deleted file mode 100644 index 7cadad1ff2..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoProvider.cs +++ /dev/null @@ -1,75 +0,0 @@ -using System; -using System.Linq; -using System.Text; -using BenchmarkDotNet.Helpers; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - /// - /// CPU information from output of the `cat /proc/info` command. - /// Linux only. - /// - internal static class ProcCpuInfoProvider - { - internal static readonly Lazy ProcCpuInfo = new (Load); - - private static CpuInfo? Load() - { - if (RuntimeInformation.IsLinux()) - { - string content = ProcessHelper.RunAndReadOutput("cat", "/proc/cpuinfo") ?? ""; - string output = GetCpuSpeed() ?? ""; - content += output; - return ProcCpuInfoParser.ParseOutput(content); - } - return null; - } - - private static string? GetCpuSpeed() - { - try - { - string[]? output = ProcessHelper.RunAndReadOutput("/bin/bash", "-c \"lscpu | grep MHz\"")? - .Split('\n') - .SelectMany(x => x.Split(':')) - .ToArray(); - - return ParseCpuFrequencies(output); - } - catch (Exception) - { - return null; - } - } - - private static string? ParseCpuFrequencies(string[]? input) - { - // Example of output we trying to parse: - // - // CPU MHz: 949.154 - // CPU max MHz: 3200,0000 - // CPU min MHz: 800,0000 - - if (input == null) - return null; - - var output = new StringBuilder(); - for (int i = 0; i + 1 < input.Length; i += 2) - { - string name = input[i].Trim(); - string value = input[i + 1].Trim(); - - if (name.EqualsWithIgnoreCase("CPU min MHz")) - if (Frequency.TryParseMHz(value.Replace(',', '.'), out var minFrequency)) - output.Append($"\n{ProcCpuInfoKeyNames.MinFrequency}\t:{minFrequency.ToMHz()}"); - - if (name.EqualsWithIgnoreCase("CPU max MHz")) - if (Frequency.TryParseMHz(value.Replace(',', '.'), out var maxFrequency)) - output.Append($"\n{ProcCpuInfoKeyNames.MaxFrequency}\t:{maxFrequency.ToMHz()}"); - } - - return output.ToString(); - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoParser.cs b/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoParser.cs deleted file mode 100644 index 7e61bbd3ec..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoParser.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Extensions; - -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class SysctlCpuInfoParser - { - [SuppressMessage("ReSharper", "StringLiteralTypo")] - internal static CpuInfo ParseOutput(string? content) - { - var sysctl = SectionsHelper.ParseSection(content, ':'); - string processorName = sysctl.GetValueOrDefault("machdep.cpu.brand_string"); - var physicalProcessorCount = GetPositiveIntValue(sysctl, "hw.packages"); - var physicalCoreCount = GetPositiveIntValue(sysctl, "hw.physicalcpu"); - var logicalCoreCount = GetPositiveIntValue(sysctl, "hw.logicalcpu"); - var nominalFrequency = GetPositiveLongValue(sysctl, "hw.cpufrequency"); - var minFrequency = GetPositiveLongValue(sysctl, "hw.cpufrequency_min"); - var maxFrequency = GetPositiveLongValue(sysctl, "hw.cpufrequency_max"); - return new CpuInfo(processorName, physicalProcessorCount, physicalCoreCount, logicalCoreCount, nominalFrequency, minFrequency, maxFrequency); - } - - private static int? GetPositiveIntValue(Dictionary sysctl, string keyName) - { - if (sysctl.TryGetValue(keyName, out string value) && - int.TryParse(value, out int result) && - result > 0) - return result; - return null; - } - - private static long? GetPositiveLongValue(Dictionary sysctl, string keyName) - { - if (sysctl.TryGetValue(keyName, out string value) && - long.TryParse(value, out long result) && - result > 0) - return result; - return null; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoProvider.cs b/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoProvider.cs deleted file mode 100644 index 32a207c1c8..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoProvider.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using BenchmarkDotNet.Helpers; - -namespace BenchmarkDotNet.Portability.Cpu -{ - /// - /// CPU information from output of the `sysctl -a` command. - /// MacOSX only. - /// - internal static class SysctlCpuInfoProvider - { - internal static readonly Lazy SysctlCpuInfo = new Lazy(Load); - - private static CpuInfo? Load() - { - if (RuntimeInformation.IsMacOS()) - { - string content = ProcessHelper.RunAndReadOutput("sysctl", "-a"); - return SysctlCpuInfoParser.ParseOutput(content); - } - return null; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoKeyNames.cs b/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoKeyNames.cs deleted file mode 100644 index 9dd2209304..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoKeyNames.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class WmicCpuInfoKeyNames - { - internal const string NumberOfLogicalProcessors = "NumberOfLogicalProcessors"; - internal const string NumberOfCores = "NumberOfCores"; - internal const string Name = "Name"; - internal const string MaxClockSpeed = "MaxClockSpeed"; - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoParser.cs b/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoParser.cs deleted file mode 100644 index 441b1a8b10..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoParser.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Helpers; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class WmicCpuInfoParser - { - internal static CpuInfo ParseOutput(string? content) - { - var processors = SectionsHelper.ParseSections(content, '='); - - var processorModelNames = new HashSet(); - int physicalCoreCount = 0; - int logicalCoreCount = 0; - int processorsCount = 0; - - var currentClockSpeed = Frequency.Zero; - var maxClockSpeed = Frequency.Zero; - var minClockSpeed = Frequency.Zero; - - foreach (var processor in processors) - { - if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfCores, out string numberOfCoresValue) && - int.TryParse(numberOfCoresValue, out int numberOfCores) && - numberOfCores > 0) - physicalCoreCount += numberOfCores; - - if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfLogicalProcessors, out string numberOfLogicalValue) && - int.TryParse(numberOfLogicalValue, out int numberOfLogical) && - numberOfLogical > 0) - logicalCoreCount += numberOfLogical; - - if (processor.TryGetValue(WmicCpuInfoKeyNames.Name, out string name)) - { - processorModelNames.Add(name); - processorsCount++; - } - - if (processor.TryGetValue(WmicCpuInfoKeyNames.MaxClockSpeed, out string frequencyValue) - && int.TryParse(frequencyValue, out int frequency) - && frequency > 0) - { - maxClockSpeed += frequency; - } - } - - return new CpuInfo( - processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null, - processorsCount > 0 ? processorsCount : (int?) null, - physicalCoreCount > 0 ? physicalCoreCount : (int?) null, - logicalCoreCount > 0 ? logicalCoreCount : (int?) null, - currentClockSpeed > 0 && processorsCount > 0 ? Frequency.FromMHz(currentClockSpeed / processorsCount) : (Frequency?) null, - minClockSpeed > 0 && processorsCount > 0 ? Frequency.FromMHz(minClockSpeed / processorsCount) : (Frequency?) null, - maxClockSpeed > 0 && processorsCount > 0 ? Frequency.FromMHz(maxClockSpeed / processorsCount) : (Frequency?) null); - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoProvider.cs b/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoProvider.cs deleted file mode 100644 index d259200701..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoProvider.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; -using System.IO; -using BenchmarkDotNet.Helpers; - -namespace BenchmarkDotNet.Portability.Cpu -{ - /// - /// CPU information from output of the `wmic cpu get Name, NumberOfCores, NumberOfLogicalProcessors /Format:List` command. - /// Windows only. - /// - internal static class WmicCpuInfoProvider - { - internal static readonly Lazy WmicCpuInfo = new (Load); - - private const string DefaultWmicPath = @"C:\Windows\System32\wbem\WMIC.exe"; - - private static CpuInfo? Load() - { - if (RuntimeInformation.IsWindows()) - { - const string argList = $"{WmicCpuInfoKeyNames.Name}, {WmicCpuInfoKeyNames.NumberOfCores}, " + - $"{WmicCpuInfoKeyNames.NumberOfLogicalProcessors}, {WmicCpuInfoKeyNames.MaxClockSpeed}"; - string wmicPath = File.Exists(DefaultWmicPath) ? DefaultWmicPath : "wmic"; - string content = ProcessHelper.RunAndReadOutput(wmicPath, $"cpu get {argList} /Format:List"); - return WmicCpuInfoParser.ParseOutput(content); - } - return null; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Libc.cs b/src/BenchmarkDotNet/Portability/Libc.cs index c9a3cf1322..036d7b584d 100644 --- a/src/BenchmarkDotNet/Portability/Libc.cs +++ b/src/BenchmarkDotNet/Portability/Libc.cs @@ -2,7 +2,9 @@ namespace BenchmarkDotNet.Portability { +#pragma warning disable CS8981 // The type name 'libc' only contains lower-cased ascii characters. Such names may become reserved for the language. internal static class libc +#pragma warning restore CS8981 { [DllImport(nameof(libc))] internal static extern int getppid(); diff --git a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs index d9ae69010e..3c0fe30149 100644 --- a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs +++ b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs @@ -8,12 +8,16 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Text.RegularExpressions; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Detectors.Cpu; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Portability.Cpu; using JetBrains.Annotations; using Microsoft.Win32; +using Perfolizer.Helpers; +using Perfolizer.Phd; +using Perfolizer.Phd.Dto; using static System.Runtime.InteropServices.RuntimeInformation; using RuntimeEnvironment = Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment; @@ -28,7 +32,8 @@ internal static class RuntimeInformation /// /// returns true for both the old (implementation of .NET Framework) and new Mono (.NET 6+ flavour) /// - public static bool IsMono { get; } = Type.GetType("Mono.RuntimeStructs") != null; // it allocates a lot of memory, we need to check it once in order to keep Engine non-allocating! + public static bool IsMono { get; } = + Type.GetType("Mono.RuntimeStructs") != null; // it allocates a lot of memory, we need to check it once in order to keep Engine non-allocating! public static bool IsOldMono { get; } = Type.GetType("Mono.Runtime") != null; @@ -46,12 +51,12 @@ internal static class RuntimeInformation public static bool IsNetCore => ((Environment.Version.Major >= 5) || FrameworkDescription.StartsWith(".NET Core", StringComparison.OrdinalIgnoreCase)) - && !string.IsNullOrEmpty(typeof(object).Assembly.Location); + && !string.IsNullOrEmpty(typeof(object).Assembly.Location); public static bool IsNativeAOT => Environment.Version.Major >= 5 - && string.IsNullOrEmpty(typeof(object).Assembly.Location) // it's merged to a single .exe and .Location returns null - && !IsWasm; // Wasm also returns "" for assembly locations + && string.IsNullOrEmpty(typeof(object).Assembly.Location) // it's merged to a single .exe and .Location returns null + && !IsWasm; // Wasm also returns "" for assembly locations #if NET6_0_OR_GREATER [System.Runtime.Versioning.SupportedOSPlatformGuard("browser")] @@ -87,155 +92,9 @@ private static bool IsAotMethod() public static bool IsRunningInContainer => string.Equals(Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"), "true"); - internal static string ExecutableExtension => IsWindows() ? ".exe" : string.Empty; - - internal static string ScriptFileExtension => IsWindows() ? ".bat" : ".sh"; internal static string GetArchitecture() => GetCurrentPlatform().ToString(); -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("windows")] -#endif - internal static bool IsWindows() => -#if NET6_0_OR_GREATER - OperatingSystem.IsWindows(); // prefer linker-friendly OperatingSystem APIs -#else - IsOSPlatform(OSPlatform.Windows); -#endif - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("linux")] -#endif - internal static bool IsLinux() => -#if NET6_0_OR_GREATER - OperatingSystem.IsLinux(); -#else - IsOSPlatform(OSPlatform.Linux); -#endif - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("macos")] -#endif - internal static bool IsMacOS() => -#if NET6_0_OR_GREATER - OperatingSystem.IsMacOS(); -#else - IsOSPlatform(OSPlatform.OSX); -#endif - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("android")] -#endif - internal static bool IsAndroid() => -#if NET6_0_OR_GREATER - OperatingSystem.IsAndroid(); -#else - Type.GetType("Java.Lang.Object, Mono.Android") != null; -#endif - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("ios")] -#endif - internal static bool IsIOS() => -#if NET6_0_OR_GREATER - OperatingSystem.IsIOS(); -#else - Type.GetType("Foundation.NSObject, Xamarin.iOS") != null; -#endif - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("tvos")] -#endif - internal static bool IsTvOS() => -#if NET6_0_OR_GREATER - OperatingSystem.IsTvOS(); -#else - IsOSPlatform(OSPlatform.Create("TVOS")); -#endif - - public static string GetOsVersion() - { - if (IsMacOS()) - { - string systemVersion = ExternalToolsHelper.MacSystemProfilerData.Value.GetValueOrDefault("System Version") ?? ""; - string kernelVersion = ExternalToolsHelper.MacSystemProfilerData.Value.GetValueOrDefault("Kernel Version") ?? ""; - if (!string.IsNullOrEmpty(systemVersion) && !string.IsNullOrEmpty(kernelVersion)) - return OsBrandStringHelper.PrettifyMacOSX(systemVersion, kernelVersion); - } - - if (IsLinux()) - { - string? version = GetLinuxOsVersion(); - if (version != null && version.Trim() != "") - return version; - } - - string operatingSystem = RuntimeEnvironment.OperatingSystem; - string operatingSystemVersion = RuntimeEnvironment.OperatingSystemVersion; - - return OsBrandStringHelper.Prettify( - operatingSystem, - operatingSystemVersion, - GetWindowsUbr()); - } - - private static string? GetLinuxOsVersion() - { - if (!IsLinux()) - return null; - try - { - return LinuxOsReleaseHelper.GetNameByOsRelease(File.ReadAllLines("/etc/os-release")); - } - catch (Exception) - { - return null; - } - } - - // TODO: Introduce a common util API for registry calls, use it also in BenchmarkDotNet.Toolchains.CsProj.GetCurrentVersionBasedOnWindowsRegistry - /// - /// On Windows, this method returns UBR (Update Build Revision) based on Registry. - /// Returns null if the value is not available - /// - /// - private static int? GetWindowsUbr() - { - if (IsWindows()) - { - try - { - using (var baseKey = RegistryKey.OpenBaseKey(RegistryHive.LocalMachine, RegistryView.Registry32)) - using (var ndpKey = baseKey.OpenSubKey(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion")) - { - if (ndpKey == null) - return null; - - return Convert.ToInt32(ndpKey.GetValue("UBR")); - } - } - catch (Exception) - { - return null; - } - } - return null; - } - - internal static CpuInfo GetCpuInfo() - { - if (IsWindows() && IsFullFramework && !IsMono) - return MosCpuInfoProvider.MosCpuInfo.Value; - if (IsWindows()) - return WmicCpuInfoProvider.WmicCpuInfo.Value; - if (IsLinux()) - return ProcCpuInfoProvider.ProcCpuInfo.Value; - if (IsMacOS()) - return SysctlCpuInfoProvider.SysctlCpuInfo.Value; - - return null; - } - internal static string GetRuntimeVersion() { if (IsWasm) @@ -299,7 +158,7 @@ internal static string GetRuntimeVersion() private static string GetNetCoreVersion() { - if (IsAndroid()) + if (OsDetector.IsAndroid()) { return $".NET {Environment.Version}"; } @@ -348,12 +207,13 @@ internal static Runtime GetCurrentRuntime() public static Platform GetCurrentPlatform() { // these are not part of .NET Standard 2.0, so we use hardcoded values taken from - // https://github.com/dotnet/runtime/blob/1a37caf773a3b857ccff49a31be3333d4fdc491f/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Architecture.cs#L9 + // https://github.com/dotnet/runtime/blob/080fcae7eaa8367abf7900e08ff2e52e3efea5bf/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Architecture.cs#L9 const Architecture Wasm = (Architecture)4; const Architecture S390x = (Architecture)5; const Architecture LoongArch64 = (Architecture)6; const Architecture Armv6 = (Architecture)7; const Architecture Ppc64le = (Architecture)8; + const Architecture RiscV64 = (Architecture)9; switch (ProcessArchitecture) { @@ -375,6 +235,8 @@ public static Platform GetCurrentPlatform() return Platform.Armv6; case Ppc64le: return Platform.Ppc64le; + case RiscV64: + return Platform.RiscV64; default: throw new ArgumentOutOfRangeException(); } @@ -407,7 +269,7 @@ internal static string GetJitInfo() if (IsNetCore || HasRyuJit()) // CoreCLR supports only RyuJIT return "RyuJIT"; if (IsFullFramework) - return "LegacyJIT"; + return "LegacyJIT"; return Unknown; } @@ -460,7 +322,7 @@ public JitModule(string name, string version) internal static ICollection GetAntivirusProducts() { var products = new List(); - if (IsWindows()) + if (OsDetector.IsWindows()) { try { @@ -468,7 +330,7 @@ internal static ICollection GetAntivirusProducts() using (var data = wmi.Get()) foreach (var o in data) { - var av = (ManagementObject) o; + var av = (ManagementObject)o; if (av != null) { string name = av["displayName"].ToString(); @@ -486,11 +348,11 @@ internal static ICollection GetAntivirusProducts() return products; } - internal static VirtualMachineHypervisor GetVirtualMachineHypervisor() + internal static VirtualMachineHypervisor? GetVirtualMachineHypervisor() { - VirtualMachineHypervisor[] hypervisors = { HyperV.Default, VirtualBox.Default, VMware.Default }; + VirtualMachineHypervisor[] hypervisors = [HyperV.Default, VirtualBox.Default, VMware.Default]; - if (IsWindows()) + if (OsDetector.IsWindows()) { try { @@ -516,4 +378,4 @@ internal static VirtualMachineHypervisor GetVirtualMachineHypervisor() return null; } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs index 5327197aa5..ee7077a55e 100644 --- a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs +++ b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs @@ -14,13 +14,17 @@ [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] +[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotMemory,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] +[assembly: InternalsVisibleTo("BenchmarkDotNet.TestAdapter,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] #else [assembly: InternalsVisibleTo("BenchmarkDotNet.Tests")] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests")] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows")] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace")] +[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotMemory")] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning")] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks")] +[assembly: InternalsVisibleTo("BenchmarkDotNet.TestAdapter")] #endif \ No newline at end of file diff --git a/src/BenchmarkDotNet/Properties/BenchmarkDotNetInfo.cs b/src/BenchmarkDotNet/Properties/BenchmarkDotNetInfo.cs index 0b9d0e6f44..f371a95990 100644 --- a/src/BenchmarkDotNet/Properties/BenchmarkDotNetInfo.cs +++ b/src/BenchmarkDotNet/Properties/BenchmarkDotNetInfo.cs @@ -1,21 +1,31 @@ using System; using System.Reflection; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Phd; +using Perfolizer.Phd.Dto; namespace BenchmarkDotNet.Properties { public class BenchmarkDotNetInfo { + public const string BenchmarkDotNetCaption = "BenchmarkDotNet"; + private static readonly Lazy LazyInstance = new (() => { var assembly = typeof(BenchmarkDotNetInfo).GetTypeInfo().Assembly; var assemblyVersion = assembly.GetName().Version; string informationVersion = assembly.GetCustomAttribute().InformationalVersion ?? ""; - return new BenchmarkDotNetInfo(assemblyVersion, informationVersion); + return new BenchmarkDotNetInfo(assemblyVersion, RemoveVersionMetadata(informationVersion)); }); public static BenchmarkDotNetInfo Instance { get; } = LazyInstance.Value; + public PhdEngine GetBdnEngine() => new () + { + Name = BenchmarkDotNetCaption, + Version = BrandVersion + }; + public Version AssemblyVersion { get; } public string FullVersion { get; } @@ -49,6 +59,12 @@ public BenchmarkDotNetInfo(Version assemblyVersion, string fullVersion) BrandTitle = "BenchmarkDotNet v" + BrandVersion; } + internal static string RemoveVersionMetadata(string version) + { + int index = version.IndexOf('+'); + return index >= 0 ? version.Substring(0, index) : version; + } + internal const string PublicKey = "00240000048000009400000006020000002400005253413100040000010001002970bbdfca4d12" + "9fc74b4845b239973f1b183684f0d7db5e1de7e085917e3656cf94884803cb800d85d5aae5838f" + diff --git a/src/BenchmarkDotNet/Reports/BenchmarkReport.cs b/src/BenchmarkDotNet/Reports/BenchmarkReport.cs index b63c5221be..8a896a84bf 100644 --- a/src/BenchmarkDotNet/Reports/BenchmarkReport.cs +++ b/src/BenchmarkDotNet/Reports/BenchmarkReport.cs @@ -3,10 +3,18 @@ using System.Collections.Immutable; using System.Linq; using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Mathematics; +using BenchmarkDotNet.Phd; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.Results; using JetBrains.Annotations; +using Perfolizer.Horology; +using Perfolizer.Phd; +using Perfolizer.Phd.Base; +using Perfolizer.Phd.Dto; namespace BenchmarkDotNet.Reports { @@ -22,11 +30,12 @@ public sealed class BenchmarkReport public IReadOnlyList ExecuteResults { get; } - public Statistics? ResultStatistics => resultStatistics ?? (resultStatistics = GetResultRuns().Any() - ? new Statistics(GetResultRuns().Select(r => r.GetAverageTime().Nanoseconds)) - : null); + public Statistics? ResultStatistics => resultStatistics ??= + GetResultRuns().Any() + ? new Statistics(GetResultRuns().Select(r => r.GetAverageTime().Nanoseconds)) + : null; - private Statistics resultStatistics; + private Statistics? resultStatistics = null; public BenchmarkReport( bool success, @@ -44,11 +53,60 @@ public BenchmarkReport( AllMeasurements = ExecuteResults.SelectMany((results, index) => results.Measurements).ToArray(); GcStats = ExecuteResults.Count > 0 ? ExecuteResults[ExecuteResults.Count - 1].GcStats : default; Metrics = metrics?.ToDictionary(metric => metric.Descriptor.Id) - ?? (IReadOnlyDictionary)ImmutableDictionary.Empty; + ?? (IReadOnlyDictionary)ImmutableDictionary.Empty; } public override string ToString() => $"{BenchmarkCase.DisplayInfo}, {AllMeasurements.Count} runs"; public IReadOnlyList GetResultRuns() => AllMeasurements.Where(r => r.Is(IterationMode.Workload, IterationStage.Result)).ToList(); + + public PhdEntry ToPhd() + { + var entry = new PhdEntry + { + Benchmark = new BdnBenchmark + { + Display = BenchmarkCase.DisplayInfo, + Namespace = BenchmarkCase.Descriptor.Type.Namespace ?? "", + Type = FullNameProvider.GetTypeName(BenchmarkCase.Descriptor.Type), + Method = BenchmarkCase.Descriptor.WorkloadMethod.Name, + Parameters = BenchmarkCase.Parameters.PrintInfo, + HardwareIntrinsics = this.GetHardwareIntrinsicsInfo() + }, + Job = new BdnJob + { + Environment = BenchmarkCase.Job.Environment.ToPhd(), + Execution = BenchmarkCase.Job.Run.ToPhd() + } + }; + + var lifecycles = AllMeasurements.GroupBy(m => new BdnLifecycle + { + LaunchIndex = m.LaunchIndex, + IterationMode = m.IterationMode, + IterationStage = m.IterationStage + }).OrderBy(x => x.Key).ToList(); + foreach (var lifecycleGroup in lifecycles) + { + var measurementsEntry = new PhdEntry + { + Lifecycle = lifecycleGroup.Key + }; + + foreach (var measurement in lifecycleGroup.ToList()) + { + measurementsEntry.Add(new PhdEntry + { + IterationIndex = measurement.IterationIndex, + InvocationCount = measurement.Operations, + Value = measurement.Nanoseconds / measurement.Operations, + Unit = TimeUnit.Nanosecond + }); + } + entry.Add(measurementsEntry); + } + + return entry; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Reports/Measurement.cs b/src/BenchmarkDotNet/Reports/Measurement.cs index ddcac16a30..9a1245ddca 100644 --- a/src/BenchmarkDotNet/Reports/Measurement.cs +++ b/src/BenchmarkDotNet/Reports/Measurement.cs @@ -1,5 +1,7 @@ using System; +#if DEBUG using System.Diagnostics; +#endif using System.Globalization; using System.Linq; using System.Text; @@ -95,7 +97,7 @@ public override string ToString() builder.Append(NsSymbol); builder.Append(", "); - builder.Append(GetAverageTime().ToString(MainCultureInfo).ToAscii()); + builder.Append(GetAverageTime().ToDefaultString("0.0000").ToAscii()); builder.Append("/op"); return builder.ToString(); diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index 7a9382f62d..2d08f2a0d9 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -8,10 +8,15 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Order; +using BenchmarkDotNet.Phd; using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; using JetBrains.Annotations; using Perfolizer.Horology; +using Perfolizer.Phd; +using Perfolizer.Phd.Base; +using Perfolizer.Phd.Dto; +using Perfolizer.Phd.Tables; namespace BenchmarkDotNet.Reports { @@ -48,7 +53,7 @@ public Summary( CultureInfo cultureInfo, ImmutableArray validationErrors, ImmutableArray columnHidingRules, - SummaryStyle summaryStyle = null) + SummaryStyle? summaryStyle = null) { Title = title; ResultsDirectoryPath = resultsDirectoryPath; @@ -62,7 +67,8 @@ public Summary( DisplayPrecisionManager = new DisplayPrecisionManager(this); Orderer = GetConfiguredOrdererOrDefaultOne(reports.Select(report => report.BenchmarkCase.Config)); - BenchmarksCases = Orderer.GetSummaryOrder(reports.Select(report => report.BenchmarkCase).ToImmutableArray(), this).ToImmutableArray(); // we sort it first + BenchmarksCases = + Orderer.GetSummaryOrder(reports.Select(report => report.BenchmarkCase).ToImmutableArray(), this).ToImmutableArray(); // we sort it first Reports = BenchmarksCases.Select(b => ReportMap[b]).ToImmutableArray(); // we use sorted collection to re-create reports list BaseliningStrategy = BaseliningStrategy.Create(BenchmarksCases); Style = (summaryStyle ?? GetConfiguredSummaryStyleOrDefaultOne(BenchmarksCases)).WithCultureInfo(cultureInfo); @@ -75,7 +81,7 @@ public Summary( /// /// Returns a report for the given benchmark or null if there is no a corresponded report. /// - public BenchmarkReport this[BenchmarkCase benchmarkCase] => ReportMap.GetValueOrDefault(benchmarkCase); + public BenchmarkReport? this[BenchmarkCase benchmarkCase] => ReportMap.GetValueOrDefault(benchmarkCase); public bool HasCriticalValidationErrors => ValidationErrors.Any(validationError => validationError.IsCritical); @@ -84,8 +90,10 @@ public Summary( public bool IsMultipleRuntimes => isMultipleRuntimes ??= BenchmarksCases.Length > 1 ? BenchmarksCases.Select(benchmark => benchmark.GetRuntime()).Distinct().Count() > 1 : false; - internal static Summary ValidationFailed(string title, string resultsDirectoryPath, string logFilePath, ImmutableArray? validationErrors = null) - => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, validationErrors ?? ImmutableArray.Empty, ImmutableArray.Empty); + internal static Summary ValidationFailed(string title, string resultsDirectoryPath, string logFilePath, + ImmutableArray? validationErrors = null) + => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, + DefaultCultureInfo.Instance, validationErrors ?? ImmutableArray.Empty, ImmutableArray.Empty); internal static Summary Join(List summaries, ClockSpan clockSpan) => new Summary( @@ -108,7 +116,7 @@ internal static string BuildAllRuntimes(HostEnvironmentInfo hostEnvironmentInfo, foreach (var benchmarkReport in reports) { - string runtime = benchmarkReport.GetRuntimeInfo(); + string? runtime = benchmarkReport.GetRuntimeInfo(); if (runtime != null) { string jobId = benchmarkReport.BenchmarkCase.Job.ResolvedId; @@ -135,7 +143,7 @@ internal static string BuildAllRuntimes(HostEnvironmentInfo hostEnvironmentInfo, public bool IsBaseline(BenchmarkCase benchmarkCase) => BaseliningStrategy.IsBaseline(benchmarkCase); - public BenchmarkCase? GetBaseline(string logicalGroupKey) + public BenchmarkCase? GetBaseline(string? logicalGroupKey) => BenchmarksCases .Where(b => GetLogicalGroupKey(b) == logicalGroupKey) .FirstOrDefault(IsBaseline); @@ -152,16 +160,52 @@ private static IOrderer GetConfiguredOrdererOrDefaultOne(IEnumerable config.Orderer != DefaultOrderer.Instance) .Select(config => config.Orderer) .Distinct() - .SingleOrDefault() + .FirstOrDefault() ?? DefaultOrderer.Instance; private static SummaryStyle GetConfiguredSummaryStyleOrDefaultOne(ImmutableArray benchmarkCases) => benchmarkCases .Where(benchmark => benchmark.Config.SummaryStyle != SummaryStyle.Default - && benchmark.Config.SummaryStyle != null) // Paranoid +#nullable disable + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract ConditionIsAlwaysTrueOrFalse + // TODO: remove this check once the nullability migration is finished + && benchmark.Config.SummaryStyle != null) // Paranoid +#nullable enable .Select(benchmark => benchmark.Config.SummaryStyle) .Distinct() - .SingleOrDefault() + .FirstOrDefault() ?? SummaryStyle.Default; + + // TODO: GcStats + public PhdEntry ToPhd() + { + var tableConfig = new PhdTableConfig + { + ColumnDefinitions = + [ + new PhdColumnDefinition(".engine") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PhdColumnDefinition(".host.os") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PhdColumnDefinition(".host.cpu") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PhdColumnDefinition(".benchmark") { Cloud = "secondary" }, + new PhdColumnDefinition(".job") { Cloud = "secondary", Compressed = true }, + new PhdColumnDefinition("=center"), + new PhdColumnDefinition("=spread") + ] + }; + + var root = new PhdEntry + { + Engine = new PhdEngine + { + Name = HostEnvironmentInfo.BenchmarkDotNetCaption, + Version = HostEnvironmentInfo.BenchmarkDotNetVersion, + }, + Host = HostEnvironmentInfo.ToPhd(), + Meta = new PhdMeta { Table = tableConfig } + }; + foreach (var benchmarkReport in Reports) + root.Add(benchmarkReport.ToPhd()); + return root; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Reports/SummaryStyle.cs b/src/BenchmarkDotNet/Reports/SummaryStyle.cs index 23451442df..56d030671b 100644 --- a/src/BenchmarkDotNet/Reports/SummaryStyle.cs +++ b/src/BenchmarkDotNet/Reports/SummaryStyle.cs @@ -3,6 +3,7 @@ using BenchmarkDotNet.Columns; using BenchmarkDotNet.Helpers; using Perfolizer.Horology; +using Perfolizer.Metrology; using static BenchmarkDotNet.Reports.SummaryTable.SummaryTableColumn; // ReSharper disable MemberCanBePrivate.Global @@ -20,9 +21,9 @@ public class SummaryStyle : IEquatable public bool PrintUnitsInContent { get; } public bool PrintZeroValuesInContent { get; } public int MaxParameterColumnWidth { get; } - public SizeUnit SizeUnit { get; } + public SizeUnit? SizeUnit { get; } internal SizeUnit CodeSizeUnit { get; } - public TimeUnit TimeUnit { get; } + public TimeUnit? TimeUnit { get; } public CultureInfo CultureInfo { get; } public RatioStyle RatioStyle { get; } @@ -30,7 +31,7 @@ public class SummaryStyle : IEquatable public TextJustification TextColumnJustification { get; } public TextJustification NumericColumnJustification { get; } - public SummaryStyle(CultureInfo? cultureInfo, bool printUnitsInHeader, SizeUnit sizeUnit, TimeUnit timeUnit, bool printUnitsInContent = true, + public SummaryStyle(CultureInfo? cultureInfo, bool printUnitsInHeader, SizeUnit? sizeUnit, TimeUnit? timeUnit, bool printUnitsInContent = true, bool printZeroValuesInContent = false, int maxParameterColumnWidth = DefaultMaxParameterColumnWidth, RatioStyle ratioStyle = RatioStyle.Value, TextJustification textColumnJustification = TextJustification.Left, TextJustification numericColumnJustification = TextJustification.Right) { diff --git a/src/BenchmarkDotNet/Reports/SummaryTable.cs b/src/BenchmarkDotNet/Reports/SummaryTable.cs index 9ab5e983fd..35e0e315fa 100644 --- a/src/BenchmarkDotNet/Reports/SummaryTable.cs +++ b/src/BenchmarkDotNet/Reports/SummaryTable.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Extensions; using JetBrains.Annotations; using Perfolizer.Horology; +using Perfolizer.Metrology; namespace BenchmarkDotNet.Reports { @@ -24,7 +25,7 @@ public class SummaryTable public SummaryStyle EffectiveSummaryStyle { get; } public bool SeparateLogicalGroups { get; } - internal SummaryTable(Summary summary, SummaryStyle style = null) + internal SummaryTable(Summary summary, SummaryStyle? style = null) { Summary = summary; diff --git a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs index dbe350b2c8..69837feec8 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs @@ -263,16 +263,7 @@ private static void AssertMethodIsAccessible(string methodType, MethodInfo metho { if (!methodInfo.IsPublic) throw new InvalidBenchmarkDeclarationException($"{methodType} method {methodInfo.Name} has incorrect access modifiers.\nMethod must be public."); - - var declaringType = methodInfo.DeclaringType; - - while (declaringType != null) - { - if (!declaringType.GetTypeInfo().IsPublic && !declaringType.GetTypeInfo().IsNestedPublic) - throw new InvalidBenchmarkDeclarationException($"{declaringType.FullName} containing {methodType} method {methodInfo.Name} has incorrect access modifiers.\nDeclaring type must be public."); - - declaringType = declaringType.DeclaringType; - } + /* Moved the code that verifies if DeclaringType of a given MethodInfo (a method) is publicly accessible to CompilationValidator */ } private static void AssertMethodIsNotGeneric(string methodType, MethodInfo methodInfo) diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 80a3ef0897..4ff49156b8 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -8,16 +8,17 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Mathematics; -using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.Parameters; @@ -36,14 +37,13 @@ internal static class BenchmarkRunnerClean internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) { - using var taskbarProgress = new TaskbarProgress(); - taskbarProgress.SetState(TaskbarProgressState.Indeterminate); + using var taskbarProgress = new TaskbarProgress(TaskbarProgressState.Indeterminate); var resolver = DefaultResolver; var artifactsToCleanup = new List(); var rootArtifactsFolderPath = GetRootArtifactsFolderPath(benchmarkRunInfos); - var maxTitleLength = RuntimeInformation.IsWindows() + var maxTitleLength = OsDetector.IsWindows() ? 254 - rootArtifactsFolderPath.Length : int.MaxValue; var title = GetTitle(benchmarkRunInfos, maxTitleLength); @@ -54,6 +54,10 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) using (var streamLogger = new StreamLogger(GetLogFileStreamWriter(benchmarkRunInfos, logFilePath))) { var compositeLogger = CreateCompositeLogger(benchmarkRunInfos, streamLogger); + using var wakeLock = WakeLock.Request(WakeLock.GetWakeLockType(benchmarkRunInfos), "BenchmarkDotNet Running Benchmarks", streamLogger); + var eventProcessor = new CompositeEventProcessor(benchmarkRunInfos); + + eventProcessor.OnStartValidationStage(); compositeLogger.WriteLineInfo("// Validating benchmarks:"); @@ -61,13 +65,15 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) validationErrors.AddRange(Validate(supportedBenchmarks)); + foreach (var validationError in validationErrors) + eventProcessor.OnValidationError(validationError); + PrintValidationErrors(compositeLogger, validationErrors); if (validationErrors.Any(validationError => validationError.IsCritical)) return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath, validationErrors.ToImmutableArray()) }; - if (!supportedBenchmarks.Any(benchmarks => benchmarks.BenchmarksCases.Any())) - return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath) }; + eventProcessor.OnEndValidationStage(); int totalBenchmarkCount = supportedBenchmarks.Sum(benchmarkInfo => benchmarkInfo.BenchmarksCases.Length); int benchmarksToRunCount = totalBenchmarkCount - (idToResume + 1); // ids are indexed from 0 @@ -75,10 +81,39 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) compositeLogger.WriteLineHeader($"// ***** Found {totalBenchmarkCount} benchmark(s) in total *****"); var globalChronometer = Chronometer.Start(); - var buildPartitions = BenchmarkPartitioner.CreateForBuild(supportedBenchmarks, resolver); - var buildResults = BuildInParallel(compositeLogger, rootArtifactsFolderPath, buildPartitions, in globalChronometer); + + var parallelBuildBenchmarks = supportedBenchmarks.Where(x => !x.Config.Options.IsSet(ConfigOptions.DisableParallelBuild)).ToArray(); + var parallelBuildPartitions = BenchmarkPartitioner.CreateForBuild(parallelBuildBenchmarks, resolver); + + var sequentialBuildBenchmarks = supportedBenchmarks.Where(x => x.Config.Options.IsSet(ConfigOptions.DisableParallelBuild)).ToArray(); + var sequentialBuildPartitions = BenchmarkPartitioner.CreateForBuild(sequentialBuildBenchmarks, resolver); + + eventProcessor.OnStartBuildStage([.. parallelBuildPartitions, .. sequentialBuildPartitions]); + + var buildResults = new Dictionary(); + if (parallelBuildBenchmarks.Length > 0) + { + var results = BuildInParallel(compositeLogger, rootArtifactsFolderPath, parallelBuildPartitions, in globalChronometer, eventProcessor); + foreach (var kvp in results) + { + buildResults.Add(kvp.Key, kvp.Value); + } + } + + if (sequentialBuildBenchmarks.Length > 0) + { + var results = BuildSequential(compositeLogger, rootArtifactsFolderPath, sequentialBuildPartitions, in globalChronometer, eventProcessor); + foreach (var kvp in results) + { + buildResults.Add(kvp.Key, kvp.Value); + } + } + var allBuildsHaveFailed = buildResults.Values.All(buildResult => !buildResult.IsBuildSuccess); + eventProcessor.OnEndBuildStage(); + eventProcessor.OnStartRunStage(); + try { var results = new List(); @@ -102,9 +137,11 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) } } - var summary = Run(benchmarkRunInfo, benchmarkToBuildResult, resolver, compositeLogger, artifactsToCleanup, + eventProcessor.OnStartRunBenchmarksInType(benchmarkRunInfo.Type, benchmarkRunInfo.BenchmarksCases); + var summary = Run(benchmarkRunInfo, benchmarkToBuildResult, resolver, compositeLogger, eventProcessor, artifactsToCleanup, resultsFolderPath, logFilePath, totalBenchmarkCount, in runsChronometer, ref benchmarksToRunCount, taskbarProgress); + eventProcessor.OnEndRunBenchmarksInType(benchmarkRunInfo.Type, summary); if (!benchmarkRunInfo.Config.Options.IsSet(ConfigOptions.JoinSummary)) PrintSummary(compositeLogger, benchmarkRunInfo.Config, summary); @@ -148,6 +185,8 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) Cleanup(compositeLogger, new HashSet(artifactsToCleanup.Distinct())); compositeLogger.WriteLineInfo("Artifacts cleanup is finished"); compositeLogger.Flush(); + + eventProcessor.OnEndRunStage(); } } } @@ -156,6 +195,7 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, Dictionary buildResults, IResolver resolver, ILogger logger, + EventProcessor eventProcessor, List artifactsToCleanup, string resultsFolderPath, string logFilePath, @@ -198,7 +238,10 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, if (!config.Options.IsSet(ConfigOptions.KeepBenchmarkFiles)) artifactsToCleanup.AddRange(buildResult.ArtifactsToCleanup); + eventProcessor.OnStartRunBenchmark(benchmark); var report = RunCore(benchmark, info.benchmarkId, logger, resolver, buildResult); + eventProcessor.OnEndRunBenchmark(benchmark, report); + if (report.AllMeasurements.Any(m => m.Operations == 0)) throw new InvalidOperationException("An iteration with 'Operations == 0' detected"); reports.Add(report); @@ -219,7 +262,7 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, reports.Add(new BenchmarkReport(false, benchmark, buildResult, buildResult, default, default)); if (buildResult.GenerateException != null) - logger.WriteLineError($"// Generate Exception: {buildResult.GenerateException.Message}"); + logger.WriteLineError($"// Generate Exception: {buildResult.GenerateException}"); else if (!buildResult.IsBuildSuccess && buildResult.TryToExplainFailureReason(out string reason)) logger.WriteLineError($"// Build Error: {reason}"); else if (buildResult.ErrorMessage != null) @@ -306,14 +349,14 @@ private static void PrintSummary(ILogger logger, ImmutableConfig config, Summary if (columnWithLegends.Any()) maxNameWidth = Math.Max(maxNameWidth, columnWithLegends.Select(c => c.ColumnName.Length).Max()); if (effectiveTimeUnit != null) - maxNameWidth = Math.Max(maxNameWidth, effectiveTimeUnit.Name.ToString(cultureInfo).Length + 2); + maxNameWidth = Math.Max(maxNameWidth, effectiveTimeUnit.Abbreviation.ToString(cultureInfo).Length + 2); foreach (var column in columnWithLegends) logger.WriteLineHint($" {column.ColumnName.PadRight(maxNameWidth, ' ')} : {column.Legend}"); if (effectiveTimeUnit != null) - logger.WriteLineHint($" {("1 " + effectiveTimeUnit.Name).PadRight(maxNameWidth, ' ')} :" + - $" 1 {effectiveTimeUnit.Description} ({TimeUnit.Convert(1, effectiveTimeUnit, TimeUnit.Second).ToString("0.#########", summary.GetCultureInfo())} sec)"); + logger.WriteLineHint($" {("1 " + effectiveTimeUnit.Abbreviation).PadRight(maxNameWidth, ' ')} :" + + $" 1 {effectiveTimeUnit.FullName} ({TimeUnit.Convert(1, effectiveTimeUnit, TimeUnit.Second).ToString("0.#########", summary.GetCultureInfo())} sec)"); } if (config.GetDiagnosers().Any()) @@ -330,24 +373,13 @@ private static ImmutableArray Validate(params BenchmarkRunInfo[ { var validationErrors = new List(); - if (benchmarks.Any(b => b.Config.Options.IsSet(ConfigOptions.JoinSummary))) - { - var joinedCases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - validationErrors.AddRange( - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(joinedCases, null)) - ); - } - foreach (var benchmarkRunInfo in benchmarks) validationErrors.AddRange(benchmarkRunInfo.Config.GetCompositeValidator().Validate(new ValidationParameters(benchmarkRunInfo.BenchmarksCases, benchmarkRunInfo.Config))); return validationErrors.ToImmutableArray(); } - private static Dictionary BuildInParallel(ILogger logger, string rootArtifactsFolderPath, BuildPartition[] buildPartitions, in StartedClock globalChronometer) + private static Dictionary BuildInParallel(ILogger logger, string rootArtifactsFolderPath, BuildPartition[] buildPartitions, in StartedClock globalChronometer, EventProcessor eventProcessor) { logger.WriteLineHeader($"// ***** Building {buildPartitions.Length} exe(s) in Parallel: Start *****"); @@ -357,8 +389,18 @@ private static Dictionary BuildInParallel(ILogger l var buildResults = buildPartitions .AsParallel() - .Select(buildPartition => (buildPartition, buildResult: Build(buildPartition, rootArtifactsFolderPath, buildLogger))) - .ToDictionary(result => result.buildPartition, result => result.buildResult); + .Select(buildPartition => (Partition: buildPartition, Result: Build(buildPartition, rootArtifactsFolderPath, buildLogger))) + .AsSequential() // Ensure that build completion events are processed sequentially + .Select(build => + { + // If the generation was successful, but the build was not, we will try building sequentially + // so don't send the OnBuildComplete event yet. + if (buildPartitions.Length <= 1 || !build.Result.IsGenerateSuccess || build.Result.IsBuildSuccess) + eventProcessor.OnBuildComplete(build.Partition, build.Result); + + return build; + }) + .ToDictionary(build => build.Partition, build => build.Result); var afterParallelBuild = globalChronometer.GetElapsed(); @@ -370,18 +412,45 @@ private static Dictionary BuildInParallel(ILogger l logger.WriteLineHeader("// ***** Failed to build in Parallel, switching to sequential build *****"); foreach (var buildPartition in buildPartitions) - if (buildResults[buildPartition].IsGenerateSuccess && !buildResults[buildPartition].IsBuildSuccess && !buildResults[buildPartition].TryToExplainFailureReason(out string _)) - buildResults[buildPartition] = Build(buildPartition, rootArtifactsFolderPath, buildLogger); + { + if (buildResults[buildPartition].IsGenerateSuccess && !buildResults[buildPartition].IsBuildSuccess) + { + if (!buildResults[buildPartition].TryToExplainFailureReason(out string _)) + buildResults[buildPartition] = Build(buildPartition, rootArtifactsFolderPath, buildLogger); + + eventProcessor.OnBuildComplete(buildPartition, buildResults[buildPartition]); + } + } var afterSequentialBuild = globalChronometer.GetElapsed(); logger.WriteLineHeader($"// ***** Done, took {GetFormattedDifference(afterParallelBuild, afterSequentialBuild)} *****"); return buildResults; + } + + private static Dictionary BuildSequential(ILogger logger, string rootArtifactsFolderPath, BuildPartition[] buildPartitions, in StartedClock globalChronometer, EventProcessor eventProcessor) + { + logger.WriteLineHeader($"// ***** Building {buildPartitions.Length} exe(s) in Sequential: Start *****"); + + var beforeBuild = globalChronometer.GetElapsed(); + + var buildResults = new Dictionary(); + foreach (var buildPartition in buildPartitions) + { + buildResults[buildPartition] = Build(buildPartition, rootArtifactsFolderPath, logger); + eventProcessor.OnBuildComplete(buildPartition, buildResults[buildPartition]); + } + + var afterBuild = globalChronometer.GetElapsed(); + + logger.WriteLineHeader($"// ***** Done, took {GetFormattedDifference(beforeBuild, afterBuild)} *****"); + + return buildResults; + } - static string GetFormattedDifference(ClockSpan before, ClockSpan after) + private static string GetFormattedDifference(ClockSpan before, ClockSpan after) => (after.GetTimeSpan() - before.GetTimeSpan()).ToFormattedTotalTime(DefaultCultureInfo.Instance); - } private static BuildResult Build(BuildPartition buildPartition, string rootArtifactsFolderPath, ILogger buildLogger) { @@ -549,20 +618,47 @@ private static void LogTotalTime(ILogger logger, TimeSpan time, int executedBenc private static (BenchmarkRunInfo[], List) GetSupportedBenchmarks(BenchmarkRunInfo[] benchmarkRunInfos, IResolver resolver) { List validationErrors = new (); + List runInfos = new (benchmarkRunInfos.Length); + + if (benchmarkRunInfos.Length == 0) + { + validationErrors.Add(new ValidationError(true, $"No benchmarks were found.")); + return (Array.Empty(), validationErrors); + } - var runInfos = benchmarkRunInfos.Select(info => new BenchmarkRunInfo( - info.BenchmarksCases.Where(benchmark => + foreach (var benchmarkRunInfo in benchmarkRunInfos) + { + if (benchmarkRunInfo.BenchmarksCases.Length == 0) + { + validationErrors.Add(new ValidationError(true, $"No [Benchmark] attribute found on '{benchmarkRunInfo.Type.Name}' benchmark case.")); + continue; + } + + var validBenchmarks = benchmarkRunInfo.BenchmarksCases + .Where(benchmark => { - var errors = benchmark.GetToolchain().Validate(benchmark, resolver).ToArray(); + + var errors = benchmark.GetToolchain() + .Validate(benchmark, resolver) + .ToArray(); + validationErrors.AddRange(errors); - return !errors.Any(); - }).ToArray(), - info.Type, - info.Config)) - .Where(infos => infos.BenchmarksCases.Any()) - .ToArray(); - - return (runInfos, validationErrors); + + return errors.Length == 0; + }) + .ToArray(); + + runInfos.Add( + new BenchmarkRunInfo( + validBenchmarks, + benchmarkRunInfo.Type, + benchmarkRunInfo.Config + + )); + + + } + return (runInfos.ToArray(), validationErrors); } private static string GetRootArtifactsFolderPath(BenchmarkRunInfo[] benchmarkRunInfos) @@ -635,6 +731,9 @@ void AddLogger(ILogger logger) foreach (var logger in benchmarkRunInfo.Config.GetLoggers()) AddLogger(logger); + if (benchmarkRunInfos.Length == 0) + AddLogger(new ConsoleLogger()); + AddLogger(streamLogger); return new CompositeLogger(loggers.Values.ToImmutableHashSet()); @@ -672,9 +771,9 @@ private static void LogProgress(ILogger logger, in StartedClock runsChronometer, $" Estimated finish {estimatedEnd:yyyy-MM-dd H:mm} ({(int)fromNow.TotalHours}h {fromNow.Minutes}m from now) **"; logger.WriteLineHeader(message); - consoleTitler.UpdateTitle ($"{benchmarksToRunCount}/{totalBenchmarkCount} Remaining - {(int)fromNow.TotalHours}h {fromNow.Minutes}m to finish"); + consoleTitler.UpdateTitle($"{benchmarksToRunCount}/{totalBenchmarkCount} Remaining - {(int)fromNow.TotalHours}h {fromNow.Minutes}m to finish"); - taskbarProgress.SetProgress((float) executedBenchmarkCount / totalBenchmarkCount); + taskbarProgress.SetProgress((float)executedBenchmarkCount / totalBenchmarkCount); } private static TimeSpan GetEstimatedFinishTime(in StartedClock runsChronometer, int benchmarksToRunCount, int executedBenchmarkCount) @@ -729,4 +828,4 @@ private static int GetIdToResume(string rootArtifactsFolderPath, string currentL return -1; } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/BuildPartition.cs b/src/BenchmarkDotNet/Running/BuildPartition.cs index f89863215c..d7fb9bc8a8 100644 --- a/src/BenchmarkDotNet/Running/BuildPartition.cs +++ b/src/BenchmarkDotNet/Running/BuildPartition.cs @@ -1,11 +1,16 @@ using System; using System.IO; using System.Reflection; +using System.Threading; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.MonoWasm; using BenchmarkDotNet.Toolchains.Roslyn; using JetBrains.Annotations; @@ -14,12 +19,20 @@ namespace BenchmarkDotNet.Running { public class BuildPartition { + // We use an auto-increment global counter instead of Guid to guarantee uniqueness per benchmark run (Guid has a small chance to collide), + // assuming there are fewer than 4 billion build partitions (a safe assumption). + internal static int s_partitionCounter; + public BuildPartition(BenchmarkBuildInfo[] benchmarks, IResolver resolver) { Resolver = resolver; RepresentativeBenchmarkCase = benchmarks[0].BenchmarkCase; Benchmarks = benchmarks; - ProgramName = benchmarks[0].Config.Options.IsSet(ConfigOptions.KeepBenchmarkFiles) ? RepresentativeBenchmarkCase.Job.FolderInfo : Guid.NewGuid().ToString(); + // Combine the benchmark's assembly name, folder info, and build partition id. + string benchmarkAssemblyName = RepresentativeBenchmarkCase.Descriptor.Type.Assembly.GetName().Name; + string folderInfo = RepresentativeBenchmarkCase.Job.FolderInfo; + int id = Interlocked.Increment(ref s_partitionCounter); + ProgramName = $"{benchmarkAssemblyName}-{folderInfo}-{id}"; LogBuildOutput = benchmarks[0].Config.Options.IsSet(ConfigOptions.LogBuildOutput); GenerateMSBuildBinLog = benchmarks[0].Config.Options.IsSet(ConfigOptions.GenerateMSBuildBinLog); } @@ -71,5 +84,20 @@ private static string GetResolvedAssemblyLocation(Assembly assembly) => // in case of SingleFile, location.Length returns 0, so we use GetName() and // manually construct the path. assembly.Location.Length == 0 ? Path.Combine(AppContext.BaseDirectory, assembly.GetName().Name) : assembly.Location; + + internal bool ForcedNoDependenciesForIntegrationTests + { + get + { + if (!XUnitHelper.IsIntegrationTest.Value || !RuntimeInformation.IsNetCore) + return false; + + var job = RepresentativeBenchmarkCase.Job; + if (job.GetToolchain().Builder is not DotNetCliBuilder) + return false; + + return !job.HasDynamicBuildCharacteristic(); + } + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/ConsoleTitler.cs b/src/BenchmarkDotNet/Running/ConsoleTitler.cs index 9f8e21a1fe..d045e498aa 100644 --- a/src/BenchmarkDotNet/Running/ConsoleTitler.cs +++ b/src/BenchmarkDotNet/Running/ConsoleTitler.cs @@ -1,6 +1,7 @@ using System; using System.IO; -using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Helpers; namespace BenchmarkDotNet.Running { @@ -8,7 +9,7 @@ namespace BenchmarkDotNet.Running /// Updates Console.Title, subject to platform capabilities and Console availability. /// Restores the original (or fallback) title upon disposal. /// - internal class ConsoleTitler : IDisposable + internal class ConsoleTitler : DisposeAtProcessTermination { /// /// Whether this instance has any effect. This will be false if the platform doesn't support Console retitling, @@ -62,7 +63,7 @@ public ConsoleTitler(string initialTitle) #if NET6_0_OR_GREATER [System.Runtime.Versioning.SupportedOSPlatformGuard("windows")] #endif - private static bool PlatformSupportsTitleRead() => RuntimeInformation.IsWindows(); + private static bool PlatformSupportsTitleRead() => OsDetector.IsWindows(); /// /// Updates Console.Title if enabled. @@ -75,13 +76,14 @@ public void UpdateTitle(string title) } } - public void Dispose() + public override void Dispose() { if (IsEnabled) { Console.Title = oldConsoleTitle; IsEnabled = false; } + base.Dispose(); } } } diff --git a/src/BenchmarkDotNet/Running/Descriptor.cs b/src/BenchmarkDotNet/Running/Descriptor.cs index 2e747bb1cd..4ff74fced7 100644 --- a/src/BenchmarkDotNet/Running/Descriptor.cs +++ b/src/BenchmarkDotNet/Running/Descriptor.cs @@ -11,10 +11,10 @@ public class Descriptor : IEquatable { public Type Type { get; } public MethodInfo WorkloadMethod { get; } - public MethodInfo GlobalSetupMethod { get; } - public MethodInfo GlobalCleanupMethod { get; } - public MethodInfo IterationSetupMethod { get; } - public MethodInfo IterationCleanupMethod { get; } + public MethodInfo? GlobalSetupMethod { get; } + public MethodInfo? GlobalCleanupMethod { get; } + public MethodInfo? IterationSetupMethod { get; } + public MethodInfo? IterationCleanupMethod { get; } public string AdditionalLogic { get; } public int OperationsPerInvoke { get; } public string WorkloadMethodDisplayInfo { get; } @@ -22,26 +22,29 @@ public class Descriptor : IEquatable public bool Baseline { get; } public string[] Categories { get; } - internal string TypeInfo => Type?.GetDisplayName() ?? "Untitled"; - private string MethodFolderInfo => WorkloadMethod?.Name ?? "Untitled"; + internal string TypeInfo => Type.GetDisplayName(); + private string MethodFolderInfo => WorkloadMethod.Name; - public string FolderInfo => (Type != null ? FolderNameHelper.ToFolderName(Type) : "Untitled") + "_" + MethodFolderInfo; + public string FolderInfo => $"{FolderNameHelper.ToFolderName(Type)}_{MethodFolderInfo}"; public string DisplayInfo => TypeInfo + "." + WorkloadMethodDisplayInfo; public Descriptor( Type type, MethodInfo workloadMethod, - MethodInfo globalSetupMethod = null, - MethodInfo globalCleanupMethod = null, - MethodInfo iterationSetupMethod = null, - MethodInfo iterationCleanupMethod = null, - string description = null, - string additionalLogic = null, + MethodInfo? globalSetupMethod = null, + MethodInfo? globalCleanupMethod = null, + MethodInfo? iterationSetupMethod = null, + MethodInfo? iterationCleanupMethod = null, + string? description = null, + string? additionalLogic = null, bool baseline = false, - string[] categories = null, + string[]? categories = null, int operationsPerInvoke = 1, int methodIndex = 0) { + Assertion.NotNull(nameof(type), type); + Assertion.NotNull(nameof(workloadMethod), workloadMethod); + Type = type; WorkloadMethod = workloadMethod; GlobalSetupMethod = globalSetupMethod; @@ -58,9 +61,9 @@ public Descriptor( public override string ToString() => DisplayInfo; - private static string FormatDescription(string? description) + private static string? FormatDescription(string? description) { - var specialSymbols = new[] { ' ', '\'', '[', ']' }; + char[] specialSymbols = { ' ', '\'', '[', ']' }; return description != null && specialSymbols.Any(description.Contains) ? "'" + description + "'" : description; @@ -70,9 +73,9 @@ private static string FormatDescription(string? description) public string GetFilterName() => $"{Type.GetCorrectCSharpTypeName(includeGenericArgumentsNamespace: false)}.{WorkloadMethod.Name}"; - public bool Equals(Descriptor other) => GetFilterName().Equals(other.GetFilterName()); + public bool Equals(Descriptor? other) => GetFilterName().Equals(other?.GetFilterName()); - public override bool Equals(object obj) => obj is Descriptor && Equals((Descriptor)obj); + public override bool Equals(object? obj) => obj is Descriptor descriptor && Equals(descriptor); public override int GetHashCode() => GetFilterName().GetHashCode(); } diff --git a/src/BenchmarkDotNet/Running/PowerManagementApplier.cs b/src/BenchmarkDotNet/Running/PowerManagementApplier.cs index fdfa0239f5..f48d95d61e 100644 --- a/src/BenchmarkDotNet/Running/PowerManagementApplier.cs +++ b/src/BenchmarkDotNet/Running/PowerManagementApplier.cs @@ -1,13 +1,13 @@ using System; using System.Collections.Generic; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Running { - internal class PowerManagementApplier : IDisposable + internal class PowerManagementApplier : DisposeAtProcessTermination { private static readonly Guid UserPowerPlan = new Guid("67b4a053-3646-4532-affd-0535c9ea82a7"); @@ -27,13 +27,17 @@ internal class PowerManagementApplier : IDisposable internal PowerManagementApplier(ILogger logger) => this.logger = logger; - public void Dispose() => ApplyUserPowerPlan(); + public override void Dispose() + { + ApplyUserPowerPlan(); + base.Dispose(); + } internal static Guid Map(PowerPlan value) => PowerPlansDict[value]; internal void ApplyPerformancePlan(Guid id) { - if (!RuntimeInformation.IsWindows() || id == Guid.Empty) + if (!OsDetector.IsWindows() || id == Guid.Empty) return; if (id != UserPowerPlan) @@ -44,7 +48,7 @@ internal void ApplyPerformancePlan(Guid id) private void ApplyUserPowerPlan() { - if (powerPlanChanged && RuntimeInformation.IsWindows()) + if (powerPlanChanged && OsDetector.IsWindows()) { try { diff --git a/src/BenchmarkDotNet/Running/TypeFilter.cs b/src/BenchmarkDotNet/Running/TypeFilter.cs index 4e58ebdeaf..4d2188844f 100644 --- a/src/BenchmarkDotNet/Running/TypeFilter.cs +++ b/src/BenchmarkDotNet/Running/TypeFilter.cs @@ -15,6 +15,32 @@ public static (bool allTypesValid, IReadOnlyList runnable) GetTypesWithRun { var validRunnableTypes = new List(); + bool hasRunnableTypeBenchmarks = types.Any(type => type.ContainsRunnableBenchmarks()); + bool hasRunnableAssemblyBenchmarks = assemblies.Any(assembly => GenericBenchmarksBuilder.GetRunnableBenchmarks(assembly.GetRunnableBenchmarks()).Length > 0); + + if (!hasRunnableTypeBenchmarks && !hasRunnableAssemblyBenchmarks) + { + if (types.Any()) + { + foreach (var type in types) + { + logger.WriteLineError($"No [Benchmark] attribute found on '{type.Name}' benchmark case."); + } + } + else if (assemblies.Any()) + { + foreach (var assembly in assemblies) + { + logger.WriteLineError($"No [Benchmark] attribute found on '{assembly.GetName().Name}' assembly."); + } + } + else + { + logger.WriteLineError("No benchmarks were found."); + } + return (false, Array.Empty()); + } + foreach (var type in types) { if (type.ContainsRunnableBenchmarks()) diff --git a/src/BenchmarkDotNet/Running/WakeLock.PInvoke.cs b/src/BenchmarkDotNet/Running/WakeLock.PInvoke.cs new file mode 100644 index 0000000000..9da3fd1eff --- /dev/null +++ b/src/BenchmarkDotNet/Running/WakeLock.PInvoke.cs @@ -0,0 +1,77 @@ +using System.ComponentModel; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Running; + +internal partial class WakeLock +{ + private static class PInvoke + { + public static SafePowerHandle PowerCreateRequest(string reason) + { + REASON_CONTEXT context = new REASON_CONTEXT() + { + Version = POWER_REQUEST_CONTEXT_VERSION, + Flags = POWER_REQUEST_CONTEXT_FLAGS.POWER_REQUEST_CONTEXT_SIMPLE_STRING, + SimpleReasonString = reason + }; + SafePowerHandle safePowerHandle = PowerCreateRequest(context); + if (safePowerHandle.IsInvalid) { throw new Win32Exception(); } + return safePowerHandle; + } + + [DllImport("kernel32.dll", ExactSpelling = true, SetLastError = true)] + private static extern SafePowerHandle PowerCreateRequest(REASON_CONTEXT Context); + + public static void PowerSetRequest(SafePowerHandle safePowerHandle, POWER_REQUEST_TYPE requestType) + { + if (!InvokePowerSetRequest(safePowerHandle, requestType)) + { + throw new Win32Exception(); + } + } + + [DllImport("kernel32.dll", EntryPoint = "PowerSetRequest", ExactSpelling = true, SetLastError = true)] + private static extern bool InvokePowerSetRequest(SafePowerHandle PowerRequest, POWER_REQUEST_TYPE RequestType); + + public static void PowerClearRequest(SafePowerHandle safePowerHandle, POWER_REQUEST_TYPE requestType) + { + if (!InvokePowerClearRequest(safePowerHandle, requestType)) + { + throw new Win32Exception(); + } + } + + [DllImport("kernel32.dll", EntryPoint = "PowerClearRequest", ExactSpelling = true, SetLastError = true)] + private static extern bool InvokePowerClearRequest(SafePowerHandle PowerRequest, POWER_REQUEST_TYPE RequestType); + + [DllImport("kernel32.dll", ExactSpelling = true, SetLastError = true)] + public static extern bool CloseHandle(nint hObject); + + private struct REASON_CONTEXT + { + public uint Version; + + public POWER_REQUEST_CONTEXT_FLAGS Flags; + + [MarshalAs(UnmanagedType.LPWStr)] + public string SimpleReasonString; + } + + private const uint POWER_REQUEST_CONTEXT_VERSION = 0U; + + private enum POWER_REQUEST_CONTEXT_FLAGS : uint + { + POWER_REQUEST_CONTEXT_DETAILED_STRING = 2U, + POWER_REQUEST_CONTEXT_SIMPLE_STRING = 1U, + } + + public enum POWER_REQUEST_TYPE + { + PowerRequestDisplayRequired = 0, + PowerRequestSystemRequired = 1, + PowerRequestAwayModeRequired = 2, + PowerRequestExecutionRequired = 3, + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/WakeLock.SafePowerHandle.cs b/src/BenchmarkDotNet/Running/WakeLock.SafePowerHandle.cs new file mode 100644 index 0000000000..8e6203574d --- /dev/null +++ b/src/BenchmarkDotNet/Running/WakeLock.SafePowerHandle.cs @@ -0,0 +1,13 @@ +using Microsoft.Win32.SafeHandles; + +namespace BenchmarkDotNet.Running; + +internal partial class WakeLock +{ + private sealed class SafePowerHandle : SafeHandleZeroOrMinusOneIsInvalid + { + private SafePowerHandle() : base(true) { } + + protected override bool ReleaseHandle() => PInvoke.CloseHandle(handle); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/WakeLock.cs b/src/BenchmarkDotNet/Running/WakeLock.cs new file mode 100644 index 0000000000..124fbd5e22 --- /dev/null +++ b/src/BenchmarkDotNet/Running/WakeLock.cs @@ -0,0 +1,69 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Loggers; +using System; +using System.ComponentModel; +using System.Linq; + +namespace BenchmarkDotNet.Running; + +internal partial class WakeLock +{ + public static WakeLockType GetWakeLockType(BenchmarkRunInfo[] benchmarkRunInfos) => + benchmarkRunInfos.Length == 0 ? WakeLockType.None : benchmarkRunInfos.Select(static i => i.Config.WakeLock).Max(); + + private static readonly bool OsVersionIsSupported = + // Must be windows 7 or greater + OsDetector.IsWindows() && Environment.OSVersion.Version >= new Version(6, 1); + + public static IDisposable? Request(WakeLockType wakeLockType, string reason, ILogger logger) => + wakeLockType == WakeLockType.None || !OsVersionIsSupported ? null : new WakeLockSentinel(wakeLockType, reason, logger); + + private class WakeLockSentinel : DisposeAtProcessTermination + { + private readonly WakeLockType wakeLockType; + private readonly SafePowerHandle? safePowerHandle; + private readonly ILogger logger; + + public WakeLockSentinel(WakeLockType wakeLockType, string reason, ILogger logger) + { + this.wakeLockType = wakeLockType; + this.logger = logger; + try + { + safePowerHandle = PInvoke.PowerCreateRequest(reason); + PInvoke.PowerSetRequest(safePowerHandle, PInvoke.POWER_REQUEST_TYPE.PowerRequestSystemRequired); + if (wakeLockType == WakeLockType.Display) + { + PInvoke.PowerSetRequest(safePowerHandle, PInvoke.POWER_REQUEST_TYPE.PowerRequestDisplayRequired); + } + } + catch (Win32Exception ex) + { + logger.WriteLineError($"Unable to prevent the system from entering sleep or turning off the display (error message: {ex.Message})."); + } + } + + public override void Dispose() + { + if (safePowerHandle != null) + { + try + { + if (wakeLockType == WakeLockType.Display) + { + PInvoke.PowerClearRequest(safePowerHandle, PInvoke.POWER_REQUEST_TYPE.PowerRequestDisplayRequired); + } + PInvoke.PowerClearRequest(safePowerHandle, PInvoke.POWER_REQUEST_TYPE.PowerRequestSystemRequired); + } + catch (Win32Exception ex) + { + logger.WriteLineError($"Unable to allow the system from entering sleep or turning off the display (error message: {ex.Message})."); + } + safePowerHandle.Dispose(); + } + base.Dispose(); + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Templates/BenchmarkType.txt b/src/BenchmarkDotNet/Templates/BenchmarkType.txt index f17737d646..ea1bbaa858 100644 --- a/src/BenchmarkDotNet/Templates/BenchmarkType.txt +++ b/src/BenchmarkDotNet/Templates/BenchmarkType.txt @@ -3,7 +3,7 @@ { public static void Run(BenchmarkDotNet.Engines.IHost host, System.String benchmarkName) { - BenchmarkDotNet.Autogenerated.Runnable_$ID$ instance = new BenchmarkDotNet.Autogenerated.Runnable_$ID$(); // do NOT change name "instance" (used in SmartParamameter) + BenchmarkDotNet.Autogenerated.Runnable_$ID$ instance = new BenchmarkDotNet.Autogenerated.Runnable_$ID$ { $ParamsInitializer$ }; // do NOT change name "instance" (used in SmartParamameter) $ParamsContent$ host.WriteLine(); diff --git a/src/BenchmarkDotNet/Templates/CsProj.txt b/src/BenchmarkDotNet/Templates/CsProj.txt index 5cd08250dc..99e536b985 100644 --- a/src/BenchmarkDotNet/Templates/CsProj.txt +++ b/src/BenchmarkDotNet/Templates/CsProj.txt @@ -9,7 +9,6 @@ $PLATFORM$ $PROGRAMNAME$ Exe - bin\$CONFIGURATIONNAME$ False false @@ -23,9 +22,10 @@ true latest - true false + + true BenchmarkDotNet.Autogenerated.UniqueProgramName diff --git a/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt b/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt index 1db40072d1..adcb87389e 100644 --- a/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt +++ b/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt @@ -9,7 +9,6 @@ Exe - bin $TFM$ $RUNTIMEPACK$ $RUNTIMEIDENTIFIER$ @@ -20,6 +19,8 @@ true BenchmarkDotNet.Autogenerated.UniqueProgramName true + + true diff --git a/src/BenchmarkDotNet/Templates/WasmCsProj.txt b/src/BenchmarkDotNet/Templates/WasmCsProj.txt index 1910911fe0..de6dc3ee5c 100644 --- a/src/BenchmarkDotNet/Templates/WasmCsProj.txt +++ b/src/BenchmarkDotNet/Templates/WasmCsProj.txt @@ -11,7 +11,6 @@ Exe - bin Release false $TFM$ @@ -23,8 +22,11 @@ $RUN_AOT$ $(RunAOTCompilation) true + false false false + + true BenchmarkDotNet.Autogenerated.UniqueProgramName diff --git a/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs b/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs index 72faf5c956..0c097f5bb0 100644 --- a/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs +++ b/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs @@ -4,11 +4,12 @@ namespace BenchmarkDotNet.Toolchains { public class ArtifactsPaths { - public static readonly ArtifactsPaths Empty = new ArtifactsPaths("", "", "", "", "", "", "", "", "", "", ""); + public static readonly ArtifactsPaths Empty = new ArtifactsPaths("", "", "", "", "", "", "", "", "", "", "", ""); [PublicAPI] public string RootArtifactsFolderPath { get; } [PublicAPI] public string BuildArtifactsDirectoryPath { get; } [PublicAPI] public string BinariesDirectoryPath { get; } + [PublicAPI] public string IntermediateDirectoryPath { get; } [PublicAPI] public string ProgramCodePath { get; } [PublicAPI] public string AppConfigPath { get; } [PublicAPI] public string NuGetConfigPath { get; } @@ -22,6 +23,7 @@ public ArtifactsPaths( string rootArtifactsFolderPath, string buildArtifactsDirectoryPath, string binariesDirectoryPath, + string intermediateDirectoryPath, string programCodePath, string appConfigPath, string nuGetConfigPath, @@ -34,6 +36,7 @@ public ArtifactsPaths( RootArtifactsFolderPath = rootArtifactsFolderPath; BuildArtifactsDirectoryPath = buildArtifactsDirectoryPath; BinariesDirectoryPath = binariesDirectoryPath; + IntermediateDirectoryPath = intermediateDirectoryPath; ProgramCodePath = programCodePath; AppConfigPath = appConfigPath; NuGetConfigPath = nuGetConfigPath; diff --git a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunPublisher.cs b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunPublisher.cs index b9605867fe..c77b56b039 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunPublisher.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunPublisher.cs @@ -11,7 +11,7 @@ namespace BenchmarkDotNet.Toolchains.CoreRun { public class CoreRunPublisher : IBuilder { - public CoreRunPublisher(FileInfo coreRun, FileInfo customDotNetCliPath = null) + public CoreRunPublisher(FileInfo coreRun, FileInfo? customDotNetCliPath = null) { CoreRun = coreRun; DotNetCliPublisher = new DotNetCliPublisher(customDotNetCliPath?.FullName); diff --git a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs index 03d677a5aa..3cbaa80f65 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs @@ -15,13 +15,13 @@ public class CoreRunToolchain : IToolchain /// /// the path to CoreRun /// /should a copy of CoreRun be performed? True by default. The toolchain replaces old dependencies in CoreRun folder with newer versions if used by the benchmarks. - /// TFM, netcoreapp2.1 is the default + /// TFM, net8.0 is the default /// path to dotnet cli, if not provided the one from PATH will be used /// display name, CoreRun is the default value /// the directory to restore packages to public CoreRunToolchain(FileInfo coreRun, bool createCopy = true, - string targetFrameworkMoniker = "netcoreapp2.1", - FileInfo customDotNetCliPath = null, DirectoryInfo restorePath = null, + string targetFrameworkMoniker = "net8.0", + FileInfo? customDotNetCliPath = null, DirectoryInfo? restorePath = null, string displayName = "CoreRun") { if (coreRun == null) throw new ArgumentNullException(nameof(coreRun)); @@ -66,7 +66,7 @@ public IEnumerable Validate(BenchmarkCase benchmark, IResolver $"Provided CoreRun path does not exist, benchmark '{benchmark.DisplayInfo}' will not be executed. Please remember that BDN expects path to CoreRun.exe (corerun on Unix), not to Core_Root folder.", benchmark); } - else if (Toolchain.IsCliPathInvalid(CustomDotNetCliPath?.FullName, benchmark, out var invalidCliError)) + else if (DotNetSdkValidator.IsCliPathInvalid(CustomDotNetCliPath?.FullName, benchmark, out var invalidCliError)) { yield return invalidCliError; } diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs index 4e1fe4a641..2761bd7c2c 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; @@ -25,7 +26,7 @@ public class CsProjClassicNetToolchain : Toolchain internal string CustomDotNetCliPath { get; } - private CsProjClassicNetToolchain(string targetFrameworkMoniker, string name, string packagesPath = null, string customDotNetCliPath = null) + private CsProjClassicNetToolchain(string targetFrameworkMoniker, string name, string? packagesPath = null, string? customDotNetCliPath = null) : base(name, new CsProjGenerator(targetFrameworkMoniker, customDotNetCliPath, packagesPath, runtimeFrameworkVersion: null, isNetCore: false), new DotNetCliBuilder(targetFrameworkMoniker, customDotNetCliPath), @@ -34,7 +35,7 @@ private CsProjClassicNetToolchain(string targetFrameworkMoniker, string name, st CustomDotNetCliPath = customDotNetCliPath; } - public static IToolchain From(string targetFrameworkMoniker, string packagesPath = null, string customDotNetCliPath = null) + public static IToolchain From(string targetFrameworkMoniker, string? packagesPath = null, string? customDotNetCliPath = null) => new CsProjClassicNetToolchain(targetFrameworkMoniker, targetFrameworkMoniker, packagesPath, customDotNetCliPath); public override IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) @@ -44,16 +45,22 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas yield return validationError; } - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { yield return new ValidationError(true, $"Classic .NET toolchain is supported only for Windows, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", benchmarkCase); + yield break; } - else if (IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError)) + else if (DotNetSdkValidator.IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError)) { yield return invalidCliError; } + + foreach (var validationError in DotNetSdkValidator.ValidateFrameworkSdks(benchmarkCase)) + { + yield return validationError; + } } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs index 63ec4573b0..4aea1327f8 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs @@ -24,6 +24,8 @@ public class CsProjCoreToolchain : Toolchain, IEquatable [PublicAPI] public static readonly IToolchain NetCoreApp60 = From(NetCoreAppSettings.NetCoreApp60); [PublicAPI] public static readonly IToolchain NetCoreApp70 = From(NetCoreAppSettings.NetCoreApp70); [PublicAPI] public static readonly IToolchain NetCoreApp80 = From(NetCoreAppSettings.NetCoreApp80); + [PublicAPI] public static readonly IToolchain NetCoreApp90 = From(NetCoreAppSettings.NetCoreApp90); + [PublicAPI] public static readonly IToolchain NetCoreApp10_0 = From(NetCoreAppSettings.NetCoreApp10_0); internal CsProjCoreToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) : base(name, generator, builder, executor) @@ -48,11 +50,6 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas yield return validationError; } - if (IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError)) - { - yield return invalidCliError; - } - if (benchmarkCase.Job.HasValue(EnvironmentMode.JitCharacteristic) && benchmarkCase.Job.ResolveValue(EnvironmentMode.JitCharacteristic, resolver) == Jit.LegacyJit) { yield return new ValidationError(true, @@ -79,6 +76,11 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas $"Currently CsProjCoreToolchain does not support LINQPad 6+. Please use {nameof(InProcessEmitToolchain)} instead.", benchmarkCase); } + + foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(CustomDotNetCliPath, benchmarkCase)) + { + yield return validationError; + } } public override bool Equals(object obj) => obj is CsProjCoreToolchain typed && Equals(typed); diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs index d732275d95..c324f66b88 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs @@ -34,7 +34,8 @@ public class CsProjGenerator : DotNetCliGenerator, IEquatable "CopyLocalLockFileAssemblies", "PreserveCompilationContext", "UserSecretsId", - "EnablePreviewFeatures" + "EnablePreviewFeatures", + "RuntimeHostConfigurationOption", }.ToImmutableArray(); public string RuntimeFrameworkVersion { get; } @@ -63,6 +64,9 @@ protected override string GetProjectFilePath(string buildArtifactsDirectoryPath) protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, TargetFrameworkMoniker); + protected override string GetIntermediateDirectoryPath(string buildArtifactsDirectoryPath, string configuration) + => Path.Combine(buildArtifactsDirectoryPath, "obj", configuration, TargetFrameworkMoniker); + [SuppressMessage("ReSharper", "StringLiteralTypo")] // R# complains about $variables$ protected override void GenerateProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) { @@ -81,7 +85,6 @@ protected override void GenerateProject(BuildPartition buildPartition, Artifacts .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) .Replace("$RUNTIMESETTINGS$", GetRuntimeSettings(benchmark.Job.Environment.Gc, buildPartition.Resolver)) .Replace("$COPIEDSETTINGS$", customProperties) - .Replace("$CONFIGURATIONNAME$", buildPartition.BuildConfiguration) .Replace("$SDKNAME$", sdkName) .ToString(); @@ -121,7 +124,7 @@ protected virtual string GetRuntimeSettings(GcMode gcMode, IResolver resolver) // custom SDKs are not added for non-netcoreapp apps (like net471), so when the TFM != netcoreapp we dont parse " - string sdkName = null; + string? sdkName = null; if (TargetFrameworkMoniker.StartsWith("netcoreapp", StringComparison.InvariantCultureIgnoreCase)) { foreach (XmlElement importElement in projectElement.GetElementsByTagName("Import")) @@ -161,8 +164,8 @@ protected virtual string GetRuntimeSettings(GcMode gcMode, IResolver resolver) sdkName = DefaultSdkName; } - XmlDocument itemGroupsettings = null; - XmlDocument propertyGroupSettings = null; + XmlDocument? itemGroupsettings = null; + XmlDocument? propertyGroupSettings = null; GetSettingsThatNeedToBeCopied(projectElement, ref itemGroupsettings, ref propertyGroupSettings, projectFile); diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs index dc48965f1e..6223f9ea3c 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Portability; using JetBrains.Annotations; @@ -49,7 +50,7 @@ public CustomDotNetCliToolchainBuilder UseNuGetClearTag(bool value) return this; } - /// TFM, example: netcoreapp2.1 + /// TFM, example: net8.0 [PublicAPI] [SuppressMessage("ReSharper", "ParameterHidesMember")] public CustomDotNetCliToolchainBuilder TargetFrameworkMoniker(string targetFrameworkMoniker) @@ -127,7 +128,7 @@ internal static string GetPortableRuntimeIdentifier() // Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment.GetRuntimeIdentifier() // returns win10-x64, we want the simpler form win-x64 // the values taken from https://docs.microsoft.com/en-us/dotnet/core/rid-catalog#macos-rids - string osPart = RuntimeInformation.IsWindows() ? "win" : (RuntimeInformation.IsMacOS() ? "osx" : "linux"); + string osPart = OsDetector.IsWindows() ? "win" : (OsDetector.IsMacOS() ? "osx" : "linux"); string architecture = #if NETSTANDARD diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs index bad11c91be..7a7afc3184 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs @@ -14,15 +14,13 @@ public class DotNetCliBuilder : IBuilder private string CustomDotNetCliPath { get; } private bool LogOutput { get; } - private bool RetryFailedBuildWithNoDeps { get; } [PublicAPI] - public DotNetCliBuilder(string targetFrameworkMoniker, string customDotNetCliPath = null, bool logOutput = false, bool retryFailedBuildWithNoDeps = true) + public DotNetCliBuilder(string targetFrameworkMoniker, string? customDotNetCliPath = null, bool logOutput = false) { TargetFrameworkMoniker = targetFrameworkMoniker; CustomDotNetCliPath = customDotNetCliPath; LogOutput = logOutput; - RetryFailedBuildWithNoDeps = retryFailedBuildWithNoDeps; } public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger) @@ -35,8 +33,7 @@ public BuildResult Build(GenerateResult generateResult, BuildPartition buildPart buildPartition, Array.Empty(), buildPartition.Timeout, - logOutput: LogOutput, - retryFailedBuildWithNoDeps: RetryFailedBuildWithNoDeps) + logOutput: LogOutput) .RestoreThenBuild(); if (buildResult.IsBuildSuccess && buildPartition.RepresentativeBenchmarkCase.Job.Environment.LargeAddressAware) diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs index 31546d3112..e9078f293f 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs @@ -1,10 +1,10 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Text; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; @@ -32,11 +32,8 @@ public class DotNetCliCommand [PublicAPI] public bool LogOutput { get; } - [PublicAPI] public bool RetryFailedBuildWithNoDeps { get; } - public DotNetCliCommand(string cliPath, string arguments, GenerateResult generateResult, ILogger logger, - BuildPartition buildPartition, IReadOnlyList environmentVariables, TimeSpan timeout, bool logOutput = false, - bool retryFailedBuildWithNoDeps = true) + BuildPartition buildPartition, IReadOnlyList environmentVariables, TimeSpan timeout, bool logOutput = false) { CliPath = cliPath ?? DotNetCliCommandExecutor.DefaultDotNetCliPath.Value; Arguments = arguments; @@ -46,7 +43,6 @@ public DotNetCliCommand(string cliPath, string arguments, GenerateResult generat EnvironmentVariables = environmentVariables; Timeout = timeout; LogOutput = logOutput || (buildPartition is not null && buildPartition.LogBuildOutput); - RetryFailedBuildWithNoDeps = retryFailedBuildWithNoDeps; } public DotNetCliCommand WithArguments(string arguments) @@ -70,30 +66,29 @@ public BuildResult RestoreThenBuild() if (BuildPartition.IsCustomBuildConfiguration) return Build().ToBuildResult(GenerateResult); - var restoreResult = Restore(); - if (!restoreResult.IsSuccess) - return BuildResult.Failure(GenerateResult, restoreResult.AllInformation); - - // On our CI (Windows+.NET 7), Integration tests take to much time because each benchmark run rebuilds the BenchmarkDotNet itself. + // On our CI, Integration tests take too much time, because each benchmark run rebuilds BenchmarkDotNet itself. // To reduce the total duration of the CI workflows, we build all the projects without dependencies - bool forceNoDependencies = XUnitHelper.IsIntegrationTest.Value && - RuntimeInformation.IsWindows() && - RuntimeInformation.IsNetCore; - - DotNetCliCommandResult buildResult; - if (forceNoDependencies) - buildResult = BuildNoRestoreNoDependencies(); - else + if (BuildPartition.ForcedNoDependenciesForIntegrationTests) { - buildResult = BuildNoRestore(); - if (!buildResult.IsSuccess && RetryFailedBuildWithNoDeps) // if we fail to do the full build, we try with --no-dependencies - buildResult = BuildNoRestoreNoDependencies(); + var restoreResult = DotNetCliCommandExecutor.Execute(WithArguments( + GetRestoreCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-dependencies", "restore-no-deps", excludeOutput: true))); + if (!restoreResult.IsSuccess) + return BuildResult.Failure(GenerateResult, restoreResult.AllInformation); + + return DotNetCliCommandExecutor.Execute(WithArguments( + GetBuildCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-restore --no-dependencies", "build-no-restore-no-deps", excludeOutput: true))) + .ToBuildResult(GenerateResult); } + else + { + var restoreResult = Restore(); + if (!restoreResult.IsSuccess) + return BuildResult.Failure(GenerateResult, restoreResult.AllInformation); - if (!buildResult.IsSuccess) - return BuildResult.Failure(GenerateResult, buildResult.AllInformation); - - return buildResult.ToBuildResult(GenerateResult); + // We no longer retry with --no-dependencies, because it fails with --output set at the same time, + // and the artifactsPaths.BinariesDirectoryPath is set before we try to build, so we cannot overwrite it. + return BuildNoRestore().ToBuildResult(GenerateResult); + } } [PublicAPI] @@ -115,14 +110,8 @@ public BuildResult RestoreThenBuildThenPublish() if (!restoreResult.IsSuccess) return BuildResult.Failure(GenerateResult, restoreResult.AllInformation); - var buildResult = BuildNoRestore(); - if (!buildResult.IsSuccess && RetryFailedBuildWithNoDeps) // if we fail to do the full build, we try with --no-dependencies - buildResult = BuildNoRestoreNoDependencies(); - - if (!buildResult.IsSuccess) - return BuildResult.Failure(GenerateResult, buildResult.AllInformation); - - return PublishNoBuildAndNoRestore().ToBuildResult(GenerateResult); + // We use the implicit build in the publish command. We stopped doing a separate build step because we set the --output. + return PublishNoRestore().ToBuildResult(GenerateResult); } public DotNetCliCommandResult AddPackages() @@ -151,22 +140,19 @@ public DotNetCliCommandResult BuildNoRestore() => DotNetCliCommandExecutor.Execute(WithArguments( GetBuildCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-restore", "build-no-restore"))); - public DotNetCliCommandResult BuildNoRestoreNoDependencies() - => DotNetCliCommandExecutor.Execute(WithArguments( - GetBuildCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-restore --no-dependencies", "build-no-restore-no-deps"))); - public DotNetCliCommandResult Publish() => DotNetCliCommandExecutor.Execute(WithArguments( GetPublishCommand(GenerateResult.ArtifactsPaths, BuildPartition, Arguments, "publish"))); - public DotNetCliCommandResult PublishNoBuildAndNoRestore() + // PublishNoBuildAndNoRestore was removed because we set --output in the build step. We use the implicit build included in the publish command. + public DotNetCliCommandResult PublishNoRestore() => DotNetCliCommandExecutor.Execute(WithArguments( - GetPublishCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-build --no-restore", "publish-no-build-no-restore"))); + GetPublishCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-restore", "publish-no-restore"))); internal static IEnumerable GetAddPackagesCommands(BuildPartition buildPartition) => GetNuGetAddPackageCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver); - internal static string GetRestoreCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string extraArguments = null, string binLogSuffix = null) + internal static string GetRestoreCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string? extraArguments = null, string? binLogSuffix = null, bool excludeOutput = false) => new StringBuilder() .AppendArgument("restore") .AppendArgument(string.IsNullOrEmpty(artifactsPaths.PackagesDirectoryName) ? string.Empty : $"--packages \"{artifactsPaths.PackagesDirectoryName}\"") @@ -174,9 +160,10 @@ internal static string GetRestoreCommand(ArtifactsPaths artifactsPaths, BuildPar .AppendArgument(extraArguments) .AppendArgument(GetMandatoryMsBuildSettings(buildPartition.BuildConfiguration)) .AppendArgument(GetMsBuildBinLogArgument(buildPartition, binLogSuffix)) + .MaybeAppendOutputPaths(artifactsPaths, true, excludeOutput) .ToString(); - internal static string GetBuildCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string extraArguments = null, string binLogSuffix = null) + internal static string GetBuildCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string? extraArguments = null, string? binLogSuffix = null, bool excludeOutput = false) => new StringBuilder() .AppendArgument($"build -c {buildPartition.BuildConfiguration}") // we don't need to specify TFM, our auto-generated project contains always single one .AppendArgument(GetCustomMsBuildArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)) @@ -184,9 +171,10 @@ internal static string GetBuildCommand(ArtifactsPaths artifactsPaths, BuildParti .AppendArgument(GetMandatoryMsBuildSettings(buildPartition.BuildConfiguration)) .AppendArgument(string.IsNullOrEmpty(artifactsPaths.PackagesDirectoryName) ? string.Empty : $"/p:NuGetPackageRoot=\"{artifactsPaths.PackagesDirectoryName}\"") .AppendArgument(GetMsBuildBinLogArgument(buildPartition, binLogSuffix)) + .MaybeAppendOutputPaths(artifactsPaths, excludeOutput: excludeOutput) .ToString(); - internal static string GetPublishCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string extraArguments = null, string binLogSuffix = null) + internal static string GetPublishCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string? extraArguments = null, string? binLogSuffix = null) => new StringBuilder() .AppendArgument($"publish -c {buildPartition.BuildConfiguration}") // we don't need to specify TFM, our auto-generated project contains always single one .AppendArgument(GetCustomMsBuildArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)) @@ -194,7 +182,7 @@ internal static string GetPublishCommand(ArtifactsPaths artifactsPaths, BuildPar .AppendArgument(GetMandatoryMsBuildSettings(buildPartition.BuildConfiguration)) .AppendArgument(string.IsNullOrEmpty(artifactsPaths.PackagesDirectoryName) ? string.Empty : $"/p:NuGetPackageRoot=\"{artifactsPaths.PackagesDirectoryName}\"") .AppendArgument(GetMsBuildBinLogArgument(buildPartition, binLogSuffix)) - .AppendArgument($"--output \"{artifactsPaths.BinariesDirectoryPath}\"") + .MaybeAppendOutputPaths(artifactsPaths) .ToString(); private static string GetMsBuildBinLogArgument(BuildPartition buildPartition, string suffix) @@ -262,4 +250,22 @@ private static string BuildAddPackageCommand(NuGetReference reference) return commandBuilder.ToString(); } } + + internal static class DotNetCliCommandExtensions + { + // Fix #1377 (see comments in #1773). + // We force the project to output binaries to a new directory. + // Specifying --output and --no-dependencies breaks the build (because the previous build was not done using the custom output path), + // so we don't include it if we're building no-deps (only supported for integration tests). + internal static StringBuilder MaybeAppendOutputPaths(this StringBuilder stringBuilder, ArtifactsPaths artifactsPaths, bool isRestore = false, bool excludeOutput = false) + => excludeOutput + ? stringBuilder + : stringBuilder + // Use AltDirectorySeparatorChar so it's not interpreted as an escaped quote `\"`. + .AppendArgument($"/p:IntermediateOutputPath=\"{artifactsPaths.IntermediateDirectoryPath}{Path.AltDirectorySeparatorChar}\"") + .AppendArgument($"/p:OutDir=\"{artifactsPaths.BinariesDirectoryPath}{Path.AltDirectorySeparatorChar}\"") + // OutputPath is legacy, per-project version of OutDir. We set both just in case. https://github.com/dotnet/msbuild/issues/87 + .AppendArgument($"/p:OutputPath=\"{artifactsPaths.BinariesDirectoryPath}{Path.AltDirectorySeparatorChar}\"") + .AppendArgument(isRestore ? string.Empty : $"--output \"{artifactsPaths.BinariesDirectoryPath}{Path.AltDirectorySeparatorChar}\""); + } } diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs index bfe9a97585..5cbff89b47 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; @@ -54,9 +55,9 @@ public static DotNetCliCommandResult Execute(DotNetCliCommand parameters) } } - internal static string GetDotNetSdkVersion() + internal static string? GetDotNetSdkVersion() { - using (var process = new Process { StartInfo = BuildStartInfo(customDotNetCliPath: null, workingDirectory: string.Empty, arguments: "--version") }) + using (var process = new Process { StartInfo = BuildStartInfo(customDotNetCliPath: null, workingDirectory: string.Empty, arguments: "--version", redirectStandardError: false) }) using (new ConsoleExitHandler(process, NullLogger.Instance)) { try @@ -98,8 +99,8 @@ internal static void LogEnvVars(DotNetCliCommand command) } } - internal static ProcessStartInfo BuildStartInfo(string customDotNetCliPath, string workingDirectory, string arguments, - IReadOnlyList environmentVariables = null, bool redirectStandardInput = false, bool redirectStandardError = true, bool redirectStandardOutput = true) + internal static ProcessStartInfo BuildStartInfo(string? customDotNetCliPath, string workingDirectory, string arguments, + IReadOnlyList? environmentVariables = null, bool redirectStandardInput = false, bool redirectStandardError = true, bool redirectStandardOutput = true) { const string dotnetMultiLevelLookupEnvVarName = "DOTNET_MULTILEVEL_LOOKUP"; @@ -137,7 +138,7 @@ internal static ProcessStartInfo BuildStartInfo(string customDotNetCliPath, stri private static string GetDefaultDotNetCliPath() { - if (!Portability.RuntimeInformation.IsLinux()) + if (!OsDetector.IsLinux()) return "dotnet"; using (var parentProcess = Process.GetProcessById(libc.getppid())) diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandResult.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandResult.cs index 3d65270621..4a5b00a1fd 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandResult.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandResult.cs @@ -35,7 +35,7 @@ public static DotNetCliCommandResult Failure(TimeSpan time, string standardError [PublicAPI] public BuildResult ToBuildResult(GenerateResult generateResult) - => IsSuccess || File.Exists(generateResult.ArtifactsPaths.ExecutablePath) // dotnet cli could have successfully built the program, but returned 1 as exit code because it had some warnings + => IsSuccess ? BuildResult.Success(generateResult) : BuildResult.Failure(generateResult, AllInformation); } diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs index 55815ed005..f8bea8236e 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs @@ -83,9 +83,12 @@ private ExecuteResult Execute(BenchmarkCase benchmarkCase, logger.WriteLineInfo($"// Execute: {process.StartInfo.FileName} {process.StartInfo.Arguments} in {process.StartInfo.WorkingDirectory}"); - diagnoser?.Handle(HostSignal.BeforeProcessStart, new DiagnoserActionParameters(process, benchmarkCase, benchmarkId)); + diagnoser?.Handle(HostSignal.BeforeProcessStart, broker.DiagnoserActionParameters); process.Start(); + + diagnoser?.Handle(HostSignal.AfterProcessStart, broker.DiagnoserActionParameters); + processOutputReader.BeginRead(); process.EnsureHighPriority(logger); diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs index 47ff074fc3..a432fbf881 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs @@ -61,17 +61,25 @@ internal static bool GetProjectRootDirectory(out DirectoryInfo directoryInfo) return GetRootDirectory(IsRootProjectFolder, out directoryInfo); } - internal static bool GetRootDirectory(Func condition, out DirectoryInfo directoryInfo) + internal static bool GetRootDirectory(Func condition, out DirectoryInfo? directoryInfo) { - directoryInfo = new DirectoryInfo(Directory.GetCurrentDirectory()); - while (directoryInfo != null) + directoryInfo = null; + try { - if (condition(directoryInfo)) + directoryInfo = new DirectoryInfo(Directory.GetCurrentDirectory()); + while (directoryInfo != null) { - return true; - } + if (condition(directoryInfo)) + { + return true; + } - directoryInfo = directoryInfo.Parent; + directoryInfo = directoryInfo.Parent; + } + } + catch + { + return false; } return false; diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliPublisher.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliPublisher.cs index 8292619e49..83158e0b1e 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliPublisher.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliPublisher.cs @@ -8,18 +8,21 @@ namespace BenchmarkDotNet.Toolchains.DotNetCli { public class DotNetCliPublisher : IBuilder { - public DotNetCliPublisher(string customDotNetCliPath = null, string extraArguments = null, IReadOnlyList environmentVariables = null) + public DotNetCliPublisher( + string? customDotNetCliPath = null, + string? extraArguments = null, + IReadOnlyList? environmentVariables = null) { CustomDotNetCliPath = customDotNetCliPath; ExtraArguments = extraArguments; EnvironmentVariables = environmentVariables; } - private string CustomDotNetCliPath { get; } + private string? CustomDotNetCliPath { get; } - private string ExtraArguments { get; } + private string? ExtraArguments { get; } - private IReadOnlyList EnvironmentVariables { get; } + private IReadOnlyList? EnvironmentVariables { get; } public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger) => new DotNetCliCommand( diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/MsBuildErrorMapper.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/MsBuildErrorMapper.cs index 6f0e28834b..9c95be192f 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/MsBuildErrorMapper.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/MsBuildErrorMapper.cs @@ -30,7 +30,7 @@ internal static class MsBuildErrorMapper ), }; - internal static bool TryToExplainFailureReason(BuildResult buildResult, out string reason) + internal static bool TryToExplainFailureReason(BuildResult buildResult, out string? reason) { reason = null; @@ -69,6 +69,8 @@ private static string Map(Capture capture) return "net472"; case ".NETFramework,Version=v4.8": return "net48"; + case ".NETFramework,Version=v4.8.1": + return "net481"; case ".NETCoreApp,Version=v2.0": return "netcoreapp2.0"; case ".NETCoreApp,Version=v2.1": @@ -83,6 +85,14 @@ private static string Map(Capture capture) return "net5.0"; case ".NETCoreApp,Version=v6.0": return "net6.0"; + case ".NETCoreApp,Version=v7.0": + return "net7.0"; + case ".NETCoreApp,Version=v8.0": + return "net8.0"; + case ".NETCoreApp,Version=v9.0": + return "net9.0"; + case ".NETCoreApp,Version=v10.0": + return "net10.0"; default: return capture.Value; // we don't want to throw for future versions of .NET } diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs index 50b1f7b614..028e4f2bd3 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs @@ -18,10 +18,12 @@ public class NetCoreAppSettings [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp60 = new ("net6.0", null, ".NET 6.0"); [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp70 = new ("net7.0", null, ".NET 7.0"); [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp80 = new ("net8.0", null, ".NET 8.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp90 = new ("net9.0", null, ".NET 9.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp10_0 = new ("net10.0", null, ".NET 10.0"); /// /// - /// sample values: netcoreapp2.0, netcoreapp2.1 + /// sample values: net6.0, net8.0 /// /// /// used in the auto-generated .csproj file @@ -42,12 +44,12 @@ public class NetCoreAppSettings [PublicAPI] public NetCoreAppSettings( string targetFrameworkMoniker, - string runtimeFrameworkVersion, + string? runtimeFrameworkVersion, string name, - string customDotNetCliPath = null, - string packagesPath = null, - string customRuntimePack = null, - string aotCompilerPath = null, + string? customDotNetCliPath = null, + string? packagesPath = null, + string? customRuntimePack = null, + string? aotCompilerPath = null, MonoAotCompilerMode aotCompilerMode = MonoAotCompilerMode.mini ) { @@ -63,43 +65,43 @@ public NetCoreAppSettings( } /// - /// sample values: netcoreapp2.0, netcoreapp2.1 + /// sample values: net6.0, net8.0 /// public string TargetFrameworkMoniker { get; } - public string RuntimeFrameworkVersion { get; } + public string? RuntimeFrameworkVersion { get; } /// /// display name used for showing the results /// public string Name { get; } - public string CustomDotNetCliPath { get; } + public string? CustomDotNetCliPath { get; } /// /// The directory to restore packages to. /// - public string PackagesPath { get; } + public string? PackagesPath { get; } /// /// Path to a custom runtime pack. /// - public string CustomRuntimePack { get; } + public string? CustomRuntimePack { get; } /// /// Path to the Mono AOT Compiler /// - public string AOTCompilerPath { get; } + public string? AOTCompilerPath { get; } /// /// Mono AOT Compiler mode, either 'mini' or 'llvm' /// public MonoAotCompilerMode AOTCompilerMode { get; } - public NetCoreAppSettings WithCustomDotNetCliPath(string customDotNetCliPath, string displayName = null) + public NetCoreAppSettings WithCustomDotNetCliPath(string customDotNetCliPath, string? displayName = null) => new NetCoreAppSettings(TargetFrameworkMoniker, RuntimeFrameworkVersion, displayName ?? Name, customDotNetCliPath, PackagesPath); - public NetCoreAppSettings WithCustomPackagesRestorePath(string packagesPath, string displayName = null) + public NetCoreAppSettings WithCustomPackagesRestorePath(string packagesPath, string? displayName = null) => new NetCoreAppSettings(TargetFrameworkMoniker, RuntimeFrameworkVersion, displayName ?? Name, CustomDotNetCliPath, packagesPath); } } diff --git a/src/BenchmarkDotNet/Toolchains/Executor.cs b/src/BenchmarkDotNet/Toolchains/Executor.cs index 735c6f7437..8fbce03516 100644 --- a/src/BenchmarkDotNet/Toolchains/Executor.cs +++ b/src/BenchmarkDotNet/Toolchains/Executor.cs @@ -79,6 +79,8 @@ private static ExecuteResult Execute(Process process, BenchmarkCase benchmarkCas return new ExecuteResult(true, null, null, Array.Empty(), Array.Empty(), Array.Empty(), launchIndex); } + broker.Diagnoser?.Handle(HostSignal.AfterProcessStart, broker.DiagnoserActionParameters); + processOutputReader.BeginRead(); process.EnsureHighPriority(logger); @@ -146,7 +148,7 @@ private static ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, Art case MonoAotLLVMRuntime _: start.FileName = exePath; start.Arguments = args; - start.WorkingDirectory = artifactsPaths.BinariesDirectoryPath; + start.WorkingDirectory = Path.Combine(artifactsPaths.BinariesDirectoryPath, "publish"); break; case CustomRuntime _: start.FileName = exePath; diff --git a/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs b/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs index 0d7ab30d7d..020788cc56 100644 --- a/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs +++ b/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs @@ -2,6 +2,8 @@ using System.IO; using System.Text; using BenchmarkDotNet.Code; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; @@ -43,16 +45,23 @@ public GenerateResult GenerateProject(BuildPartition buildPartition, ILogger log [PublicAPI] protected abstract string GetBuildArtifactsDirectoryPath(BuildPartition assemblyLocation, string programName); /// - /// returns a path where executable should be found after the build + /// returns a path where executable should be found after the build (usually \bin) /// [PublicAPI] protected virtual string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) => buildArtifactsDirectoryPath; + /// + /// returns a path where intermediate files should be found after the build (usually \obj) + /// + [PublicAPI] + protected virtual string GetIntermediateDirectoryPath(string buildArtifactsDirectoryPath, string configuration) + => string.Empty; + /// /// returns OS-specific executable extension /// [PublicAPI] protected virtual string GetExecutableExtension() - => RuntimeInformation.ExecutableExtension; + => OsDetector.ExecutableExtension; /// /// returns a path to the auto-generated .csproj file @@ -96,6 +105,7 @@ [PublicAPI] protected virtual void GenerateProject(BuildPartition buildPartition [PublicAPI] protected virtual void GenerateAppConfig(BuildPartition buildPartition, ArtifactsPaths artifactsPaths) { string sourcePath = buildPartition.AssemblyLocation + ".config"; + artifactsPaths.AppConfigPath.EnsureFolderExists(); using (var source = File.Exists(sourcePath) ? new StreamReader(File.OpenRead(sourcePath)) : TextReader.Null) using (var destination = new StreamWriter(File.Create(artifactsPaths.AppConfigPath), Encoding.UTF8)) @@ -128,11 +138,12 @@ private ArtifactsPaths GetArtifactsPaths(BuildPartition buildPartition, string r rootArtifactsFolderPath: rootArtifactsFolderPath, buildArtifactsDirectoryPath: buildArtifactsDirectoryPath, binariesDirectoryPath: binariesDirectoryPath, + intermediateDirectoryPath: GetIntermediateDirectoryPath(buildArtifactsDirectoryPath, buildPartition.BuildConfiguration), programCodePath: Path.Combine(buildArtifactsDirectoryPath, $"{programName}{codeFileExtension}"), appConfigPath: $"{executablePath}.config", nuGetConfigPath: Path.Combine(buildArtifactsDirectoryPath, "NuGet.config"), projectFilePath: GetProjectFilePath(buildArtifactsDirectoryPath), - buildScriptFilePath: Path.Combine(buildArtifactsDirectoryPath, $"{programName}{RuntimeInformation.ScriptFileExtension}"), + buildScriptFilePath: Path.Combine(buildArtifactsDirectoryPath, $"{programName}{OsDetector.ScriptFileExtension}"), executablePath: executablePath, programName: programName, packagesDirectoryName: GetPackagesDirectoryPath(buildArtifactsDirectoryPath)); diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/ConsumableTypeInfo.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/ConsumableTypeInfo.cs index f8d344b1c2..147a514058 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/ConsumableTypeInfo.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/ConsumableTypeInfo.cs @@ -1,5 +1,7 @@ using BenchmarkDotNet.Engines; using System; +using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; using System.Threading.Tasks; @@ -16,28 +18,24 @@ public ConsumableTypeInfo(Type methodReturnType) OriginMethodReturnType = methodReturnType; - // Please note this code does not support await over extension methods. - var getAwaiterMethod = methodReturnType.GetMethod(nameof(Task.GetAwaiter), BindingFlagsPublicInstance); - if (getAwaiterMethod == null) + // Only support (Value)Task for parity with other toolchains (and so we can use AwaitHelper). + IsAwaitable = methodReturnType == typeof(Task) || methodReturnType == typeof(ValueTask) + || (methodReturnType.GetTypeInfo().IsGenericType + && (methodReturnType.GetTypeInfo().GetGenericTypeDefinition() == typeof(Task<>) + || methodReturnType.GetTypeInfo().GetGenericTypeDefinition() == typeof(ValueTask<>))); + + if (!IsAwaitable) { WorkloadMethodReturnType = methodReturnType; } else { - var getResultMethod = getAwaiterMethod + WorkloadMethodReturnType = methodReturnType + .GetMethod(nameof(Task.GetAwaiter), BindingFlagsPublicInstance) .ReturnType - .GetMethod(nameof(TaskAwaiter.GetResult), BindingFlagsPublicInstance); - - if (getResultMethod == null) - { - WorkloadMethodReturnType = methodReturnType; - } - else - { - WorkloadMethodReturnType = getResultMethod.ReturnType; - GetAwaiterMethod = getAwaiterMethod; - GetResultMethod = getResultMethod; - } + .GetMethod(nameof(TaskAwaiter.GetResult), BindingFlagsPublicInstance) + .ReturnType; + GetResultMethod = Helpers.AwaitHelper.GetGetResultMethod(methodReturnType); } if (WorkloadMethodReturnType == null) @@ -74,7 +72,6 @@ public ConsumableTypeInfo(Type methodReturnType) public Type WorkloadMethodReturnType { get; } public Type OverheadMethodReturnType { get; } - public MethodInfo? GetAwaiterMethod { get; } public MethodInfo? GetResultMethod { get; } public bool IsVoid { get; } @@ -82,6 +79,6 @@ public ConsumableTypeInfo(Type methodReturnType) public bool IsConsumable { get; } public FieldInfo? WorkloadConsumableField { get; } - public bool IsAwaitable => GetAwaiterMethod != null && GetResultMethod != null; + public bool IsAwaitable { get; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/ConsumeEmitter.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/ConsumeEmitter.cs index 62fe06c649..9767cb8263 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/ConsumeEmitter.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/ConsumeEmitter.cs @@ -31,9 +31,9 @@ protected ConsumeEmitter(ConsumableTypeInfo consumableTypeInfo) protected ConsumableTypeInfo ConsumableInfo { get; } - protected ILGenerator IlBuilder { get; private set; } - protected MethodBuilder ActionMethodBuilder { get; private set; } - protected MethodInfo ActionInvokeMethod { get; private set; } + protected ILGenerator? IlBuilder { get; private set; } + protected MethodBuilder? ActionMethodBuilder { get; private set; } + protected MethodInfo? ActionInvokeMethod { get; private set; } protected RunnableActionKind? ActionKind { get; private set; } [AssertionMethod] diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs index c6c6d8e05c..0e9d386a24 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs @@ -247,6 +247,10 @@ private static void EmitNoArgsMethodCallPopReturn( private TypeBuilder runnableBuilder; private ConsumableTypeInfo consumableInfo; private ConsumeEmitter consumeEmitter; + private ConsumableTypeInfo globalSetupReturnInfo; + private ConsumableTypeInfo globalCleanupReturnInfo; + private ConsumableTypeInfo iterationSetupReturnInfo; + private ConsumableTypeInfo iterationCleanupReturnInfo; private FieldBuilder globalSetupActionField; private FieldBuilder globalCleanupActionField; @@ -358,6 +362,10 @@ private void InitForEmitRunnable(BenchmarkBuildInfo newBenchmark) consumableInfo = new ConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.WorkloadMethod.ReturnType); consumeEmitter = ConsumeEmitter.GetConsumeEmitter(consumableInfo); + globalSetupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.GlobalSetupMethod?.ReturnType); + globalCleanupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.GlobalCleanupMethod?.ReturnType); + iterationSetupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.IterationSetupMethod?.ReturnType); + iterationCleanupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.IterationCleanupMethod?.ReturnType); // Init types runnableBuilder = DefineRunnableTypeBuilder(benchmark, moduleBuilder); @@ -365,6 +373,11 @@ private void InitForEmitRunnable(BenchmarkBuildInfo newBenchmark) workloadDelegateType = EmitWorkloadDelegateType(); } + private static ConsumableTypeInfo GetConsumableTypeInfo(Type methodReturnType) + { + return methodReturnType == null ? null : new ConsumableTypeInfo(methodReturnType); + } + private Type EmitOverheadDelegateType() { // .class public auto ansi sealed BenchmarkDotNet.Autogenerated.Runnable_0OverheadDelegate @@ -582,42 +595,28 @@ private MethodInfo EmitWorkloadImplementation(string methodName) workloadInvokeMethod.ReturnParameter, args); args = methodBuilder.GetEmitParameters(args); - var callResultType = consumableInfo.OriginMethodReturnType; - var awaiterType = consumableInfo.GetAwaiterMethod?.ReturnType - ?? throw new InvalidOperationException($"Bug: {nameof(consumableInfo.GetAwaiterMethod)} is null"); var ilBuilder = methodBuilder.GetILGenerator(); /* - .locals init ( - [0] valuetype [mscorlib]System.Runtime.CompilerServices.TaskAwaiter`1 - ) - */ - var callResultLocal = - ilBuilder.DeclareOptionalLocalForInstanceCall(callResultType, consumableInfo.GetAwaiterMethod); - var awaiterLocal = - ilBuilder.DeclareOptionalLocalForInstanceCall(awaiterType, consumableInfo.GetResultMethod); - - /* - // return TaskSample(arg0). ... ; - IL_0000: ldarg.0 - IL_0001: ldarg.1 - IL_0002: call instance class [mscorlib]System.Threading.Tasks.Task`1 [BenchmarkDotNet]BenchmarkDotNet.Samples.SampleBenchmark::TaskSample(int64) - */ + IL_0026: ldarg.0 + IL_0027: ldloc.0 + IL_0028: ldloc.1 + IL_0029: ldloc.2 + IL_002a: ldloc.3 + IL_002b: call instance class [System.Private.CoreLib]System.Threading.Tasks.Task`1 BenchmarkDotNet.Helpers.Runnable_0::WorkloadMethod(string, string, string, string) + */ if (!Descriptor.WorkloadMethod.IsStatic) ilBuilder.Emit(OpCodes.Ldarg_0); ilBuilder.EmitLdargs(args); ilBuilder.Emit(OpCodes.Call, Descriptor.WorkloadMethod); /* - // ... .GetAwaiter().GetResult(); - IL_0007: callvirt instance valuetype [mscorlib]System.Runtime.CompilerServices.TaskAwaiter`1 class [mscorlib]System.Threading.Tasks.Task`1::GetAwaiter() - IL_000c: stloc.0 - IL_000d: ldloca.s 0 - IL_000f: call instance !0 valuetype [mscorlib]System.Runtime.CompilerServices.TaskAwaiter`1::GetResult() - */ - ilBuilder.EmitInstanceCallThisValueOnStack(callResultLocal, consumableInfo.GetAwaiterMethod); - ilBuilder.EmitInstanceCallThisValueOnStack(awaiterLocal, consumableInfo.GetResultMethod); + // BenchmarkDotNet.Helpers.AwaitHelper.GetResult(...); + IL_000e: call !!0 BenchmarkDotNet.Helpers.AwaitHelper::GetResult(valuetype [System.Runtime]System.Threading.Tasks.ValueTask`1) + */ + + ilBuilder.Emit(OpCodes.Call, consumableInfo.GetResultMethod); /* IL_0014: ret @@ -833,19 +832,6 @@ .locals init ( var skipFirstArg = workloadMethod.IsStatic; var argLocals = EmitDeclareArgLocals(ilBuilder, skipFirstArg); - LocalBuilder callResultLocal = null; - LocalBuilder awaiterLocal = null; - if (consumableInfo.IsAwaitable) - { - var callResultType = consumableInfo.OriginMethodReturnType; - var awaiterType = consumableInfo.GetAwaiterMethod?.ReturnType - ?? throw new InvalidOperationException($"Bug: {nameof(consumableInfo.GetAwaiterMethod)} is null"); - callResultLocal = - ilBuilder.DeclareOptionalLocalForInstanceCall(callResultType, consumableInfo.GetAwaiterMethod); - awaiterLocal = - ilBuilder.DeclareOptionalLocalForInstanceCall(awaiterType, consumableInfo.GetResultMethod); - } - consumeEmitter.DeclareDisassemblyDiagnoserLocals(ilBuilder); var notElevenLabel = ilBuilder.DefineLabel(); @@ -870,29 +856,27 @@ .locals init ( EmitLoadArgFieldsToLocals(ilBuilder, argLocals, skipFirstArg); /* - // return TaskSample(_argField) ... ; - IL_0011: ldarg.0 - IL_0012: ldloc.0 - IL_0013: call instance class [mscorlib]System.Threading.Tasks.Task`1 [BenchmarkDotNet]BenchmarkDotNet.Samples.SampleBenchmark::TaskSample(int64) - IL_0018: ret + IL_0026: ldarg.0 + IL_0027: ldloc.0 + IL_0028: ldloc.1 + IL_0029: ldloc.2 + IL_002a: ldloc.3 + IL_002b: call instance class [System.Private.CoreLib]System.Threading.Tasks.Task`1 BenchmarkDotNet.Helpers.Runnable_0::WorkloadMethod(string, string, string, string) */ - if (!workloadMethod.IsStatic) + { ilBuilder.Emit(OpCodes.Ldarg_0); + } ilBuilder.EmitLdLocals(argLocals); ilBuilder.Emit(OpCodes.Call, workloadMethod); if (consumableInfo.IsAwaitable) { /* - // ... .GetAwaiter().GetResult(); - IL_0007: callvirt instance valuetype [mscorlib]System.Runtime.CompilerServices.TaskAwaiter`1 class [mscorlib]System.Threading.Tasks.Task`1::GetAwaiter() - IL_000c: stloc.0 - IL_000d: ldloca.s 0 - IL_000f: call instance !0 valuetype [mscorlib]System.Runtime.CompilerServices.TaskAwaiter`1::GetResult() - */ - ilBuilder.EmitInstanceCallThisValueOnStack(callResultLocal, consumableInfo.GetAwaiterMethod); - ilBuilder.EmitInstanceCallThisValueOnStack(awaiterLocal, consumableInfo.GetResultMethod); + // BenchmarkDotNet.Helpers.AwaitHelper.GetResult(...); + IL_000e: call !!0 BenchmarkDotNet.Helpers.AwaitHelper::GetResult(valuetype [System.Runtime]System.Threading.Tasks.ValueTask`1) + */ + ilBuilder.Emit(OpCodes.Call, consumableInfo.GetResultMethod); } /* @@ -919,34 +903,84 @@ private void EmitSetupCleanupMethods() { // Emit Setup/Cleanup methods // We emit empty method instead of EmptyAction = "() => { }" - globalSetupMethod = EmitWrapperMethod( - GlobalSetupMethodName, - Descriptor.GlobalSetupMethod); - globalCleanupMethod = EmitWrapperMethod( - GlobalCleanupMethodName, - Descriptor.GlobalCleanupMethod); - iterationSetupMethod = EmitWrapperMethod( - IterationSetupMethodName, - Descriptor.IterationSetupMethod); - iterationCleanupMethod = EmitWrapperMethod( - IterationCleanupMethodName, - Descriptor.IterationCleanupMethod); + globalSetupMethod = EmitWrapperMethod(GlobalSetupMethodName, Descriptor.GlobalSetupMethod, globalSetupReturnInfo); + globalCleanupMethod = EmitWrapperMethod(GlobalCleanupMethodName, Descriptor.GlobalCleanupMethod, globalCleanupReturnInfo); + iterationSetupMethod = EmitWrapperMethod(IterationSetupMethodName, Descriptor.IterationSetupMethod, iterationSetupReturnInfo); + iterationCleanupMethod = EmitWrapperMethod(IterationCleanupMethodName, Descriptor.IterationCleanupMethod, iterationCleanupReturnInfo); } - private MethodBuilder EmitWrapperMethod(string methodName, MethodInfo optionalTargetMethod) + private MethodBuilder EmitWrapperMethod(string methodName, MethodInfo optionalTargetMethod, ConsumableTypeInfo returnTypeInfo) { var methodBuilder = runnableBuilder.DefinePrivateVoidInstanceMethod(methodName); var ilBuilder = methodBuilder.GetILGenerator(); if (optionalTargetMethod != null) - EmitNoArgsMethodCallPopReturn(methodBuilder, optionalTargetMethod, ilBuilder, forceDirectCall: true); + { + if (returnTypeInfo?.IsAwaitable == true) + { + EmitAwaitableSetupTeardown(methodBuilder, optionalTargetMethod, ilBuilder, returnTypeInfo); + } + else + { + EmitNoArgsMethodCallPopReturn(methodBuilder, optionalTargetMethod, ilBuilder, forceDirectCall: true); + } + } ilBuilder.EmitVoidReturn(methodBuilder); return methodBuilder; } + private void EmitAwaitableSetupTeardown( + MethodBuilder methodBuilder, + MethodInfo targetMethod, + ILGenerator ilBuilder, + ConsumableTypeInfo returnTypeInfo) + { + if (targetMethod == null) + throw new ArgumentNullException(nameof(targetMethod)); + + if (returnTypeInfo.WorkloadMethodReturnType == typeof(void)) + { + ilBuilder.Emit(OpCodes.Ldarg_0); + } + /* + // call for instance + // GlobalSetup(); + IL_0006: ldarg.0 + IL_0007: call instance void [BenchmarkDotNet]BenchmarkDotNet.Samples.SampleBenchmark::GlobalSetup() + */ + /* + // call for static + // GlobalSetup(); + IL_0006: call string [BenchmarkDotNet]BenchmarkDotNet.Samples.SampleBenchmark::GlobalCleanup() + */ + if (targetMethod.IsStatic) + { + ilBuilder.Emit(OpCodes.Call, targetMethod); + + } + else if (methodBuilder.IsStatic) + { + throw new InvalidOperationException( + $"[BUG] Static method {methodBuilder.Name} tries to call instance member {targetMethod.Name}"); + } + else + { + ilBuilder.Emit(OpCodes.Ldarg_0); + ilBuilder.Emit(OpCodes.Call, targetMethod); + } + + /* + // BenchmarkDotNet.Helpers.AwaitHelper.GetResult(...); + IL_000e: call !!0 BenchmarkDotNet.Helpers.AwaitHelper::GetResult(valuetype [System.Runtime]System.Threading.Tasks.ValueTask`1) + */ + + ilBuilder.Emit(OpCodes.Call, returnTypeInfo.GetResultMethod); + ilBuilder.Emit(OpCodes.Pop); + } + private void EmitCtorBody() { var ilBuilder = ctorMethod.GetILGenerator(); diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitArtifactsPath.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitArtifactsPath.cs index c2b3199d93..0686a322ac 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitArtifactsPath.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitArtifactsPath.cs @@ -12,6 +12,7 @@ public InProcessEmitArtifactsPath( baseArtifacts.RootArtifactsFolderPath, baseArtifacts.BuildArtifactsDirectoryPath, baseArtifacts.BinariesDirectoryPath, + baseArtifacts.IntermediateDirectoryPath, baseArtifacts.ProgramCodePath, baseArtifacts.AppConfigPath, baseArtifacts.NuGetConfigPath, diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitBuilder.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitBuilder.cs index f46f5c99b6..7a75eb3263 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitBuilder.cs @@ -11,8 +11,8 @@ public class InProcessEmitBuilder : IBuilder { public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger) { - Assembly assembly = null; - Exception buildError = null; + Assembly? assembly = null; + Exception? buildError = null; try { assembly = RunnableEmitter.EmitPartitionAssembly(generateResult, buildPartition, logger); diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs index 64d2019a75..60d6f8a6ae 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Reflection; using System.Threading; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; @@ -59,7 +60,7 @@ public ExecuteResult Execute(ExecuteParameters executeParameters) if (executeParameters.BenchmarkCase.Descriptor.WorkloadMethod .GetCustomAttributes(false) .Any() && - Portability.RuntimeInformation.IsWindows()) + OsDetector.IsWindows()) { runThread.SetApartmentState(ApartmentState.STA); } diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitGenerator.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitGenerator.cs index c169c91903..18734fcf46 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitGenerator.cs @@ -47,6 +47,7 @@ private ArtifactsPaths GetArtifactsPaths(BuildPartition buildPartition, string r rootArtifactsFolderPath: rootArtifactsFolderPath, buildArtifactsDirectoryPath: buildArtifactsDirectoryPath, binariesDirectoryPath: binariesDirectoryPath, + intermediateDirectoryPath: null, programCodePath: null, appConfigPath: null, nuGetConfigPath: null, diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/BenchmarkActionFactory_Implementations.cs b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/BenchmarkActionFactory_Implementations.cs index 551e3001c9..aee0a8f998 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/BenchmarkActionFactory_Implementations.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/BenchmarkActionFactory_Implementations.cs @@ -118,7 +118,7 @@ public BenchmarkActionTask(object instance, MethodInfo method, int unrollFactor) private void Overhead() { } // must be kept in sync with TaskDeclarationsProvider.TargetMethodDelegate - private void ExecuteBlocking() => startTaskCallback.Invoke().GetAwaiter().GetResult(); + private void ExecuteBlocking() => Helpers.AwaitHelper.GetResult(startTaskCallback.Invoke()); [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] private void WorkloadActionUnroll(long repeatCount) @@ -165,7 +165,7 @@ public BenchmarkActionTask(object instance, MethodInfo method, int unrollFactor) private T Overhead() => default; // must be kept in sync with GenericTaskDeclarationsProvider.TargetMethodDelegate - private T ExecuteBlocking() => startTaskCallback().GetAwaiter().GetResult(); + private T ExecuteBlocking() => Helpers.AwaitHelper.GetResult(startTaskCallback.Invoke()); private void InvokeSingleHardcoded() => result = callback(); @@ -217,7 +217,7 @@ public BenchmarkActionValueTask(object instance, MethodInfo method, int unrollFa private T Overhead() => default; // must be kept in sync with GenericTaskDeclarationsProvider.TargetMethodDelegate - private T ExecuteBlocking() => startTaskCallback().GetAwaiter().GetResult(); + private T ExecuteBlocking() => Helpers.AwaitHelper.GetResult(startTaskCallback.Invoke()); private void InvokeSingleHardcoded() => result = callback(); diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs index 49cd24e3d4..b50ba77c1c 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Reflection; using System.Threading; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; @@ -59,7 +60,7 @@ public ExecuteResult Execute(ExecuteParameters executeParameters) var runThread = new Thread(() => exitCode = ExecuteCore(host, executeParameters)); if (executeParameters.BenchmarkCase.Descriptor.WorkloadMethod.GetCustomAttributes(false).Any() && - Portability.RuntimeInformation.IsWindows()) + OsDetector.IsWindows()) { runThread.SetApartmentState(ApartmentState.STA); } diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs index a52e7ebcf8..18ee40c30a 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs @@ -1,4 +1,5 @@ -using System; +using System; +using System.Diagnostics.CodeAnalysis; using System.Reflection; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; @@ -15,6 +16,9 @@ namespace BenchmarkDotNet.Toolchains.InProcess.NoEmit /// internal class InProcessNoEmitRunner { +#if NET6_0_OR_GREATER + [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Runnable))] +#endif public static int Run(IHost host, BenchmarkCase benchmarkCase) { // the first thing to do is to let diagnosers hook in before anything happens @@ -157,4 +161,4 @@ public static void RunCore(IHost host, BenchmarkCase benchmarkCase) } } } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs index ba2ae2ac96..222eb2111b 100644 --- a/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs @@ -11,6 +11,8 @@ public class MonoToolchain : CsProjCoreToolchain, IEquatable [PublicAPI] public static readonly IToolchain Mono60 = From(new NetCoreAppSettings("net6.0", null, "mono60")); [PublicAPI] public static readonly IToolchain Mono70 = From(new NetCoreAppSettings("net7.0", null, "mono70")); [PublicAPI] public static readonly IToolchain Mono80 = From(new NetCoreAppSettings("net8.0", null, "mono80")); + [PublicAPI] public static readonly IToolchain Mono90 = From(new NetCoreAppSettings("net9.0", null, "mono90")); + [PublicAPI] public static readonly IToolchain Mono10_0 = From(new NetCoreAppSettings("net10.0", null, "mono10_0")); private MonoToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) : base(name, generator, builder, executor, customDotNetCliPath) diff --git a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs index d17fad7d00..0816ac48f0 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs @@ -1,6 +1,7 @@ using System.IO; using System.Text; using System.Xml; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; @@ -42,7 +43,6 @@ protected override void GenerateProject(BuildPartition buildPartition, Artifacts .Replace("$TFM$", TargetFrameworkMoniker) .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) .Replace("$COPIEDSETTINGS$", customProperties) - .Replace("$CONFIGURATIONNAME$", buildPartition.BuildConfiguration) .Replace("$SDKNAME$", sdkName) .Replace("$RUNTIMEPACK$", CustomRuntimePack ?? "") .Replace("$COMPILERBINARYPATH$", AotCompilerPath) @@ -54,11 +54,11 @@ protected override void GenerateProject(BuildPartition buildPartition, Artifacts } protected override string GetExecutablePath(string binariesDirectoryPath, string programName) - => Portability.RuntimeInformation.IsWindows() - ? Path.Combine(binariesDirectoryPath, $"{programName}.exe") - : Path.Combine(binariesDirectoryPath, programName); + => OsDetector.IsWindows() + ? Path.Combine(binariesDirectoryPath, "publish", $"{programName}.exe") + : Path.Combine(binariesDirectoryPath, "publish", programName); protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => Path.Combine(buildArtifactsDirectoryPath, "bin", TargetFrameworkMoniker, CustomDotNetCliToolchainBuilder.GetPortableRuntimeIdentifier(), "publish"); + => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, TargetFrameworkMoniker, CustomDotNetCliToolchainBuilder.GetPortableRuntimeIdentifier()); } } diff --git a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs index 0d77160e06..06ef8ea8c6 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs @@ -1,24 +1,45 @@ +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Validators; +using System.Collections.Generic; namespace BenchmarkDotNet.Toolchains.MonoAotLLVM { public class MonoAotLLVMToolChain : Toolchain { - public MonoAotLLVMToolChain(string name, IGenerator generator, IBuilder builder, IExecutor executor) + private readonly string _customDotNetCliPath; + + public MonoAotLLVMToolChain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) : base(name, generator, builder, executor) { + _customDotNetCliPath = customDotNetCliPath; } public static IToolchain From(NetCoreAppSettings netCoreAppSettings) - => new MonoAotLLVMToolChain(netCoreAppSettings.Name, - new MonoAotLLVMGenerator(netCoreAppSettings.TargetFrameworkMoniker, - netCoreAppSettings.CustomDotNetCliPath, - netCoreAppSettings.PackagesPath, - netCoreAppSettings.CustomRuntimePack, - netCoreAppSettings.AOTCompilerPath, - netCoreAppSettings.AOTCompilerMode), - new DotNetCliBuilder(netCoreAppSettings.TargetFrameworkMoniker, - netCoreAppSettings.CustomDotNetCliPath), - new Executor()); + => new MonoAotLLVMToolChain(netCoreAppSettings.Name, + new MonoAotLLVMGenerator(netCoreAppSettings.TargetFrameworkMoniker, + netCoreAppSettings.CustomDotNetCliPath, + netCoreAppSettings.PackagesPath, + netCoreAppSettings.CustomRuntimePack, + netCoreAppSettings.AOTCompilerPath, + netCoreAppSettings.AOTCompilerMode), + new DotNetCliBuilder(netCoreAppSettings.TargetFrameworkMoniker, + netCoreAppSettings.CustomDotNetCliPath), + new Executor(), + netCoreAppSettings.CustomDotNetCliPath); + + public override IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) + { + foreach (var validationError in base.Validate(benchmarkCase, resolver)) + { + yield return validationError; + } + + foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(_customDotNetCliPath, benchmarkCase)) + { + yield return validationError; + } + } } } diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs index 024c8e4242..7e857ff364 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs @@ -61,7 +61,7 @@ private static Process CreateProcess(BenchmarkCase benchmarkCase, ArtifactsPaths { FileName = runtime.JavaScriptEngine, Arguments = $"{runtime.JavaScriptEngineArguments} {mainJs} -- --run {artifactsPaths.ProgramName}.dll {args} ", - WorkingDirectory = artifactsPaths.BinariesDirectoryPath, + WorkingDirectory = Path.Combine(artifactsPaths.BinariesDirectoryPath, "AppBundle"), UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardInput = false, // not supported by WASM! diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs index acc1be303a..c33642f6a9 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs @@ -57,7 +57,6 @@ protected void GenerateProjectFile(BuildPartition buildPartition, ArtifactsPaths .Replace("$TFM$", TargetFrameworkMoniker) .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) .Replace("$COPIEDSETTINGS$", customProperties) - .Replace("$CONFIGURATIONNAME$", buildPartition.BuildConfiguration) .Replace("$SDKNAME$", sdkName) .Replace("$WASMDATADIR$", runtime.WasmDataDir) .Replace("$TARGET$", CustomRuntimePack != null ? "PublishWithCustomRuntimePack" : "Publish") @@ -66,11 +65,9 @@ protected void GenerateProjectFile(BuildPartition buildPartition, ArtifactsPaths File.WriteAllText(artifactsPaths.ProjectFilePath, content); } - protected override string GetExecutablePath(string binariesDirectoryPath, string programName) => Path.Combine(binariesDirectoryPath, MainJS); + protected override string GetExecutablePath(string binariesDirectoryPath, string programName) => Path.Combine(binariesDirectoryPath, "AppBundle", MainJS); protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - { - return Path.Combine(buildArtifactsDirectoryPath, "bin", TargetFrameworkMoniker, "browser-wasm", "AppBundle"); - } + => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, TargetFrameworkMoniker, "browser-wasm"); } } diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs index 37f816bde9..71ad2ade2b 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; @@ -26,15 +27,16 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas yield return validationError; } - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { yield return new ValidationError(true, $"{nameof(WasmToolchain)} is supported only on Unix, benchmark '{benchmarkCase.DisplayInfo}' might not work correctly", benchmarkCase); } - else if (IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError)) + + foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(CustomDotNetCliPath, benchmarkCase)) { - yield return invalidCliError; + yield return validationError; } } @@ -49,8 +51,7 @@ public static IToolchain From(NetCoreAppSettings netCoreAppSettings) new DotNetCliBuilder(netCoreAppSettings.TargetFrameworkMoniker, netCoreAppSettings.CustomDotNetCliPath, // aot builds can be very slow - logOutput: netCoreAppSettings.AOTCompilerMode == MonoAotLLVM.MonoAotCompilerMode.wasm, - retryFailedBuildWithNoDeps: false), + logOutput: netCoreAppSettings.AOTCompilerMode == MonoAotLLVM.MonoAotCompilerMode.wasm), new WasmExecutor(), netCoreAppSettings.CustomDotNetCliPath); } diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs index 6fb5e5421f..f1e32d6c6b 100644 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs @@ -3,11 +3,15 @@ using System.IO; using System.Linq; using System.Text; +using System.Xml; +using BenchmarkDotNet.ConsoleArguments; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Detectors.Cpu; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Portability.Cpu; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.DotNetCli; @@ -55,7 +59,7 @@ internal Generator(string ilCompilerVersion, private readonly string ilcOptimizationPreference; private readonly string ilcInstructionSet; - protected override string GetExecutableExtension() => RuntimeInformation.ExecutableExtension; + protected override string GetExecutableExtension() => OsDetector.ExecutableExtension; protected override string GetBuildArtifactsDirectoryPath(BuildPartition buildPartition, string programName) => useTempFolderForRestore @@ -71,7 +75,6 @@ protected override void GenerateBuildScript(BuildPartition buildPartition, Artif var content = new StringBuilder(300) .AppendLine($"call {CliPath ?? "dotnet"} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition, extraArguments)}") - .AppendLine($"call {CliPath ?? "dotnet"} {DotNetCliCommand.GetBuildCommand(artifactsPaths, buildPartition, extraArguments)}") .AppendLine($"call {CliPath ?? "dotnet"} {DotNetCliCommand.GetPublishCommand(artifactsPaths, buildPartition, extraArguments)}") .ToString(); @@ -131,14 +134,16 @@ private string GenerateProjectForNuGetBuild(BuildPartition buildPartition, Artif false true false - true + true {ilcOptimizationPreference} + {ilcOptimizationPreference} {GetTrimmingSettings()} - {ilcGenerateCompleteTypeMetadata} {ilcGenerateStackTraceData} + {ilcGenerateStackTraceData} false false false + true {GetInstructionSetSettings(buildPartition)} {GetRuntimeSettings(buildPartition.RepresentativeBenchmarkCase.Job.Environment.Gc, buildPartition.Resolver)} @@ -152,14 +157,27 @@ private string GenerateProjectForNuGetBuild(BuildPartition buildPartition, Artif {string.Join(Environment.NewLine, GetRdXmlFiles(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger).Select(file => $""))} +{GetCustomProperties(buildPartition, logger)} "; + private string GetCustomProperties(BuildPartition buildPartition, ILogger logger) + { + var projectFile = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger); + var xmlDoc = new XmlDocument(); + xmlDoc.Load(projectFile.FullName); + + (string customProperties, _) = GetSettingsThatNeedToBeCopied(xmlDoc, projectFile); + return customProperties; + } + + private string GetILCompilerPackageReference() => string.IsNullOrEmpty(ilCompilerVersion) ? "" : $@""; private string GetTrimmingSettings() => rootAllApplicationAssemblies - ? "" // use the defaults + // Use the defaults + ? "" // TrimMode is set in explicit way as for older versions it might have different default value : "linklink"; @@ -219,8 +237,20 @@ private string GetCurrentInstructionSet(Platform platform) => string.Join(",", GetCurrentProcessInstructionSets(platform)); // based on https://github.com/dotnet/runtime/blob/ce61c09a5f6fc71d8f717d3fc4562f42171869a0/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs#L727 - private static IEnumerable GetCurrentProcessInstructionSets(Platform platform) + private IEnumerable GetCurrentProcessInstructionSets(Platform platform) { + if (!ConfigParser.TryParse(TargetFrameworkMoniker, out RuntimeMoniker runtimeMoniker)) + { + throw new NotSupportedException($"Invalid TFM: '{TargetFrameworkMoniker}'"); + } + + if (platform == RuntimeInformation.GetCurrentPlatform() // "native" does not support cross-compilation (so does BDN for now) + && runtimeMoniker >= RuntimeMoniker.NativeAot80) + { + yield return "native"; // added in .NET 8 https://github.com/dotnet/runtime/pull/87865 + yield break; + } + switch (platform) { case Platform.X86: @@ -234,6 +264,11 @@ private static IEnumerable GetCurrentProcessInstructionSets(Platform pla if (HardwareIntrinsics.IsX86Sse42Supported) yield return "sse4.2"; if (HardwareIntrinsics.IsX86AvxSupported) yield return "avx"; if (HardwareIntrinsics.IsX86Avx2Supported) yield return "avx2"; + if (HardwareIntrinsics.IsX86Avx512FSupported) yield return "avx512f"; + if (HardwareIntrinsics.IsX86Avx512BWSupported) yield return "avx512bw"; + if (HardwareIntrinsics.IsX86Avx512CDSupported) yield return "avx512cd"; + if (HardwareIntrinsics.IsX86Avx512DQSupported) yield return "avx512dq"; + if (HardwareIntrinsics.IsX86Avx512VbmiSupported) yield return "avx512vbmi"; if (HardwareIntrinsics.IsX86AesSupported) yield return "aes"; if (HardwareIntrinsics.IsX86Bmi1Supported) yield return "bmi"; if (HardwareIntrinsics.IsX86Bmi2Supported) yield return "bmi2"; @@ -242,7 +277,7 @@ private static IEnumerable GetCurrentProcessInstructionSets(Platform pla if (HardwareIntrinsics.IsX86PclmulqdqSupported) yield return "pclmul"; if (HardwareIntrinsics.IsX86PopcntSupported) yield return "popcnt"; if (HardwareIntrinsics.IsX86AvxVnniSupported) yield return "avxvnni"; - if (HardwareIntrinsics.IsX86SerializeSupported) yield return "serialize"; + if (HardwareIntrinsics.IsX86SerializeSupported && runtimeMoniker > RuntimeMoniker.NativeAot70) yield return "serialize"; // https://github.com/dotnet/BenchmarkDotNet/issues/2463#issuecomment-1809625008 break; case Platform.Arm64: if (HardwareIntrinsics.IsArmBaseSupported) yield return "base"; diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs index 0beeff9627..9d476bf0e0 100644 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs @@ -1,5 +1,8 @@ using System.Collections.Generic; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Validators; namespace BenchmarkDotNet.Toolchains.NativeAot { @@ -22,13 +25,29 @@ public class NativeAotToolchain : Toolchain .ToToolchain(); /// - /// compiled as net8.0, targets latest NativeAOT build from the .NET 8 feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json + /// compiled as net8.0, targets latest NativeAOT build from the NuGet.org feed: "https://api.nuget.org/v3/index.json" /// public static readonly IToolchain Net80 = CreateBuilder() - .UseNuGet("", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8/nuget/v3/index.json") + .UseNuGet("", "https://api.nuget.org/v3/index.json") .TargetFrameworkMoniker("net8.0") .ToToolchain(); + /// + /// compiled as net9.0, targets latest NativeAOT build from the .NET 9 feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json + /// + public static readonly IToolchain Net90 = CreateBuilder() + .UseNuGet("", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json") + .TargetFrameworkMoniker("net9.0") + .ToToolchain(); + + /// + /// compiled as net10.0, targets latest NativeAOT build from the .NET 10 feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json + /// + public static readonly IToolchain Net10_0 = CreateBuilder() + .UseNuGet("", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json") + .TargetFrameworkMoniker("net10.0") + .ToToolchain(); + internal NativeAotToolchain(string displayName, string ilCompilerVersion, string runtimeFrameworkVersion, string targetFrameworkMoniker, string runtimeIdentifier, @@ -52,5 +71,18 @@ internal NativeAotToolchain(string displayName, public static NativeAotToolchainBuilder CreateBuilder() => NativeAotToolchainBuilder.Create(); public static string GetExtraArguments(string runtimeIdentifier) => $"-r {runtimeIdentifier}"; + + public override IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) + { + foreach (var error in base.Validate(benchmarkCase, resolver)) + { + yield return error; + } + + foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(CustomDotNetCliPath, benchmarkCase)) + { + yield return validationError; + } + } } } diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs index 4d37882007..1bd248e5eb 100644 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs @@ -17,7 +17,7 @@ public class NativeAotToolchainBuilder : CustomDotNetCliToolchainBuilder private bool ilcGenerateCompleteTypeMetadata = true; private bool ilcGenerateStackTraceData = true; private string ilcOptimizationPreference = "Speed"; - private string ilcInstructionSet = null; + private string? ilcInstructionSet = null; private bool isIlCompilerConfigured; @@ -53,8 +53,8 @@ public NativeAotToolchainBuilder UseLocalBuild(DirectoryInfo ilcPackages) if (!ilcPackages.Exists) throw new DirectoryNotFoundException($"{ilcPackages} provided as {nameof(ilcPackages)} does NOT exist"); Feeds["local"] = ilcPackages.FullName; - ilCompilerVersion = "7.0.0-dev"; - Feeds["dotnet7"] = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json"; + ilCompilerVersion = "10.0.0-dev"; + Feeds["dotnet10"] = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json"; useTempFolderForRestore = true; DisplayName("local ILCompiler build"); diff --git a/src/BenchmarkDotNet/Toolchains/Parameters/ExecuteParameters.cs b/src/BenchmarkDotNet/Toolchains/Parameters/ExecuteParameters.cs index bdfbd78a37..e10c81b49b 100644 --- a/src/BenchmarkDotNet/Toolchains/Parameters/ExecuteParameters.cs +++ b/src/BenchmarkDotNet/Toolchains/Parameters/ExecuteParameters.cs @@ -11,7 +11,7 @@ public class ExecuteParameters { internal static readonly TimeSpan ProcessExitTimeout = TimeSpan.FromSeconds(2); - public ExecuteParameters(BuildResult buildResult, BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, IResolver resolver, int launchIndex, IDiagnoser diagnoser = null) + public ExecuteParameters(BuildResult buildResult, BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, IResolver resolver, int launchIndex, IDiagnoser? diagnoser = null) { BuildResult = buildResult; BenchmarkCase = benchmarkCase; diff --git a/src/BenchmarkDotNet/Toolchains/Results/GenerateResult.cs b/src/BenchmarkDotNet/Toolchains/Results/GenerateResult.cs index b6de670b8d..8763fcbb7a 100644 --- a/src/BenchmarkDotNet/Toolchains/Results/GenerateResult.cs +++ b/src/BenchmarkDotNet/Toolchains/Results/GenerateResult.cs @@ -22,7 +22,7 @@ public GenerateResult(ArtifactsPaths artifactsPaths, bool isGenerateSuccess, Exc public static GenerateResult Success(ArtifactsPaths artifactsPaths, IReadOnlyCollection artifactsToCleanup) => new GenerateResult(artifactsPaths, true, null, artifactsToCleanup); - public static GenerateResult Failure(ArtifactsPaths artifactsPaths, IReadOnlyCollection artifactsToCleanup, Exception exception = null) + public static GenerateResult Failure(ArtifactsPaths artifactsPaths, IReadOnlyCollection artifactsToCleanup, Exception? exception = null) => new GenerateResult(artifactsPaths, false, exception, artifactsToCleanup); public override string ToString() => "GenerateResult: " + (IsGenerateSuccess ? "Success" : "Fail"); diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs index 0718d6b8cb..3ba7054f7a 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using System.Reflection; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; @@ -27,9 +28,9 @@ protected override string[] GetArtifactsToCleanup(ArtifactsPaths artifactsPaths) protected override void GenerateBuildScript(BuildPartition buildPartition, ArtifactsPaths artifactsPaths) { - string prefix = RuntimeInformation.IsWindows() ? "" : "#!/bin/bash\n"; + string prefix = OsDetector.IsWindows() ? "" : "#!/bin/bash\n"; var list = new List(); - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) list.Add("mono"); list.Add("csc"); list.Add("/noconfig"); diff --git a/src/BenchmarkDotNet/Toolchains/Toolchain.cs b/src/BenchmarkDotNet/Toolchains/Toolchain.cs index bbe5f1d94f..4ff773a797 100644 --- a/src/BenchmarkDotNet/Toolchains/Toolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Toolchain.cs @@ -57,31 +57,6 @@ public virtual IEnumerable Validate(BenchmarkCase benchmarkCase } } - internal static bool IsCliPathInvalid(string customDotNetCliPath, BenchmarkCase benchmarkCase, out ValidationError validationError) - { - validationError = null; - - if (string.IsNullOrEmpty(customDotNetCliPath) && !HostEnvironmentInfo.GetCurrent().IsDotNetCliInstalled()) - { - validationError = new ValidationError(true, - $"BenchmarkDotNet requires dotnet SDK to be installed or path to local dotnet cli provided in explicit way using `--cli` argument, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", - benchmarkCase); - - return true; - } - - if (!string.IsNullOrEmpty(customDotNetCliPath) && !File.Exists(customDotNetCliPath)) - { - validationError = new ValidationError(true, - $"Provided custom dotnet cli path does not exist, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", - benchmarkCase); - - return true; - } - - return false; - } - public override string ToString() => Name; } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs index ad3b4a5a6b..2beab5018c 100644 --- a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs +++ b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs @@ -1,4 +1,5 @@ using System; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; @@ -28,11 +29,9 @@ private static IToolchain GetToolchain(Job job, Descriptor descriptor) : GetToolchain( job.ResolveValue(EnvironmentMode.RuntimeCharacteristic, EnvironmentResolver.Instance), descriptor, - job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic) - || job.HasValue(InfrastructureMode.BuildConfigurationCharacteristic) - || job.HasValue(InfrastructureMode.ArgumentsCharacteristic)); + job.HasDynamicBuildCharacteristic()); - internal static IToolchain GetToolchain(this Runtime runtime, Descriptor descriptor = null, bool preferMsBuildToolchains = false) + internal static IToolchain GetToolchain(this Runtime runtime, Descriptor? descriptor = null, bool preferMsBuildToolchains = false) { switch (runtime) { @@ -48,9 +47,9 @@ internal static IToolchain GetToolchain(this Runtime runtime, Descriptor descrip : CsProjClassicNetToolchain.From(clrRuntime.MsBuildMoniker); case MonoRuntime mono: - if (RuntimeInformation.IsAndroid()) + if (OsDetector.IsAndroid()) return InProcessEmitToolchain.Instance; - if (RuntimeInformation.IsIOS()) + if (OsDetector.IsIOS()) return InProcessNoEmitToolchain.Instance; if (!string.IsNullOrEmpty(mono.AotArgs)) return MonoAotToolchain.Instance; @@ -66,6 +65,8 @@ internal static IToolchain GetToolchain(this Runtime runtime, Descriptor descrip RuntimeMoniker.Mono60 => GetToolchain(RuntimeMoniker.Net60), RuntimeMoniker.Mono70 => GetToolchain(RuntimeMoniker.Net70), RuntimeMoniker.Mono80 => GetToolchain(RuntimeMoniker.Net80), + RuntimeMoniker.Mono90 => GetToolchain(RuntimeMoniker.Net90), + RuntimeMoniker.Mono10_0 => GetToolchain(RuntimeMoniker.Net10_0), _ => CsProjCoreToolchain.From(new NetCoreAppSettings(mono.MsBuildMoniker, null, mono.Name)) }; } @@ -152,6 +153,12 @@ private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker) case RuntimeMoniker.Net80: return CsProjCoreToolchain.NetCoreApp80; + case RuntimeMoniker.Net90: + return CsProjCoreToolchain.NetCoreApp90; + + case RuntimeMoniker.Net10_0: + return CsProjCoreToolchain.NetCoreApp10_0; + case RuntimeMoniker.NativeAot60: return NativeAotToolchain.Net60; @@ -161,6 +168,12 @@ private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker) case RuntimeMoniker.NativeAot80: return NativeAotToolchain.Net80; + case RuntimeMoniker.NativeAot90: + return NativeAotToolchain.Net90; + + case RuntimeMoniker.NativeAot10_0: + return NativeAotToolchain.Net10_0; + case RuntimeMoniker.Mono60: return MonoToolchain.Mono60; @@ -170,6 +183,12 @@ private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker) case RuntimeMoniker.Mono80: return MonoToolchain.Mono80; + case RuntimeMoniker.Mono90: + return MonoToolchain.Mono90; + + case RuntimeMoniker.Mono10_0: + return MonoToolchain.Mono10_0; + default: throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "RuntimeMoniker not supported"); } diff --git a/src/BenchmarkDotNet/Validators/CompilationValidator.cs b/src/BenchmarkDotNet/Validators/CompilationValidator.cs index d8c2ec6c5b..b0bb87681d 100644 --- a/src/BenchmarkDotNet/Validators/CompilationValidator.cs +++ b/src/BenchmarkDotNet/Validators/CompilationValidator.cs @@ -7,6 +7,7 @@ using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains; using Microsoft.CodeAnalysis.CSharp; +using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Validators { @@ -25,8 +26,38 @@ private CompilationValidator() { } public IEnumerable Validate(ValidationParameters validationParameters) => ValidateCSharpNaming(validationParameters.Benchmarks) .Union(ValidateNamingConflicts(validationParameters.Benchmarks)) + .Union(ValidateClassModifiers((validationParameters.Benchmarks)) .Union(ValidateAccessModifiers(validationParameters.Benchmarks)) - .Union(ValidateBindingModifiers(validationParameters.Benchmarks)); + .Union(ValidateBindingModifiers(validationParameters.Benchmarks)) + ); + + private static IEnumerable ValidateClassModifiers(IEnumerable benchmarks) + { + return benchmarks + .Distinct(BenchmarkMethodEqualityComparer.Instance) + .SelectMany(benchmark => + { + var type = benchmark.Descriptor.Type; + var errors = new List(); + + if (type.IsSealed) + { + errors.Add(new ValidationError( + true, + $"Benchmarked method `{benchmark.Descriptor.WorkloadMethod.Name}` is within a sealed class, Declaring type must be unsealed.", + benchmark)); + } + if (!type.IsVisible) + { + errors.Add(new ValidationError( + true, + $"Benchmarked method `{benchmark.Descriptor.WorkloadMethod.Name}` is within a non-visible class, all declaring types must be public.", + benchmark)); + } + // TODO: Generics validation + return errors; + }); + } private static IEnumerable ValidateCSharpNaming(IEnumerable benchmarks) => benchmarks diff --git a/src/BenchmarkDotNet/Validators/ConfigCompatibilityValidator.cs b/src/BenchmarkDotNet/Validators/ConfigCompatibilityValidator.cs deleted file mode 100644 index 9732f40383..0000000000 --- a/src/BenchmarkDotNet/Validators/ConfigCompatibilityValidator.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System.Linq; -using System.Collections.Generic; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Validators -{ - public class ConfigCompatibilityValidator : IValidator - { - public static readonly ConfigCompatibilityValidator FailOnError = new ConfigCompatibilityValidator(); - - public bool TreatsWarningsAsErrors => true; - - public IEnumerable Validate(ValidationParameters validationParameters) - { - var orderers = - validationParameters - .Benchmarks - .Where(benchmark => benchmark.Config.Orderer != Order.DefaultOrderer.Instance) - .Select(benchmark => benchmark.Config.Orderer) - .Distinct(); - - if (orderers.Count() > 1) - yield return new ValidationError(true, "You use JoinSummary options, but provided configurations cannot be joined. Only one Orderer per benchmark cases is allowed."); - - var styles = - validationParameters - .Benchmarks - .Where(benchmark => benchmark.Config.SummaryStyle != SummaryStyle.Default - && benchmark.Config.SummaryStyle != null) // Paranoid - .Select(benchmark => benchmark.Config.SummaryStyle) - .Distinct(); - - if (styles.Count() > 1) - yield return new ValidationError(true, "You use JoinSummary options, but provided configurations cannot be joined. Only one SummaryStyle per benchmark cases is allowed."); - } - } -} diff --git a/src/BenchmarkDotNet/Validators/DotNetSdkVersionValidator.cs b/src/BenchmarkDotNet/Validators/DotNetSdkVersionValidator.cs new file mode 100644 index 0000000000..17331b49f8 --- /dev/null +++ b/src/BenchmarkDotNet/Validators/DotNetSdkVersionValidator.cs @@ -0,0 +1,251 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Validators +{ + internal static class DotNetSdkValidator + { + private static readonly Lazy> cachedFrameworkSdks = new Lazy>(GetInstalledFrameworkSdks, true); + + public static IEnumerable ValidateCoreSdks(string? customDotNetCliPath, BenchmarkCase benchmark) + { + if (IsCliPathInvalid(customDotNetCliPath, benchmark, out ValidationError? cliPathError)) + { + yield return cliPathError; + } + else if (TryGetSdkVersion(benchmark, out Version requiredSdkVersion)) + { + var installedSdks = GetInstalledDotNetSdks(customDotNetCliPath); + if (!installedSdks.Any(sdk => sdk >= requiredSdkVersion)) + { + yield return new ValidationError(true, $"The required .NET Core SDK version {requiredSdkVersion} or higher for runtime moniker {benchmark.Job.Environment.Runtime.RuntimeMoniker} is not installed.", benchmark); + } + } + } + + public static IEnumerable ValidateFrameworkSdks(BenchmarkCase benchmark) + { + if (!TryGetSdkVersion(benchmark, out Version requiredSdkVersion)) + { + yield break; + } + + var installedVersionString = cachedFrameworkSdks.Value.FirstOrDefault(); + + if (installedVersionString == null || Version.TryParse(installedVersionString, out var installedVersion) && installedVersion < requiredSdkVersion) + { + yield return new ValidationError(true, $"The required .NET Framework SDK version {requiredSdkVersion} or higher is not installed.", benchmark); + } + } + + public static bool IsCliPathInvalid(string customDotNetCliPath, BenchmarkCase benchmarkCase, out ValidationError? validationError) + { + validationError = null; + + if (string.IsNullOrEmpty(customDotNetCliPath) && !HostEnvironmentInfo.GetCurrent().IsDotNetCliInstalled()) + { + validationError = new ValidationError(true, + $"BenchmarkDotNet requires dotnet SDK to be installed or path to local dotnet cli provided in explicit way using `--cli` argument, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); + + return true; + } + + if (!string.IsNullOrEmpty(customDotNetCliPath) && !File.Exists(customDotNetCliPath)) + { + validationError = new ValidationError(true, + $"Provided custom dotnet cli path does not exist, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); + + return true; + } + + return false; + } + + private static bool TryGetSdkVersion(BenchmarkCase benchmark, out Version sdkVersion) + { + sdkVersion = default; + if (benchmark?.Job?.Environment?.Runtime?.RuntimeMoniker != null) + { + sdkVersion = GetSdkVersionFromMoniker(benchmark.Job.Environment.Runtime.RuntimeMoniker); + return true; + } + return false; + } + + private static IEnumerable GetInstalledDotNetSdks(string? customDotNetCliPath) + { + string dotnetExecutable = string.IsNullOrEmpty(customDotNetCliPath) ? "dotnet" : customDotNetCliPath; + var startInfo = new ProcessStartInfo(dotnetExecutable, "--list-sdks") + { + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + try + { + using (var process = Process.Start(startInfo)) + { + if (process == null) + { + return Enumerable.Empty(); + } + + process.WaitForExit(); + + if (process.ExitCode == 0) + { + var output = process.StandardOutput.ReadToEnd(); + var lines = output.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + + var versions = new List(lines.Count()); + foreach (var line in lines) + { + // Version.TryParse does not handle things like 3.0.0-WORD, so this will get just the 3.0.0 part + var parsableVersionPart = CoreRuntime.GetParsableVersionPart(line); + if (Version.TryParse(parsableVersionPart, out var version)) + { + versions.Add(version); + } + } + + return versions; + } + else + { + return Enumerable.Empty(); + } + } + } + catch (Win32Exception) // dotnet CLI is not installed or not found in the path. + { + return Enumerable.Empty(); + } + } + + public static List GetInstalledFrameworkSdks() + { + var versions = new List(); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Get45PlusFromRegistry(versions); + } + + return versions; + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Interoperability", "CA1416:Validate platform compatibility", Justification = "This code is protected with a runtime OS platform check")] + private static void Get45PlusFromRegistry(List versions) + { + const string subkey = @"SOFTWARE\Microsoft\NET Framework Setup\NDP\v4\Full\"; + + using (var ndpKey = Microsoft.Win32.Registry.LocalMachine.OpenSubKey(subkey)) + { + if (ndpKey == null) + { + return; + } + + if (ndpKey.GetValue("Version") != null) + { + versions.Add(ndpKey.GetValue("Version").ToString()); + } + else + { + if (ndpKey.GetValue("Release") != null) + { + versions.Add(CheckFor45PlusVersion((int)ndpKey.GetValue("Release"))); + } + } + } + } + + private static string CheckFor45PlusVersion(int releaseKey) + { + if (releaseKey >= 533320) + return "4.8.1"; + if (releaseKey >= 528040) + return "4.8"; + if (releaseKey >= 461808) + return "4.7.2"; + if (releaseKey >= 461308) + return "4.7.1"; + if (releaseKey >= 460798) + return "4.7"; + if (releaseKey >= 394802) + return "4.6.2"; + if (releaseKey >= 394254) + return "4.6.1"; + if (releaseKey >= 393295) + return "4.6"; + if (releaseKey >= 379893) + return "4.5.2"; + if (releaseKey >= 378675) + return "4.5.1"; + if (releaseKey >= 378389) + return "4.5"; + + return ""; + } + + private static Version GetSdkVersionFromMoniker(RuntimeMoniker runtimeMoniker) + { + return runtimeMoniker switch + { + RuntimeMoniker.Net461 => new Version(4, 6, 1), + RuntimeMoniker.Net462 => new Version(4, 6, 2), + RuntimeMoniker.Net47 => new Version(4, 7), + RuntimeMoniker.Net471 => new Version(4, 7, 1), + RuntimeMoniker.Net472 => new Version(4, 7, 2), + RuntimeMoniker.Net48 => new Version(4, 8), + RuntimeMoniker.Net481 => new Version(4, 8, 1), + RuntimeMoniker.NetCoreApp20 => new Version(2, 0), + RuntimeMoniker.NetCoreApp21 => new Version(2, 1), + RuntimeMoniker.NetCoreApp22 => new Version(2, 2), + RuntimeMoniker.NetCoreApp30 => new Version(3, 0), + RuntimeMoniker.NetCoreApp31 => new Version(3, 1), + RuntimeMoniker.Net50 => new Version(5, 0), + RuntimeMoniker.Net60 => new Version(6, 0), + RuntimeMoniker.Net70 => new Version(7, 0), + RuntimeMoniker.Net80 => new Version(8, 0), + RuntimeMoniker.Net90 => new Version(9, 0), + RuntimeMoniker.Net10_0 => new Version(10, 0), + RuntimeMoniker.NativeAot60 => new Version(6, 0), + RuntimeMoniker.NativeAot70 => new Version(7, 0), + RuntimeMoniker.NativeAot80 => new Version(8, 0), + RuntimeMoniker.NativeAot90 => new Version(9, 0), + RuntimeMoniker.NativeAot10_0 => new Version(10, 0), + RuntimeMoniker.Mono60 => new Version(6, 0), + RuntimeMoniker.Mono70 => new Version(7, 0), + RuntimeMoniker.Mono80 => new Version(8, 0), + RuntimeMoniker.Mono90 => new Version(9, 0), + RuntimeMoniker.Mono10_0 => new Version(10, 0), + RuntimeMoniker.Wasm => Portability.RuntimeInformation.IsNetCore && CoreRuntime.TryGetVersion(out var version) ? version : new Version(5, 0), + RuntimeMoniker.WasmNet50 => new Version(5, 0), + RuntimeMoniker.WasmNet60 => new Version(6, 0), + RuntimeMoniker.WasmNet70 => new Version(7, 0), + RuntimeMoniker.WasmNet80 => new Version(8, 0), + RuntimeMoniker.WasmNet90 => new Version(9, 0), + RuntimeMoniker.WasmNet10_0 => new Version(10, 0), + RuntimeMoniker.MonoAOTLLVM => Portability.RuntimeInformation.IsNetCore && CoreRuntime.TryGetVersion(out var version) ? version : new Version(6, 0), + RuntimeMoniker.MonoAOTLLVMNet60 => new Version(6, 0), + RuntimeMoniker.MonoAOTLLVMNet70 => new Version(7, 0), + RuntimeMoniker.MonoAOTLLVMNet80 => new Version(8, 0), + RuntimeMoniker.MonoAOTLLVMNet90 => new Version(9, 0), + RuntimeMoniker.MonoAOTLLVMNet10_0 => new Version(10, 0), + _ => throw new NotImplementedException($"SDK version check not implemented for {runtimeMoniker}") + }; + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Validators/ExecutionValidatorBase.cs b/src/BenchmarkDotNet/Validators/ExecutionValidatorBase.cs index 92d7422ae2..bb94aec254 100644 --- a/src/BenchmarkDotNet/Validators/ExecutionValidatorBase.cs +++ b/src/BenchmarkDotNet/Validators/ExecutionValidatorBase.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Validators @@ -52,7 +53,7 @@ public IEnumerable Validate(ValidationParameters validationPara return errors; } - private bool TryCreateBenchmarkTypeInstance(Type type, List errors, out object instance) + private bool TryCreateBenchmarkTypeInstance(Type type, List errors, out object? instance) { try { @@ -130,21 +131,8 @@ private void TryToGetTaskResult(object result) return; } - var returnType = result.GetType(); - if (returnType.IsGenericType && returnType.GetGenericTypeDefinition() == typeof(ValueTask<>)) - { - var asTaskMethod = result.GetType().GetMethod("AsTask"); - result = asTaskMethod.Invoke(result, null); - } - - if (result is Task task) - { - task.GetAwaiter().GetResult(); - } - else if (result is ValueTask valueTask) - { - valueTask.GetAwaiter().GetResult(); - } + AwaitHelper.GetGetResultMethod(result.GetType()) + ?.Invoke(null, new[] { result }); } private bool TryToSetParamsFields(object benchmarkTypeInstance, List errors) diff --git a/src/BenchmarkDotNet/Validators/ParamsValidator.cs b/src/BenchmarkDotNet/Validators/ParamsValidator.cs index 630811b62b..5a503148d5 100644 --- a/src/BenchmarkDotNet/Validators/ParamsValidator.cs +++ b/src/BenchmarkDotNet/Validators/ParamsValidator.cs @@ -24,19 +24,19 @@ private IEnumerable Validate(Type type) BindingFlags.FlattenHierarchy; foreach (var memberInfo in type.GetMembers(reflectionFlags)) { - var attributes = new Attribute[] + var attributes = new Attribute?[] { memberInfo.ResolveAttribute(), memberInfo.ResolveAttribute(), memberInfo.ResolveAttribute() } - .Where(attribute => attribute != null) + .WhereNotNull() .ToList(); if (attributes.IsEmpty()) continue; string name = $"{type.Name}.{memberInfo.Name}"; - string? attributeString = string.Join(", ", attributes.Select(attribute => $"[{attribute.GetType().Name.Replace(nameof(Attribute), "")}]")); + string attributeString = string.Join(", ", attributes.Select(attribute => $"[{attribute.GetType().Name.Replace(nameof(Attribute), "")}]")); if (attributes.Count > 1) yield return new ValidationError(TreatsWarningsAsErrors, diff --git a/src/BenchmarkDotNet/Validators/ValidationError.cs b/src/BenchmarkDotNet/Validators/ValidationError.cs index e8f5e40f6b..9bec081479 100644 --- a/src/BenchmarkDotNet/Validators/ValidationError.cs +++ b/src/BenchmarkDotNet/Validators/ValidationError.cs @@ -6,7 +6,7 @@ namespace BenchmarkDotNet.Validators { public class ValidationError : IEquatable { - public ValidationError(bool isCritical, string message, BenchmarkCase benchmarkCase = null) + public ValidationError(bool isCritical, string message, BenchmarkCase? benchmarkCase = null) { IsCritical = isCritical; Message = message; @@ -15,11 +15,11 @@ public ValidationError(bool isCritical, string message, BenchmarkCase benchmarkC [PublicAPI] public bool IsCritical { get; } [PublicAPI] public string Message { get; } - [PublicAPI] public BenchmarkCase BenchmarkCase { get; } + [PublicAPI] public BenchmarkCase? BenchmarkCase { get; } public override string ToString() => Message; - public bool Equals(ValidationError other) + public bool Equals(ValidationError? other) { if (ReferenceEquals(null, other)) return false; @@ -28,13 +28,13 @@ public bool Equals(ValidationError other) return IsCritical == other.IsCritical && string.Equals(Message, other.Message) && Equals(BenchmarkCase, other.BenchmarkCase); } - public override bool Equals(object obj) + public override bool Equals(object? obj) { if (ReferenceEquals(null, obj)) return false; if (ReferenceEquals(this, obj)) return true; - if (obj.GetType() != this.GetType()) + if (obj.GetType() != GetType()) return false; return Equals((ValidationError)obj); } diff --git a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json index 567259b34d..108d1248cb 100644 --- a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json +++ b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json @@ -34,6 +34,14 @@ "description": "The target framework for the project.", "datatype": "choice", "choices": [ + { + "choice": "net10.0", + "description": ".NET 10" + }, + { + "choice": "net9.0", + "description": ".NET 9" + }, { "choice": "net8.0", "description": ".NET 8" @@ -90,35 +98,22 @@ "datatype": "string", "cases": [ { - "condition": "(framework == '' && consoleApp == true)", - "value": "net6.0" + "condition": "(consoleApp == true)", + "value": "net8.0" }, { - "condition": "(framework == '' && consoleApp == false)", + "condition": "(consoleApp == false)", "value": "netstandard2.0" - }, - { - "condition": "(framework != '')", - "value": "" } ] } }, "frameworkValue": { "type": "generated", - "generator": "join", - "description": "join framework and frameworkDefault", + "generator": "coalesce", "parameters": { - "symbols": [ - { - "type": "ref", - "value": "framework" - }, - { - "type": "ref", - "value": "frameworkDefault" - } - ] + "sourceVariableName": "framework", + "fallbackVariableName": "frameworkDefault" }, "replaces": "$(Framework)" }, @@ -144,7 +139,7 @@ "type": "parameter", "datatype": "string", "description": "Version of BenchmarkDotNet that will be referenced.", - "defaultValue": "0.13.8", + "defaultValue": "0.14.1", "replaces": "$(BenchmarkDotNetVersion)" } }, diff --git a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json index 12f8c5b9bd..27270458e2 100644 --- a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json +++ b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json @@ -34,6 +34,14 @@ "description": "The target framework for the project.", "datatype": "choice", "choices": [ + { + "choice": "net10.0", + "description": ".NET 10" + }, + { + "choice": "net9.0", + "description": ".NET 9" + }, { "choice": "net8.0", "description": ".NET 8" @@ -90,35 +98,22 @@ "datatype": "string", "cases": [ { - "condition": "(framework == '' && consoleApp == true)", - "value": "net6.0" + "condition": "(consoleApp == true)", + "value": "net8.0" }, { - "condition": "(framework == '' && consoleApp == false)", + "condition": "(consoleApp == false)", "value": "netstandard2.0" - }, - { - "condition": "(framework != '')", - "value": "" } ] } }, "frameworkValue": { "type": "generated", - "generator": "join", - "description": "join framework and frameworkDefault", + "generator": "coalesce", "parameters": { - "symbols": [ - { - "type": "ref", - "value": "framework" - }, - { - "type": "ref", - "value": "frameworkDefault" - } - ] + "sourceVariableName": "framework", + "fallbackVariableName": "frameworkDefault" }, "replaces": "$(Framework)" }, @@ -144,7 +139,7 @@ "type": "parameter", "datatype": "string", "description": "Version of BenchmarkDotNet that will be referenced.", - "defaultValue": "0.13.8", + "defaultValue": "0.14.1", "replaces": "$(BenchmarkDotNetVersion)" } }, diff --git a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json index 0e8597a653..07ac31147e 100644 --- a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json +++ b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json @@ -34,6 +34,14 @@ "description": "The target framework for the project.", "datatype": "choice", "choices": [ + { + "choice": "net10.0", + "description": ".NET 10" + }, + { + "choice": "net9.0", + "description": ".NET 9" + }, { "choice": "net8.0", "description": ".NET 8" @@ -90,35 +98,22 @@ "datatype": "string", "cases": [ { - "condition": "(framework == '' && consoleApp == true)", - "value": "net6.0" + "condition": "(consoleApp == true)", + "value": "net8.0" }, { - "condition": "(framework == '' && consoleApp == false)", + "condition": "(consoleApp == false)", "value": "netstandard2.0" - }, - { - "condition": "(framework != '')", - "value": "" } ] } }, "frameworkValue": { "type": "generated", - "generator": "join", - "description": "join framework and frameworkDefault", + "generator": "coalesce", "parameters": { - "symbols": [ - { - "type": "ref", - "value": "framework" - }, - { - "type": "ref", - "value": "frameworkDefault" - } - ] + "sourceVariableName": "framework", + "fallbackVariableName": "frameworkDefault" }, "replaces": "$(Framework)" }, @@ -144,7 +139,7 @@ "type": "parameter", "datatype": "string", "description": "Version of BenchmarkDotNet that will be referenced.", - "defaultValue": "0.13.8", + "defaultValue": "0.14.1", "replaces": "$(BenchmarkDotNetVersion)" } }, diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj new file mode 100644 index 0000000000..cb7dd76de2 --- /dev/null +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj @@ -0,0 +1,26 @@ + + + + net8.0;net462 + false + true + true + false + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs new file mode 100644 index 0000000000..e717b6b84d --- /dev/null +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs @@ -0,0 +1,297 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Mocks; +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Exporters.Plotting.Tests +{ + public class ScottPlotExporterTests(ITestOutputHelper output) + { + public static TheoryData GetGroupBenchmarkTypes() + { + var data = new TheoryData(); + foreach (var type in typeof(BaselinesBenchmarks).GetNestedTypes()) + data.Add(type); + return data; + } + + [Theory] + [MemberData(nameof(GetGroupBenchmarkTypes))] + public void BarPlots(Type benchmarkType) + { + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + benchmarkType.Name + " ==="); + + var exporter = new ScottPlotExporter() + { + IncludeBarPlot = true, + IncludeBoxPlot = false, + }; + var summary = MockFactory.CreateSummary(benchmarkType); + var filePaths = exporter.ExportToFiles(summary, logger).ToList(); + Assert.NotEmpty(filePaths); + Assert.All(filePaths, f => File.Exists(f)); + + foreach (string filePath in filePaths) + logger.WriteLine($"* {filePath}"); + output.WriteLine(logger.GetLog()); + } + + [Theory] + [MemberData(nameof(GetGroupBenchmarkTypes))] + public void BoxPlots(Type benchmarkType) + { + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + benchmarkType.Name + " ==="); + + var exporter = new ScottPlotExporter() + { + IncludeBarPlot = false, + IncludeBoxPlot = true, + }; + var summary = MockFactory.CreateSummaryWithBiasedDistribution(benchmarkType, 1, 4, 10, 9); + var filePaths = exporter.ExportToFiles(summary, logger).ToList(); + Assert.NotEmpty(filePaths); + Assert.All(filePaths, f => File.Exists(f)); + + foreach (string filePath in filePaths) + logger.WriteLine($"* {filePath}"); + output.WriteLine(logger.GetLog()); + } + + [Theory] + [MemberData(nameof(GetGroupBenchmarkTypes))] + public void BoxPlotsWithOneMeasurement(Type benchmarkType) + { + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + benchmarkType.Name + " ==="); + + var exporter = new ScottPlotExporter() + { + IncludeBarPlot = false, + IncludeBoxPlot = true, + }; + var summary = MockFactory.CreateSummaryWithBiasedDistribution(benchmarkType, 1, 4, 10, 1); + var filePaths = exporter.ExportToFiles(summary, logger).ToList(); + Assert.NotEmpty(filePaths); + Assert.All(filePaths, f => File.Exists(f)); + + foreach (string filePath in filePaths) + logger.WriteLine($"* {filePath}"); + output.WriteLine(logger.GetLog()); + } + + [SuppressMessage("ReSharper", "InconsistentNaming")] + public static class BaselinesBenchmarks + { + /* NoBaseline */ + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + public class NoBaseline_MethodsParamsJobs + { + [Params(2, 10)] public int Param; + + [Benchmark] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByMethod)] + public class NoBaseline_MethodsParamsJobs_GroupByMethod + { + [Params(2, 10)] public int Param; + + [Benchmark, BenchmarkCategory("CatA")] public void Base() { } + [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } + [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByJob)] + public class NoBaseline_MethodsParamsJobs_GroupByJob + { + [Params(2, 10)] public int Param; + + [Benchmark, BenchmarkCategory("CatA")] public void Base() { } + [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } + [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByParams)] + public class NoBaseline_MethodsParamsJobs_GroupByParams + { + [Params(2, 10)] public int Param; + + [Benchmark, BenchmarkCategory("CatA")] public void Base() { } + [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } + [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] + public class NoBaseline_MethodsParamsJobs_GroupByCategory + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true), BenchmarkCategory("CatA")] + public void A1() { } + + [Benchmark, BenchmarkCategory("CatA")] public void A2() { } + + [Benchmark(Baseline = true), BenchmarkCategory("CatB")] + public void B1() { } + + [Benchmark, BenchmarkCategory("CatB")] public void B2() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy( + BenchmarkLogicalGroupRule.ByMethod, + BenchmarkLogicalGroupRule.ByJob, + BenchmarkLogicalGroupRule.ByParams, + BenchmarkLogicalGroupRule.ByCategory)] + public class NoBaseline_MethodsParamsJobs_GroupByAll + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true), BenchmarkCategory("CatA")] + public void A1() { } + + [Benchmark, BenchmarkCategory("CatA")] public void A2() { } + + [Benchmark(Baseline = true), BenchmarkCategory("CatB")] + public void B1() { } + + [Benchmark, BenchmarkCategory("CatB")] public void B2() { } + } + + /* MethodBaseline */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + public class MethodBaseline_Methods + { + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + public class MethodBaseline_MethodsParams + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + public class MethodBaseline_MethodsJobs + { + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + public class MethodBaseline_MethodsParamsJobs + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* JobBaseline */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class JobBaseline_MethodsJobs + { + [Benchmark] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class JobBaseline_MethodsParamsJobs + { + [Params(2, 10)] public int Param; + + [Benchmark] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* MethodJobBaseline */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class MethodJobBaseline_MethodsJobs + { + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class MethodJobBaseline_MethodsJobsParams + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* Invalid */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + public class Invalid_TwoMethodBaselines + { + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark(Baseline = true)] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2", baseline: true)] + public class Invalid_TwoJobBaselines + { + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* Escape Params */ + + public class Escape_ParamsAndArguments + { + [Params("\t", "\n")] public string StringParam; + + [Arguments('\t')] + [Arguments('\n')] + [Benchmark] public void Foo(char charArg) { } + + [Benchmark] public void Bar() { } + } + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj b/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj index 70ca871259..1f1c48deec 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet.IntegrationTests.ConfigPerAssembly - net462;net7.0 + net462;net8.0 true BenchmarkDotNet.IntegrationTests.ConfigPerAssembly BenchmarkDotNet.IntegrationTests.ConfigPerAssembly @@ -14,7 +14,7 @@ - + \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.CustomPaths/BenchmarkDotNet.IntegrationTests.CustomPaths.csproj b/tests/BenchmarkDotNet.IntegrationTests.CustomPaths/BenchmarkDotNet.IntegrationTests.CustomPaths.csproj index cb4df7361e..c7f0cb3ae1 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.CustomPaths/BenchmarkDotNet.IntegrationTests.CustomPaths.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.CustomPaths/BenchmarkDotNet.IntegrationTests.CustomPaths.csproj @@ -14,7 +14,7 @@ - + diff --git a/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj b/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj index aad9d6ac33..754b32efcf 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet.IntegrationTests.DisabledOptimizations - net462;net7.0 + net462;net8.0 true BenchmarkDotNet.IntegrationTests.DisabledOptimizations BenchmarkDotNet.IntegrationTests.DisabledOptimizations @@ -16,7 +16,7 @@ - + \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj b/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj index da7558c82e..6a12d2c02c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet.IntegrationTests.EnabledOptimizations - net462;net7.0 + net462;net8.0 true BenchmarkDotNet.IntegrationTests.EnabledOptimizations BenchmarkDotNet.IntegrationTests.EnabledOptimizations @@ -16,7 +16,7 @@ - + \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.FSharp/BenchmarkDotNet.IntegrationTests.FSharp.fsproj b/tests/BenchmarkDotNet.IntegrationTests.FSharp/BenchmarkDotNet.IntegrationTests.FSharp.fsproj index 6feefadc37..f3a86d5579 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.FSharp/BenchmarkDotNet.IntegrationTests.FSharp.fsproj +++ b/tests/BenchmarkDotNet.IntegrationTests.FSharp/BenchmarkDotNet.IntegrationTests.FSharp.fsproj @@ -5,7 +5,7 @@ - net462;net7.0 + net462;net8.0 false diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj index f5f83b5cda..12457479b4 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj @@ -3,7 +3,10 @@ BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks - net461;net48;netcoreapp2.0;net7.0 + net461;net48;netcoreapp2.0;net8.0 + + $(NoWarn);NETSDK1138;NU1901;NU1902;NU1903;NU1904 + false true BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks @@ -22,7 +25,7 @@ - + @@ -36,8 +39,9 @@ - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs index 701b7d7ee1..b9cf360c8f 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs @@ -5,7 +5,7 @@ using BenchmarkDotNet.Jobs; using Xunit; -namespace BenchmarkDotNet.IntegrationTests +namespace BenchmarkDotNet.IntegrationTests.ManualRunning { public class MultipleFrameworksTest : BenchmarkTestExecutor { @@ -15,9 +15,23 @@ public class MultipleFrameworksTest : BenchmarkTestExecutor [InlineData(RuntimeMoniker.Net461)] [InlineData(RuntimeMoniker.Net48)] [InlineData(RuntimeMoniker.NetCoreApp20)] - [InlineData(RuntimeMoniker.Net70)] + [InlineData(RuntimeMoniker.Net80)] public void EachFrameworkIsRebuilt(RuntimeMoniker runtime) { +#if NET461 + // We cannot detect what target framework version the host was compiled for on full Framework, + // which causes the RoslynToolchain to be used instead of CsProjClassicNetToolchain when the host is full Framework + // (because full Framework always uses the version that's installed on the machine, unlike Core), + // which means if the machine has net48 installed (not net481), the net461 host with net48 runtime moniker + // will not be recompiled, causing the test to fail. + + // If we ever change the default toolchain to CsProjClassicNetToolchain instead of RoslynToolchain, we can remove this check. + if (runtime == RuntimeMoniker.Net48) + { + // XUnit doesn't provide Assert.Skip API yet. + return; + } +#endif var config = ManualConfig.CreateEmpty().AddJob(Job.Dry.WithRuntime(runtime.GetRuntime()).WithEnvironmentVariable(TfmEnvVarName, runtime.ToString())); CanExecute(config); } @@ -31,8 +45,8 @@ public class ValuePerTfm RuntimeMoniker.Net48; #elif NETCOREAPP2_0 RuntimeMoniker.NetCoreApp20; -#elif NET7_0 - RuntimeMoniker.Net70; +#elif NET8_0 + RuntimeMoniker.Net80; #else RuntimeMoniker.NotRecognized; #endif @@ -42,7 +56,7 @@ public void ThrowWhenWrong() { if (Environment.GetEnvironmentVariable(TfmEnvVarName) != moniker.ToString()) { - throw new InvalidOperationException($"Has not been recompiled, the value was {Environment.GetEnvironmentVariable(TfmEnvVarName)}"); + throw new InvalidOperationException($"Has not been recompiled, the value was {moniker}, expected {Environment.GetEnvironmentVariable(TfmEnvVarName)}"); } } } diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj index 136d505b6f..4d8d86600f 100755 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj @@ -1,8 +1,8 @@ - + BenchmarkDotNet.IntegrationTests.ManualRunning - net462;net7.0 + net462;net8.0 true BenchmarkDotNet.IntegrationTests.ManualRunning BenchmarkDotNet.IntegrationTests.ManualRunning @@ -11,6 +11,11 @@ AnyCPU true + + + $(DefineConstants);CUSTOM_PROP + + @@ -29,9 +34,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs index 4fd4108973..29bf8275dd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Extensions; @@ -13,7 +14,7 @@ using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.InProcess.Emit; using Perfolizer.Horology; -using Perfolizer.Mathematics.Thresholds; +using Perfolizer.Metrology; using Xunit; using Xunit.Abstractions; @@ -110,20 +111,17 @@ private void AssertZeroResults(Type benchmarkType, IConfig config) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); - var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; - var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds); + var cpuResolution = CpuDetector.Cpu?.MaxFrequency()?.ToResolution() ?? FallbackCpuResolutionValue; + var threshold = new NumberValue(cpuResolution.Nanoseconds).ToThreshold(); foreach (var report in summary.Reports) { - var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers(); - var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers(); + var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().Sample; + var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().Sample; - bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold); + bool isZero = ZeroMeasurementHelper.AreIndistinguishable(workloadMeasurements, overheadMeasurements, threshold); Assert.True(isZero, $"Actual time was not 0."); - isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold); - Assert.True(isZero, "Overhead took more time than workload."); - Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); } } @@ -166,20 +164,17 @@ private void AssertDifferentSizedStructsResults(IConfig config) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); - var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; - var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds); + var cpuResolution = CpuDetector.Cpu?.MaxFrequency()?.ToResolution() ?? FallbackCpuResolutionValue; + var threshold = (cpuResolution / 2).ToThreshold(); foreach (var report in summary.Reports) { - var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers(); - var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers(); + var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().Sample; + var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().Sample; - bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold); + bool isZero = ZeroMeasurementHelper.AreIndistinguishable(workloadMeasurements, overheadMeasurements, threshold); Assert.False(isZero, $"Actual time was 0."); - isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold); - Assert.True(isZero, "Overhead took more time than workload."); - Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0."); } } @@ -197,16 +192,12 @@ public struct Struct32 public struct Struct64 { - public long l1, l2, l3, l4, - l5, l6, l7, l8; + public long l1, l2, l3, l4, l5, l6, l7, l8; } public struct Struct128 { - public long l1, l2, l3, l4, - l5, l6, l7, l8, - l9, l10, l11, l12, - l13, l14, l15, l16; + public long l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16; } public class DifferentSizedStructs @@ -218,17 +209,72 @@ public class DifferentSizedStructs } } -public class EmptyVoid { [Benchmark] public void Void() { } } -public class EmptyByte { [Benchmark] public byte Byte() => default; } -public class EmptySByte { [Benchmark] public sbyte SByte() => default; } -public class EmptyShort { [Benchmark] public short Short() => default; } -public class EmptyUShort { [Benchmark] public ushort UShort() => default; } -public class EmptyChar { [Benchmark] public char Char() => default; } -public class EmptyInt32 { [Benchmark] public int Int32() => default; } -public class EmptyUInt32 { [Benchmark] public uint UInt32() => default; } -public class EmptyInt64 { [Benchmark] public long Int64() => default; } -public class EmptyUInt64 { [Benchmark] public ulong UInt64() => default; } -public class EmptyIntPtr { [Benchmark] public IntPtr IntPtr() => default; } -public class EmptyUIntPtr { [Benchmark] public UIntPtr UIntPtr() => default; } -public class EmptyVoidPointer { [Benchmark] public unsafe void* VoidPointer() => default; } -public class EmptyClass { [Benchmark] public object Class() => default; } \ No newline at end of file +public class EmptyVoid +{ + [Benchmark] public void Void() { } +} + +public class EmptyByte +{ + [Benchmark] public byte Byte() => default; +} + +public class EmptySByte +{ + [Benchmark] public sbyte SByte() => default; +} + +public class EmptyShort +{ + [Benchmark] public short Short() => default; +} + +public class EmptyUShort +{ + [Benchmark] public ushort UShort() => default; +} + +public class EmptyChar +{ + [Benchmark] public char Char() => default; +} + +public class EmptyInt32 +{ + [Benchmark] public int Int32() => default; +} + +public class EmptyUInt32 +{ + [Benchmark] public uint UInt32() => default; +} + +public class EmptyInt64 +{ + [Benchmark] public long Int64() => default; +} + +public class EmptyUInt64 +{ + [Benchmark] public ulong UInt64() => default; +} + +public class EmptyIntPtr +{ + [Benchmark] public IntPtr IntPtr() => default; +} + +public class EmptyUIntPtr +{ + [Benchmark] public UIntPtr UIntPtr() => default; +} + +public class EmptyVoidPointer +{ + [Benchmark] public unsafe void* VoidPointer() => default; +} + +public class EmptyClass +{ + [Benchmark] public object Class() => default; +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalNativeAotToolchainTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalNativeAotToolchainTests.cs index c8740a3829..64f469efc7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalNativeAotToolchainTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalNativeAotToolchainTests.cs @@ -13,7 +13,7 @@ namespace BenchmarkDotNet.IntegrationTests.ManualRunning /// to run these tests please clone and build NativeAOT first, /// then update the hardcoded path /// and run following command from console: - /// dotnet test -c Release -f netcoreapp2.1 --filter "FullyQualifiedName~BenchmarkDotNet.IntegrationTests.ManualRunning.LocalNativeAotToolchainTests" + /// dotnet test -c Release -f net8.0 --filter "FullyQualifiedName~BenchmarkDotNet.IntegrationTests.ManualRunning.LocalNativeAotToolchainTests" /// /// in perfect world we would do this OOB for you, but building NativeAOT /// so it's not part of our CI jobs @@ -29,7 +29,7 @@ public void CanBenchmarkLocalBuildUsingRyuJit() { var config = ManualConfig.CreateEmpty() .AddJob(Job.Dry - .WithRuntime(NativeAotRuntime.Net60) + .WithRuntime(NativeAotRuntime.Net80) .WithToolchain( NativeAotToolchain.CreateBuilder() .UseLocalBuild(new System.IO.DirectoryInfo(IlcPath)) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/MsBuildArgumentTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/MsBuildArgumentTests.cs new file mode 100644 index 0000000000..f4619040cc --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/MsBuildArgumentTests.cs @@ -0,0 +1,65 @@ +using System; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using Xunit; + +namespace BenchmarkDotNet.IntegrationTests.ManualRunning +{ + public class MsBuildArgumentTests : BenchmarkTestExecutor + { + private const string CustomPropEnvVarName = "CustomPropEnvVarName"; + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ProcessIsBuiltWithCustomProperty(bool setCustomProperty) + { + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry + .WithArguments(new Argument[] { new MsBuildArgument($"/p:CustomProp={setCustomProperty}") }) + .WithEnvironmentVariable(CustomPropEnvVarName, setCustomProperty.ToString()) + ); + CanExecute(config); + } + + [Fact] + public void MultipleProcessesAreBuiltWithCorrectProperties() + { + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry + .WithArguments(new Argument[] { new MsBuildArgument($"/p:CustomProp={true}") }) + .WithEnvironmentVariable(CustomPropEnvVarName, true.ToString()) + ) + .AddJob(Job.Dry + .WithRuntime(NativeAotRuntime.Net80) + .WithArguments(new Argument[] { new MsBuildArgument($"/p:CustomProp={true}") }) + .WithEnvironmentVariable(CustomPropEnvVarName, true.ToString()) + ) + .AddJob(Job.Dry + .WithEnvironmentVariable(CustomPropEnvVarName, false.ToString()) + ); + CanExecute(config); + } + + public class PropertyDefine + { + private const bool customPropWasSet = +#if CUSTOM_PROP + true; +#else + false; +#endif + + [Benchmark] + public void ThrowWhenWrong() + { + if (Environment.GetEnvironmentVariable(CustomPropEnvVarName) != customPropWasSet.ToString()) + { + throw new InvalidOperationException($"Custom property was not set properly, the expected value was {Environment.GetEnvironmentVariable(CustomPropEnvVarName)}"); + } + } + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeEmpty.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeEmpty.cs deleted file mode 100644 index f8607e4263..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeEmpty.cs +++ /dev/null @@ -1,85 +0,0 @@ -using BenchmarkDotNet.Attributes; - -namespace BenchmarkDotNet.IntegrationTests.ManualRunning.Smoke -{ - [MedianColumn, Q3Column, MaxColumn] - [LegacyJitX64Job, RyuJitX64Job, MonoJob] - [KeepBenchmarkFiles] - public class SmokeEmpty - { - [Benchmark] public void Void1() {} - [Benchmark] public void Void2() {} - [Benchmark] public void Void3() {} - [Benchmark] public void Void4() {} - - [Benchmark] public byte Byte1() => 0; - [Benchmark] public byte Byte2() => 0; - [Benchmark] public byte Byte3() => 0; - [Benchmark] public byte Byte4() => 0; - - [Benchmark] public sbyte Sbyte1() => 0; - [Benchmark] public sbyte Sbyte2() => 0; - [Benchmark] public sbyte Sbyte3() => 0; - [Benchmark] public sbyte Sbyte4() => 0; - - [Benchmark] public short Short1() => 0; - [Benchmark] public short Short2() => 0; - [Benchmark] public short Short3() => 0; - [Benchmark] public short Short4() => 0; - - [Benchmark] public ushort Ushort1() => 0; - [Benchmark] public ushort Ushort2() => 0; - [Benchmark] public ushort Ushort3() => 0; - [Benchmark] public ushort Ushort4() => 0; - - [Benchmark] public int Int1() => 0; - [Benchmark] public int Int2() => 0; - [Benchmark] public int Int3() => 0; - [Benchmark] public int Int4() => 0; - - [Benchmark] public uint Uint1() => 0u; - [Benchmark] public uint Uint2() => 0u; - [Benchmark] public uint Uint3() => 0u; - [Benchmark] public uint Uint4() => 0u; - - [Benchmark] public bool Bool1() => false; - [Benchmark] public bool Bool2() => false; - [Benchmark] public bool Bool3() => false; - [Benchmark] public bool Bool4() => false; - - [Benchmark] public char Char1() => 'a'; - [Benchmark] public char Char2() => 'a'; - [Benchmark] public char Char3() => 'a'; - [Benchmark] public char Char4() => 'a'; - - [Benchmark] public float Float1() => 0f; - [Benchmark] public float Float2() => 0f; - [Benchmark] public float Float3() => 0f; - [Benchmark] public float Float4() => 0f; - - [Benchmark] public double Double1() => 0d; - [Benchmark] public double Double2() => 0d; - [Benchmark] public double Double3() => 0d; - [Benchmark] public double Double4() => 0d; - - [Benchmark] public long Long1() => 0L; - [Benchmark] public long Long2() => 0L; - [Benchmark] public long Long3() => 0L; - [Benchmark] public long Long4() => 0L; - - [Benchmark] public ulong Ulong1() => 0uL; - [Benchmark] public ulong Ulong2() => 0uL; - [Benchmark] public ulong Ulong3() => 0uL; - [Benchmark] public ulong Ulong4() => 0uL; - - [Benchmark] public string String1() => ""; - [Benchmark] public string String2() => ""; - [Benchmark] public string String3() => ""; - [Benchmark] public string String4() => ""; - - [Benchmark] public object Object1() => null; - [Benchmark] public object Object2() => null; - [Benchmark] public object Object3() => null; - [Benchmark] public object Object4() => null; - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeValueTypes.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeValueTypes.cs deleted file mode 100644 index 34b93a6d75..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeValueTypes.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System; -using System.Threading.Tasks; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Environments; - -namespace BenchmarkDotNet.IntegrationTests.ManualRunning.Smoke -{ - // ReSharper disable InconsistentNaming - - [RyuJitX64Job, LegacyJitX64Job, LegacyJitX86Job] - [MemoryDiagnoser] - public class SmokeValueTypes - { - [Benchmark] public Jit ReturnEnum() => Jit.RyuJit; - - [Benchmark] public DateTime ReturnDateTime() => new DateTime(); - - [Benchmark] public DateTime? ReturnNullableDateTime() => new DateTime(); - [Benchmark] public int? ReturnNullableInt() => 0; - - public struct StructWithReferencesOnly { public object _ref; } - [Benchmark] public StructWithReferencesOnly ReturnStructWithReferencesOnly() => new StructWithReferencesOnly(); - - public struct EmptyStruct { } - [Benchmark] public EmptyStruct ReturnEmptyStruct() => new EmptyStruct(); - - [Benchmark] public ValueTuple ReturnGenericStructOfValueType() => new ValueTuple(0); - [Benchmark] public ValueTuple ReturnGenericStructOfReferenceType() => new ValueTuple(null); - - [Benchmark] public ValueTask ReturnValueTaskOfValueType() => new ValueTask(0); - [Benchmark] public ValueTask ReturnValueTaskOfReferenceType() => new ValueTask(result: null); - - [Benchmark] public byte ReturnByte() => 0; - public struct Byte1 { public byte _1; } - [Benchmark] public Byte1 ReturnByte1() => new Byte1(); - public struct Byte2 { public byte _1, _2; } - [Benchmark] public Byte2 ReturnByte2() => new Byte2(); - public struct Byte3 { public byte _1, _2, _3; } - [Benchmark] public Byte3 ReturnByte3() => new Byte3(); - public struct Byte4 { public byte _1, _2, _3, _4; } - [Benchmark] public Byte4 ReturnByte4() => new Byte4(); - - [Benchmark] public short ReturnShort() => 0; - public struct Short1 { public short _1; } - [Benchmark] public Short1 ReturnShort1() => new Short1(); - public struct Short2 { public short _1, _2; } - [Benchmark] public Short2 ReturnShort2() => new Short2(); - public struct Short3 { public short _1, _2, _3; } - [Benchmark] public Short3 ReturnShort3() => new Short3(); - public struct Short4 { public short _1, _2, _3, _4; } - [Benchmark] public Short4 ReturnShort4() => new Short4(); - - [Benchmark] public int ReturnInt() => 0; - public struct Int1 { public int _1; } - [Benchmark] public Int1 ReturnInt1() => new Int1(); - public struct Int2 { public int _1, _2; } - [Benchmark] public Int2 ReturnInt2() => new Int2(); - public struct Int3 { public int _1, _2, _3; } - [Benchmark] public Int3 ReturnInt3() => new Int3(); - public struct Int4 { public int _1, _2, _3, _4; } - [Benchmark] public Int4 ReturnInt4() => new Int4(); - - [Benchmark] public long ReturnLong() => 0; - public struct Long1 { public long _1; } - [Benchmark] public Long1 ReturnLong1() => new Long1(); - public struct Long2 { public long _1, _2; } - [Benchmark] public Long2 ReturnLong2() => new Long2(); - public struct Long3 { public long _1, _2, _3; } - [Benchmark] public Long3 ReturnLong3() => new Long3(); - public struct Long4 { public long _1, _2, _3, _4; } - [Benchmark] public Long4 ReturnLong4() => new Long4(); - } - // ReSharper restore InconsistentNaming -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.Static/BenchmarkDotNet.IntegrationTests.Static.csproj b/tests/BenchmarkDotNet.IntegrationTests.Static/BenchmarkDotNet.IntegrationTests.Static.csproj index 4a52743deb..0944134348 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.Static/BenchmarkDotNet.IntegrationTests.Static.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.Static/BenchmarkDotNet.IntegrationTests.Static.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet.IntegrationTests.Static - net462;net7.0 + net462;net8.0 true BenchmarkDotNet.IntegrationTests.Static BenchmarkDotNet.IntegrationTests.Static @@ -14,7 +14,7 @@ - + \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.VisualBasic/BenchmarkDotNet.IntegrationTests.VisualBasic.vbproj b/tests/BenchmarkDotNet.IntegrationTests.VisualBasic/BenchmarkDotNet.IntegrationTests.VisualBasic.vbproj index 973fbb08c2..0a456372e8 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.VisualBasic/BenchmarkDotNet.IntegrationTests.VisualBasic.vbproj +++ b/tests/BenchmarkDotNet.IntegrationTests.VisualBasic/BenchmarkDotNet.IntegrationTests.VisualBasic.vbproj @@ -5,7 +5,7 @@ - net462;net7.0 + net462;net8.0 diff --git a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs index 60bdd52845..459c23c2c2 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs @@ -48,13 +48,20 @@ public AllSetupAndCleanupTest(ITestOutputHelper output) : base(output) { } private static string[] GetActualLogLines(Summary summary) => GetSingleStandardOutput(summary).Where(line => line.StartsWith(Prefix)).ToArray(); - [Fact] - public void AllSetupAndCleanupMethodRunsTest() + [Theory] + [InlineData(typeof(AllSetupAndCleanupAttributeBenchmarks))] + [InlineData(typeof(AllSetupAndCleanupAttributeBenchmarksTask))] + [InlineData(typeof(AllSetupAndCleanupAttributeBenchmarksGenericTask))] + [InlineData(typeof(AllSetupAndCleanupAttributeBenchmarksValueTask))] + [InlineData(typeof(AllSetupAndCleanupAttributeBenchmarksGenericValueTask))] + [InlineData(typeof(AllSetupAndCleanupAttributeBenchmarksValueTaskSource))] + [InlineData(typeof(AllSetupAndCleanupAttributeBenchmarksGenericValueTaskSource))] + public void AllSetupAndCleanupMethodRunsTest(Type benchmarkType) { var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); var config = CreateSimpleConfig(job: miniJob); - var summary = CanExecute(config); + var summary = CanExecute(benchmarkType, config); var actualLogLines = GetActualLogLines(summary); foreach (string line in actualLogLines) @@ -83,21 +90,7 @@ public class AllSetupAndCleanupAttributeBenchmarks public void Benchmark() => Console.WriteLine(BenchmarkCalled); } - [Fact] - public void AllSetupAndCleanupMethodRunsAsyncTest() - { - var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(job: miniJob); - - var summary = CanExecute(config); - - var actualLogLines = GetActualLogLines(summary); - foreach (string line in actualLogLines) - Output.WriteLine(line); - Assert.Equal(expectedLogLines, actualLogLines); - } - - public class AllSetupAndCleanupAttributeBenchmarksAsync + public class AllSetupAndCleanupAttributeBenchmarksTask { private int setupCounter; private int cleanupCounter; @@ -115,24 +108,10 @@ public class AllSetupAndCleanupAttributeBenchmarksAsync public Task GlobalCleanup() => Console.Out.WriteLineAsync(GlobalCleanupCalled); [Benchmark] - public Task Benchmark() => Console.Out.WriteLineAsync(BenchmarkCalled); - } - - [Fact] - public void AllSetupAndCleanupMethodRunsAsyncTaskSetupTest() - { - var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(job: miniJob); - - var summary = CanExecute(config); - - var actualLogLines = GetActualLogLines(summary); - foreach (string line in actualLogLines) - Output.WriteLine(line); - Assert.Equal(expectedLogLines, actualLogLines); + public void Benchmark() => Console.WriteLine(BenchmarkCalled); } - public class AllSetupAndCleanupAttributeBenchmarksAsyncTaskSetup + public class AllSetupAndCleanupAttributeBenchmarksGenericTask { private int setupCounter; private int cleanupCounter; @@ -144,30 +123,47 @@ public class AllSetupAndCleanupAttributeBenchmarksAsyncTaskSetup public void IterationCleanup() => Console.WriteLine(IterationCleanupCalled + " (" + ++cleanupCounter + ")"); [GlobalSetup] - public Task GlobalSetup() => Console.Out.WriteLineAsync(GlobalSetupCalled); + public async Task GlobalSetup() + { + await Console.Out.WriteLineAsync(GlobalSetupCalled); + + return 42; + } [GlobalCleanup] - public Task GlobalCleanup() => Console.Out.WriteLineAsync(GlobalCleanupCalled); + public async Task GlobalCleanup() + { + await Console.Out.WriteLineAsync(GlobalCleanupCalled); + + return 42; + } [Benchmark] public void Benchmark() => Console.WriteLine(BenchmarkCalled); } - [Fact] - public void AllSetupAndCleanupMethodRunsAsyncGenericTaskSetupTest() + public class AllSetupAndCleanupAttributeBenchmarksValueTask { - var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(job: miniJob); + private int setupCounter; + private int cleanupCounter; - var summary = CanExecute(config); + [IterationSetup] + public void IterationSetup() => Console.WriteLine(IterationSetupCalled + " (" + ++setupCounter + ")"); - var actualLogLines = GetActualLogLines(summary); - foreach (string line in actualLogLines) - Output.WriteLine(line); - Assert.Equal(expectedLogLines, actualLogLines); + [IterationCleanup] + public void IterationCleanup() => Console.WriteLine(IterationCleanupCalled + " (" + ++cleanupCounter + ")"); + + [GlobalSetup] + public ValueTask GlobalSetup() => new ValueTask(Console.Out.WriteLineAsync(GlobalSetupCalled)); + + [GlobalCleanup] + public ValueTask GlobalCleanup() => new ValueTask(Console.Out.WriteLineAsync(GlobalCleanupCalled)); + + [Benchmark] + public void Benchmark() => Console.WriteLine(BenchmarkCalled); } - public class AllSetupAndCleanupAttributeBenchmarksAsyncGenericTaskSetup + public class AllSetupAndCleanupAttributeBenchmarksGenericValueTask { private int setupCounter; private int cleanupCounter; @@ -179,7 +175,7 @@ public class AllSetupAndCleanupAttributeBenchmarksAsyncGenericTaskSetup public void IterationCleanup() => Console.WriteLine(IterationCleanupCalled + " (" + ++cleanupCounter + ")"); [GlobalSetup] - public async Task GlobalSetup() + public async ValueTask GlobalSetup() { await Console.Out.WriteLineAsync(GlobalSetupCalled); @@ -187,7 +183,7 @@ public async Task GlobalSetup() } [GlobalCleanup] - public async Task GlobalCleanup() + public async ValueTask GlobalCleanup() { await Console.Out.WriteLineAsync(GlobalCleanupCalled); @@ -198,22 +194,9 @@ public async Task GlobalCleanup() public void Benchmark() => Console.WriteLine(BenchmarkCalled); } - [Fact] - public void AllSetupAndCleanupMethodRunsAsyncValueTaskSetupTest() - { - var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(job: miniJob); - - var summary = CanExecute(config); - - var actualLogLines = GetActualLogLines(summary); - foreach (string line in actualLogLines) - Output.WriteLine(line); - Assert.Equal(expectedLogLines, actualLogLines); - } - - public class AllSetupAndCleanupAttributeBenchmarksAsyncValueTaskSetup + public class AllSetupAndCleanupAttributeBenchmarksValueTaskSource { + private readonly ValueTaskSource valueTaskSource = new (); private int setupCounter; private int cleanupCounter; @@ -224,31 +207,28 @@ public class AllSetupAndCleanupAttributeBenchmarksAsyncValueTaskSetup public void IterationCleanup() => Console.WriteLine(IterationCleanupCalled + " (" + ++cleanupCounter + ")"); [GlobalSetup] - public ValueTask GlobalSetup() => new ValueTask(Console.Out.WriteLineAsync(GlobalSetupCalled)); + public ValueTask GlobalSetup() + { + valueTaskSource.Reset(); + Console.Out.WriteLineAsync(GlobalSetupCalled).ContinueWith(_ => valueTaskSource.SetResult(42)); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } [GlobalCleanup] - public ValueTask GlobalCleanup() => new ValueTask(Console.Out.WriteLineAsync(GlobalCleanupCalled)); + public ValueTask GlobalCleanup() + { + valueTaskSource.Reset(); + Console.Out.WriteLineAsync(GlobalCleanupCalled).ContinueWith(_ => valueTaskSource.SetResult(42)); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } [Benchmark] public void Benchmark() => Console.WriteLine(BenchmarkCalled); } - [FactEnvSpecific(EnvRequirement.NonWindows)] - public void AllSetupAndCleanupMethodRunsAsyncGenericValueTaskSetupTest() - { - var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(job: miniJob); - - var summary = CanExecute(config); - - var actualLogLines = GetActualLogLines(summary); - foreach (string line in actualLogLines) - Output.WriteLine(line); - Assert.Equal(expectedLogLines, actualLogLines); - } - - public class AllSetupAndCleanupAttributeBenchmarksAsyncGenericValueTaskSetup + public class AllSetupAndCleanupAttributeBenchmarksGenericValueTaskSource { + private readonly ValueTaskSource valueTaskSource = new (); private int setupCounter; private int cleanupCounter; @@ -259,19 +239,19 @@ public class AllSetupAndCleanupAttributeBenchmarksAsyncGenericValueTaskSetup public void IterationCleanup() => Console.WriteLine(IterationCleanupCalled + " (" + ++cleanupCounter + ")"); [GlobalSetup] - public async ValueTask GlobalSetup() + public ValueTask GlobalSetup() { - await Console.Out.WriteLineAsync(GlobalSetupCalled); - - return 42; + valueTaskSource.Reset(); + Console.Out.WriteLineAsync(GlobalSetupCalled).ContinueWith(_ => valueTaskSource.SetResult(42)); + return new ValueTask(valueTaskSource, valueTaskSource.Token); } [GlobalCleanup] - public async ValueTask GlobalCleanup() + public ValueTask GlobalCleanup() { - await Console.Out.WriteLineAsync(GlobalCleanupCalled); - - return 42; + valueTaskSource.Reset(); + Console.Out.WriteLineAsync(GlobalCleanupCalled).ContinueWith(_ => valueTaskSource.SetResult(42)); + return new ValueTask(valueTaskSource, valueTaskSource.Token); } [Benchmark] diff --git a/tests/BenchmarkDotNet.IntegrationTests/AppBundle/test-main.js b/tests/BenchmarkDotNet.IntegrationTests/AppBundle/test-main.js new file mode 100644 index 0000000000..c9f705d547 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/AppBundle/test-main.js @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +import { dotnet } from './_framework/dotnet.js' + +await dotnet + .withDiagnosticTracing(false) + .withApplicationArguments(...arguments) + .run() diff --git a/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs b/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs index 8eb149b5dc..d795b1a102 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs @@ -1,10 +1,44 @@ -using System.Threading.Tasks; +using System; +using System.Threading.Tasks; +using System.Threading.Tasks.Sources; using BenchmarkDotNet.Attributes; using Xunit; using Xunit.Abstractions; namespace BenchmarkDotNet.IntegrationTests { + internal class ValueTaskSource : IValueTaskSource, IValueTaskSource + { + private ManualResetValueTaskSourceCore _core; + + T IValueTaskSource.GetResult(short token) => _core.GetResult(token); + void IValueTaskSource.GetResult(short token) => _core.GetResult(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _core.GetStatus(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _core.GetStatus(token); + void IValueTaskSource.OnCompleted(Action continuation, object state, short token, ValueTaskSourceOnCompletedFlags flags) => _core.OnCompleted(continuation, state, token, flags); + void IValueTaskSource.OnCompleted(Action continuation, object state, short token, ValueTaskSourceOnCompletedFlags flags) => _core.OnCompleted(continuation, state, token, flags); + public void Reset() => _core.Reset(); + public short Token => _core.Version; + public void SetResult(T result) => _core.SetResult(result); + } + + // This is used to test the case of ValueTaskAwaiter.IsCompleted returns false, then OnCompleted invokes the callback immediately because it happened to complete between the 2 calls. + internal class ValueTaskSourceCallbackOnly : IValueTaskSource, IValueTaskSource + { + private ManualResetValueTaskSourceCore _core; + + T IValueTaskSource.GetResult(short token) => _core.GetResult(token); + void IValueTaskSource.GetResult(short token) => _core.GetResult(token); + // Always return pending state so OnCompleted will be called. + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => ValueTaskSourceStatus.Pending; + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => ValueTaskSourceStatus.Pending; + void IValueTaskSource.OnCompleted(Action continuation, object state, short token, ValueTaskSourceOnCompletedFlags flags) => _core.OnCompleted(continuation, state, token, flags); + void IValueTaskSource.OnCompleted(Action continuation, object state, short token, ValueTaskSourceOnCompletedFlags flags) => _core.OnCompleted(continuation, state, token, flags); + public void Reset() => _core.Reset(); + public short Token => _core.Version; + public void SetResult(T result) => _core.SetResult(result); + } + public class AsyncBenchmarksTests : BenchmarkTestExecutor { public AsyncBenchmarksTests(ITestOutputHelper output) : base(output) { } @@ -24,8 +58,13 @@ public void TaskReturningMethodsAreAwaited() } } + [Fact] + public void TaskReturningMethodsAreAwaited_AlreadyComplete() => CanExecute(); + public class TaskDelayMethods { + private readonly ValueTaskSource valueTaskSource = new (); + private const int MillisecondsDelay = 100; internal const double NanosecondsDelay = MillisecondsDelay * 1e+6; @@ -39,6 +78,17 @@ public class TaskDelayMethods [Benchmark] public ValueTask ReturningValueTask() => new ValueTask(Task.Delay(MillisecondsDelay)); + [Benchmark] + public ValueTask ReturningValueTaskBackByIValueTaskSource() + { + valueTaskSource.Reset(); + Task.Delay(MillisecondsDelay).ContinueWith(_ => + { + valueTaskSource.SetResult(default); + }); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + [Benchmark] public async Task Awaiting() => await Task.Delay(MillisecondsDelay); @@ -47,6 +97,70 @@ public class TaskDelayMethods [Benchmark] public ValueTask ReturningGenericValueTask() => new ValueTask(ReturningGenericTask()); + + [Benchmark] + public ValueTask ReturningGenericValueTaskBackByIValueTaskSource() + { + valueTaskSource.Reset(); + Task.Delay(MillisecondsDelay).ContinueWith(_ => + { + valueTaskSource.SetResult(default); + }); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + } + + public class TaskImmediateMethods + { + private readonly ValueTaskSource valueTaskSource = new (); + private readonly ValueTaskSourceCallbackOnly valueTaskSourceCallbackOnly = new (); + + [Benchmark] + public Task ReturningTask() => Task.CompletedTask; + + [Benchmark] + public ValueTask ReturningValueTask() => new ValueTask(); + + [Benchmark] + public ValueTask ReturningValueTaskBackByIValueTaskSource() + { + valueTaskSource.Reset(); + valueTaskSource.SetResult(default); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + + [Benchmark] + public ValueTask ReturningValueTaskBackByIValueTaskSource_ImmediateCallback() + { + valueTaskSourceCallbackOnly.Reset(); + valueTaskSourceCallbackOnly.SetResult(default); + return new ValueTask(valueTaskSourceCallbackOnly, valueTaskSourceCallbackOnly.Token); + } + + [Benchmark] + public async Task Awaiting() => await Task.CompletedTask; + + [Benchmark] + public Task ReturningGenericTask() => ReturningTask().ContinueWith(_ => default(int)); + + [Benchmark] + public ValueTask ReturningGenericValueTask() => new ValueTask(ReturningGenericTask()); + + [Benchmark] + public ValueTask ReturningGenericValueTaskBackByIValueTaskSource() + { + valueTaskSource.Reset(); + valueTaskSource.SetResult(default); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + + [Benchmark] + public ValueTask ReturningGenericValueTaskBackByIValueTaskSource_ImmediateCallback() + { + valueTaskSourceCallbackOnly.Reset(); + valueTaskSourceCallbackOnly.SetResult(default); + return new ValueTask(valueTaskSourceCallbackOnly, valueTaskSourceCallbackOnly.Token); + } } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj index 378822308f..16f12c41b4 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj @@ -2,8 +2,8 @@ BenchmarkDotNet.IntegrationTests - net462;net7.0 - net7.0 + net462;net8.0 + net8.0 true BenchmarkDotNet.IntegrationTests BenchmarkDotNet.IntegrationTests @@ -29,12 +29,14 @@ + - - - - + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -46,7 +48,7 @@ - + @@ -55,6 +57,7 @@ + diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs index 695a3baeb6..622aee3ffd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs @@ -66,7 +66,7 @@ public void WhenInvalidTypeIsProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAre .Run(new[] { "--filter", "*" }, config); Assert.Empty(summaries); - Assert.Contains("Type BenchmarkDotNet.IntegrationTests.ClassC is invalid.", logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(ClassC)), logger.GetLog()); } [Fact] @@ -80,7 +80,7 @@ public void WhenNoTypesAreProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAreExe .Run(new[] { "--filter", "*" }, config); Assert.Empty(summaries); - Assert.Contains("No benchmarks to choose from. Make sure you provided public non-sealed non-static types with public [Benchmark] methods.", logger.GetLog()); + Assert.Contains("No benchmarks were found.", logger.GetLog()); } [Fact] @@ -135,7 +135,7 @@ public void WhenDisableLogFileWeDontWriteToFile() var logger = new OutputLogger(Output); var config = ManualConfig.CreateEmpty().AddLogger(logger).WithOptions(ConfigOptions.DisableLogFile).AddJob(Job.Dry); - string logFilePath = null; + string? logFilePath = null; try { var summaries = BenchmarkSwitcher @@ -161,7 +161,7 @@ public void EnsureLogFileIsWritten() var logger = new OutputLogger(Output); var config = ManualConfig.CreateEmpty().AddLogger(logger).AddJob(Job.Dry); - string logFilePath = null; + string? logFilePath = null; try { var summaries = BenchmarkSwitcher @@ -369,6 +369,11 @@ public IReadOnlyList AskUser(IReadOnlyList allTypes, ILogger logger) return returnValue; } } + + private string GetValidationErrorForType(Type type) + { + return $"No [Benchmark] attribute found on '{type.Name}' benchmark case."; + } } } @@ -424,6 +429,8 @@ public override void ExportToLog(Summary summary, ILogger logger) exported = true; } } + + } namespace BenchmarkDotNet.NOTIntegrationTests diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs index 1171b0f014..e835065b27 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs @@ -36,7 +36,7 @@ protected BenchmarkTestExecutor(ITestOutputHelper output) /// Optional custom config to be used instead of the default /// Optional: disable validation (default = true/enabled) /// The summary from the benchmark run - public Reports.Summary CanExecute(IConfig config = null, bool fullValidation = true) + public Reports.Summary CanExecute(IConfig? config = null, bool fullValidation = true) { return CanExecute(typeof(TBenchmark), config, fullValidation); } @@ -50,7 +50,7 @@ public Reports.Summary CanExecute(IConfig config = null, bool fullVa /// Optional custom config to be used instead of the default /// Optional: disable validation (default = true/enabled) /// The summary from the benchmark run - public Reports.Summary CanExecute(Type type, IConfig config = null, bool fullValidation = true) + public Reports.Summary CanExecute(Type type, IConfig? config = null, bool fullValidation = true) { // Add logging, so the Benchmark execution is in the TestRunner output (makes Debugging easier) if (config == null) @@ -90,7 +90,7 @@ public Reports.Summary CanExecute(Type type, IConfig config = null, bool fullVal return summary; } - protected IConfig CreateSimpleConfig(OutputLogger logger = null, Job job = null) + protected IConfig CreateSimpleConfig(OutputLogger? logger = null, Job? job = null) { var baseConfig = job == null ? (IConfig)new SingleRunFastConfig() : new SingleJobConfig(job); return baseConfig diff --git a/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs b/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs index b495489d78..101fa50139 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs @@ -1,6 +1,7 @@ using System; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -19,7 +20,7 @@ public void WhenBuildTakesMoreTimeThanTheTimeoutTheBuildIsCancelled() { if (!RuntimeInformation.Is64BitPlatform()) // NativeAOT does not support 32bit yet return; - if (RuntimeInformation.IsMacOS()) + if (OsDetector.IsMacOS()) return; // currently not supported // we use NativeAOT on purpose because it takes a LOT of time to build it @@ -29,10 +30,10 @@ public void WhenBuildTakesMoreTimeThanTheTimeoutTheBuildIsCancelled() var config = ManualConfig.CreateEmpty() .WithBuildTimeout(timeout) .AddJob(Job.Dry - .WithRuntime(NativeAotRuntime.Net70) + .WithRuntime(NativeAotRuntime.Net80) .WithToolchain(NativeAotToolchain.CreateBuilder() - .UseNuGet("7.0.0", "https://api.nuget.org/v3/index.json") - .TargetFrameworkMoniker("net7.0") + .UseNuGet("8.0.0", "https://api.nuget.org/v3/index.json") + .TargetFrameworkMoniker("net8.0") .ToToolchain())); var summary = CanExecute(config, fullValidation: false); diff --git a/tests/BenchmarkDotNet.IntegrationTests/ContinuousIntegration.cs b/tests/BenchmarkDotNet.IntegrationTests/ContinuousIntegration.cs index 0f112aa328..8b8710efe5 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ContinuousIntegration.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ContinuousIntegration.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet.Portability; using System; +using BenchmarkDotNet.Detectors; namespace BenchmarkDotNet.IntegrationTests { @@ -8,7 +9,7 @@ internal static class ContinuousIntegration private static bool IsGitHubActions() => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTION")); internal static bool IsGitHubActionsOnWindows() - => RuntimeInformation.IsWindows() && IsGitHubActions(); + => OsDetector.IsWindows() && IsGitHubActions(); internal static bool IsLocalRun() => !IsGitHubActions(); } diff --git a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs index 49bd585709..26c17eed04 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Disassemblers; using BenchmarkDotNet.Engines; @@ -34,14 +35,14 @@ public static IEnumerable GetAllJits() { if (RuntimeInformation.GetCurrentPlatform() is Platform.X86 or Platform.X64) { - yield return new object[] { Jit.RyuJit, Platform.X64, CoreRuntime.Core70 }; // .NET Core x64 + yield return new object[] { Jit.RyuJit, Platform.X64, CoreRuntime.Core80 }; // .NET Core x64 } - else if (RuntimeInformation.GetCurrentPlatform() is Platform.Arm64 && RuntimeInformation.IsLinux()) + else if (RuntimeInformation.GetCurrentPlatform() is Platform.Arm64 && OsDetector.IsLinux()) { - yield return new object[] { Jit.RyuJit, Platform.Arm64, CoreRuntime.Core70 }; // .NET Core arm64 + yield return new object[] { Jit.RyuJit, Platform.Arm64, CoreRuntime.Core80 }; // .NET Core arm64 } } - if (RuntimeInformation.IsMacOS()) + if (OsDetector.IsMacOS()) { // This scope of tests is not supported on macOS // However, when the MemberData method provides no data, xUnit throws an "No data found" InvalidOperationException @@ -91,7 +92,7 @@ public void Recursive() [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, Runtime runtime) { - if (RuntimeInformation.IsMacOS()) return; // currently not supported + if (OsDetector.IsMacOS()) return; // currently not supported var disassemblyDiagnoser = new DisassemblyDiagnoser( new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); @@ -110,7 +111,7 @@ public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, Runtime run [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, Runtime runtime) { - if (RuntimeInformation.IsMacOS()) return; // currently not supported + if (OsDetector.IsMacOS()) return; // currently not supported var disassemblyDiagnoser = new DisassemblyDiagnoser( new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 1, filters: new[] { "*WithCalls*" })); @@ -135,7 +136,7 @@ public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleGenericTypes(Jit jit, Platform platform, Runtime runtime) { - if (RuntimeInformation.IsMacOS()) return; // currently not supported + if (OsDetector.IsMacOS()) return; // currently not supported var disassemblyDiagnoser = new DisassemblyDiagnoser( new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); @@ -157,7 +158,7 @@ [Benchmark] public void JustReturn() { } [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleInlinableBenchmarks(Jit jit, Platform platform, Runtime runtime) { - if (RuntimeInformation.IsMacOS()) return; // currently not supported + if (OsDetector.IsMacOS()) return; // currently not supported var disassemblyDiagnoser = new DisassemblyDiagnoser( new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); diff --git a/tests/BenchmarkDotNet.IntegrationTests/DotMemoryTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DotMemoryTests.cs new file mode 100644 index 0000000000..2f21fb9f3c --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/DotMemoryTests.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Diagnostics.dotMemory; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class DotMemoryTests : BenchmarkTestExecutor + { + public DotMemoryTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void DotMemorySmokeTest() + { + if (!OsDetector.IsWindows() && RuntimeInformation.IsMono) + { + Output.WriteLine("Skip Mono on non-Windows"); + return; + } + + var config = new ManualConfig().AddJob( + Job.Dry.WithId("ExternalProcess"), + Job.Dry.WithToolchain(InProcessEmitToolchain.Instance).WithId("InProcess") + ); + string snapshotDirectory = Path.Combine(Directory.GetCurrentDirectory(), "BenchmarkDotNet.Artifacts", "snapshots"); + if (Directory.Exists(snapshotDirectory)) + Directory.Delete(snapshotDirectory, true); + + CanExecute(config); + + Output.WriteLine("---------------------------------------------"); + Output.WriteLine("SnapshotDirectory:" + snapshotDirectory); + var snapshots = Directory.EnumerateFiles(snapshotDirectory) + .Where(filePath => Path.GetExtension(filePath).Equals(".dmw", StringComparison.OrdinalIgnoreCase)) + .Select(Path.GetFileName) + .OrderBy(fileName => fileName) + .ToList(); + Output.WriteLine("Snapshots:"); + foreach (string snapshot in snapshots) + Output.WriteLine("* " + snapshot); + Assert.Equal(4, snapshots.Count); + } + + [DotMemoryDiagnoser] + public class Benchmarks + { + [Benchmark] + public int Foo0() + { + var list = new List(); + for (int i = 0; i < 1000; i++) + list.Add(new object()); + return list.Count; + } + + [Benchmark] + public int Foo1() + { + var list = new List(); + for (int i = 0; i < 1000; i++) + list.Add(new object()); + return list.Count; + } + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/DotTraceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DotTraceTests.cs index 80f11519e8..3c40ee31a0 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DotTraceTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DotTraceTests.cs @@ -4,6 +4,7 @@ using System.Linq; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnostics.dotTrace; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -20,7 +21,7 @@ public DotTraceTests(ITestOutputHelper output) : base(output) { } [Fact] public void DotTraceSmokeTest() { - if (!RuntimeInformation.IsWindows() && RuntimeInformation.IsMono) + if (!OsDetector.IsWindows() && RuntimeInformation.IsMono) { Output.WriteLine("Skip Mono on non-Windows"); return; diff --git a/tests/BenchmarkDotNet.IntegrationTests/DryRunTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DryRunTests.cs index b9ca511236..479581c815 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DryRunTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DryRunTests.cs @@ -8,12 +8,10 @@ namespace BenchmarkDotNet.IntegrationTests { - public class DryRunTests : BenchmarkTestExecutor + public class DryRunTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) { - public DryRunTests(ITestOutputHelper output) : base(output) { } - [Fact] - public void WelchTTest() => CanExecute(); + public void BenchWithStatTest() => CanExecute(); [DryJob, StatisticalTestColumn] public class WelchTTestBench diff --git a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs new file mode 100644 index 0000000000..f9272edd2d --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs @@ -0,0 +1,313 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.EventProcessors; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.Results; +using BenchmarkDotNet.Validators; +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class EventProcessorTests + { + [Fact] + public void WhenUsingEventProcessorAndNoBenchmarks() + { + var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassEmpty) }); + Assert.Equal(2, events.Count); + Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnValidationError), events[1].EventType); + } + + [Fact] + public void WhenUsingEventProcessorOnSingleClass() + { + var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassA) }); + + Assert.Equal(13, events.Count); + + Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnEndValidationStage), events[1].EventType); + Assert.Equal(nameof(EventProcessor.OnStartBuildStage), events[2].EventType); + Assert.Equal(nameof(EventProcessor.OnBuildComplete), events[3].EventType); + Assert.Equal(nameof(EventProcessor.OnEndBuildStage), events[4].EventType); + Assert.Equal(nameof(EventProcessor.OnStartRunStage), events[5].EventType); + + var benchmarkTypeAndMethods = new List<(Type Type, string[] MethodNames)> + { + (typeof(ClassA), new[]{ nameof(ClassA.Method1), nameof(ClassA.Method2) }) + }; + + int eventIndex = 6; + foreach ((var type, var methodNames) in benchmarkTypeAndMethods) + { + Assert.Equal(nameof(EventProcessor.OnStartRunBenchmarksInType), events[eventIndex].EventType); + Assert.Equal(type, events[eventIndex++].Args[0] as Type); + + foreach (var method in methodNames) + { + var methodDescriptor = type.GetMethod(method); + Assert.Equal(nameof(EventProcessor.OnStartRunBenchmark), events[eventIndex].EventType); + Assert.Equal(methodDescriptor, (events[eventIndex++].Args[0] as BenchmarkCase).Descriptor.WorkloadMethod); + + Assert.Equal(nameof(EventProcessor.OnEndRunBenchmark), events[eventIndex].EventType); + Assert.Equal(methodDescriptor, (events[eventIndex++].Args[0] as BenchmarkCase).Descriptor.WorkloadMethod); + } + + Assert.Equal(nameof(EventProcessor.OnEndRunBenchmarksInType), events[eventIndex].EventType); + Assert.Equal(type, events[eventIndex++].Args[0] as Type); + } + + Assert.Equal(nameof(EventProcessor.OnEndRunStage), events[eventIndex].EventType); + } + + [Fact] + public void WhenUsingEventProcessorOnMultipleClasses() + { + var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassA), typeof(ClassB) }); + + Assert.Equal(23, events.Count); + + Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnEndValidationStage), events[1].EventType); + Assert.Equal(nameof(EventProcessor.OnStartBuildStage), events[2].EventType); + Assert.Equal(nameof(EventProcessor.OnBuildComplete), events[3].EventType); + Assert.Equal(nameof(EventProcessor.OnEndBuildStage), events[4].EventType); + Assert.Equal(nameof(EventProcessor.OnStartRunStage), events[5].EventType); + + var benchmarkTypeAndMethods = new List<(Type Type, string[] MethodNames)> + { + (typeof(ClassA), new[]{ nameof(ClassA.Method1), nameof(ClassA.Method2) }), + (typeof(ClassB), new[]{ nameof(ClassB.Method1), nameof(ClassB.Method2), nameof(ClassB.Method3), nameof(ClassB.Method4) }) + }; + + int eventIndex = 6; + foreach ((var type, var methodNames) in benchmarkTypeAndMethods) + { + Assert.Equal(nameof(EventProcessor.OnStartRunBenchmarksInType), events[eventIndex].EventType); + Assert.Equal(type, events[eventIndex++].Args[0] as Type); + + foreach (var method in methodNames) + { + var methodDescriptor = type.GetMethod(method); + Assert.Equal(nameof(EventProcessor.OnStartRunBenchmark), events[eventIndex].EventType); + Assert.Equal(methodDescriptor, (events[eventIndex++].Args[0] as BenchmarkCase).Descriptor.WorkloadMethod); + + Assert.Equal(nameof(EventProcessor.OnEndRunBenchmark), events[eventIndex].EventType); + Assert.Equal(methodDescriptor, (events[eventIndex++].Args[0] as BenchmarkCase).Descriptor.WorkloadMethod); + } + + Assert.Equal(nameof(EventProcessor.OnEndRunBenchmarksInType), events[eventIndex].EventType); + Assert.Equal(type, events[eventIndex++].Args[0] as Type); + } + + Assert.Equal(nameof(EventProcessor.OnEndRunStage), events[eventIndex].EventType); + } + + [Fact] + public void WhenUsingEventProcessorWithValidationErrors() + { + var validator = new ErrorAllCasesValidator(); + var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassA) }, validator); + + Assert.Equal(15, events.Count); + Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnValidationError), events[1].EventType); + Assert.Equal(typeof(ClassA).GetMethod(nameof(ClassA.Method1)), (events[1].Args[0] as ValidationError).BenchmarkCase.Descriptor.WorkloadMethod); + Assert.Equal(nameof(EventProcessor.OnValidationError), events[2].EventType); + Assert.Equal(typeof(ClassA).GetMethod(nameof(ClassA.Method2)), (events[2].Args[0] as ValidationError).BenchmarkCase.Descriptor.WorkloadMethod); + Assert.Equal(nameof(EventProcessor.OnEndValidationStage), events[3].EventType); + Assert.Equal(nameof(EventProcessor.OnStartBuildStage), events[4].EventType); + } + + [Fact] + public void WhenUsingEventProcessorWithUnsupportedBenchmark() + { + var toolchain = new AllUnsupportedToolchain(); + var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassA) }, toolchain: toolchain); + + Assert.Equal(3, events.Count); + Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnValidationError), events[1].EventType); + Assert.Equal(typeof(ClassA).GetMethod(nameof(ClassA.Method1)), (events[1].Args[0] as ValidationError).BenchmarkCase.Descriptor.WorkloadMethod); + Assert.Equal(nameof(EventProcessor.OnValidationError), events[2].EventType); + Assert.Equal(typeof(ClassA).GetMethod(nameof(ClassA.Method2)), (events[2].Args[0] as ValidationError).BenchmarkCase.Descriptor.WorkloadMethod); + } + + [Fact] + public void WhenUsingEventProcessorWithBuildFailures() + { + var toolchain = new Toolchain("Build Failure", new AllFailsGenerator(), null, null); + var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassA) }, toolchain: toolchain); + + Assert.Equal(9, events.Count); + Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnEndValidationStage), events[1].EventType); + Assert.Equal(nameof(EventProcessor.OnStartBuildStage), events[2].EventType); + Assert.Equal(nameof(EventProcessor.OnBuildComplete), events[3].EventType); + Assert.False((events[3].Args[1] as BuildResult).IsGenerateSuccess); + Assert.Equal(nameof(EventProcessor.OnEndBuildStage), events[4].EventType); + Assert.Equal(nameof(EventProcessor.OnStartRunStage), events[5].EventType); + } + + private List RunBenchmarksAndRecordEvents(Type[] types, IValidator? validator = null, IToolchain? toolchain = null) + { + var eventProcessor = new LoggingEventProcessor(); + var job = new Job(Job.Dry); + if (toolchain != null) + job.Infrastructure.Toolchain = toolchain; + + var config = new ManualConfig() + .AddJob(job) + .AddEventProcessor(eventProcessor) + .WithOptions(ConfigOptions.DisableOptimizationsValidator) + .AddExporter(new MockExporter()) // only added to prevent validation warnings about a lack of exporters + .AddLogger(ConsoleLogger.Default) + .AddColumnProvider(DefaultColumnProviders.Instance) + .AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()); + if (validator != null) + config = config.AddValidator(validator); + _ = BenchmarkRunner.Run(types, config); + return eventProcessor.Events; + } + + public class ClassA + { + [Benchmark] + public void Method1() { } + [Benchmark] + public void Method2() { } + } + + public class ClassB + { + [Benchmark] + public void Method1() { } + [Benchmark] + public void Method2() { } + [Benchmark] + public void Method3() { } + [Benchmark] + public void Method4() { } + } + + public class ClassEmpty { } + + public class ErrorAllCasesValidator : IValidator + { + public bool TreatsWarningsAsErrors => true; + + public IEnumerable Validate(ValidationParameters validationParameters) + { + foreach (var benchmark in validationParameters.Benchmarks) + yield return new ValidationError(false, "Mock Validation", benchmark); + } + } + + public class AllUnsupportedToolchain : Toolchain + { + public AllUnsupportedToolchain() : base("AllUnsupported", null, null, null) + { + } + + public override IEnumerable Validate(BenchmarkCase benchmarkCase, IResolver resolver) + { + yield return new ValidationError(true, "Unsupported Benchmark", benchmarkCase); + } + } + + public class AllFailsGenerator : IGenerator + { + public GenerateResult GenerateProject(BuildPartition buildPartition, ILogger logger, string rootArtifactsFolderPath) + { + return GenerateResult.Failure(ArtifactsPaths.Empty, new List(), new Exception("Generation Failed")); + } + } + + public class LoggingEventProcessor : EventProcessor + { + public class EventData + { + public EventData(string eventType, IReadOnlyList args) + { + EventType = eventType; + Args = args; + } + + public string EventType { get; } + public IReadOnlyList Args { get; } + } + + public List Events { get; } = new List(); + + public override void OnBuildComplete(BuildPartition buildPartition, BuildResult buildResult) + { + Events.Add(new EventData(nameof(OnBuildComplete), new object[] { buildPartition, buildResult })); + } + + public override void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkReport report) + { + Events.Add(new EventData(nameof(OnEndRunBenchmark), new object[] { benchmarkCase, report })); + } + + public override void OnEndRunBenchmarksInType(Type type, Summary summary) + { + Events.Add(new EventData(nameof(OnEndRunBenchmarksInType), new object[] { type, summary })); + } + + public override void OnStartRunBenchmark(BenchmarkCase benchmarkCase) + { + Events.Add(new EventData(nameof(OnStartRunBenchmark), new object[] { benchmarkCase })); + } + + public override void OnStartRunBenchmarksInType(Type type, IReadOnlyList benchmarks) + { + Events.Add(new EventData(nameof(OnStartRunBenchmarksInType), new object[] { type, benchmarks })); + } + + public override void OnStartBuildStage(IReadOnlyList partitions) + { + Events.Add(new EventData(nameof(OnStartBuildStage), new object[] { partitions })); + } + + public override void OnStartRunStage() + { + Events.Add(new EventData(nameof(OnStartRunStage), new object[] { })); + } + + public override void OnStartValidationStage() + { + Events.Add(new EventData(nameof(OnStartValidationStage), new object[] { })); + } + + public override void OnValidationError(ValidationError validationError) + { + Events.Add(new EventData(nameof(OnValidationError), new object[] { validationError })); + } + + public override void OnEndValidationStage() + { + Events.Add(new EventData(nameof(OnEndValidationStage), new object[] { })); + } + + public override void OnEndBuildStage() + { + Events.Add(new EventData(nameof(OnEndBuildStage), new object[] { })); + } + + public override void OnEndRunStage() + { + Events.Add(new EventData(nameof(OnEndRunStage), new object[] { })); + } + } + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs index 3a532b9a61..18328994cb 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs @@ -81,7 +81,7 @@ public void ExporterUsesFullyQualifiedTypeNameAsFileName() var exporter = new MockExporter(); var mockSummary = GetMockSummary(resultsDirectoryPath, config: null, typeof(Generic)); var expectedFilePath = $"{Path.Combine(mockSummary.ResultsDirectoryPath, "BenchmarkDotNet.IntegrationTests.Generic_Int32_")}-report.txt"; - string actualFilePath = null; + string? actualFilePath = null; try { @@ -104,7 +104,7 @@ public void ExporterUsesSummaryTitleAsFileNameWhenBenchmarksJoinedToSingleSummar var joinConfig = ManualConfig.CreateEmpty().WithOptions(ConfigOptions.JoinSummary); var mockSummary = GetMockSummary(resultsDirectoryPath, joinConfig, typeof(ClassA), typeof(ClassB)); var expectedFilePath = $"{Path.Combine(mockSummary.ResultsDirectoryPath, mockSummary.Title)}-report.txt"; - string actualFilePath = null; + string? actualFilePath = null; try { @@ -144,7 +144,7 @@ public override void ExportToLog(Summary summary, ILogger logger) } } - private ImmutableArray CreateReports(Type[] types, IConfig config = null) + private ImmutableArray CreateReports(Type[] types, IConfig? config = null) => CreateBenchmarks(types, config).Select(CreateReport).ToImmutableArray(); private BenchmarkCase[] CreateBenchmarks(Type[] types, IConfig config) diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests.T4/RunnableStructCaseBenchmark.tt b/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests.T4/RunnableStructCaseBenchmark.tt index 45dbfdad79..a0fe0c5282 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests.T4/RunnableStructCaseBenchmark.tt +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests.T4/RunnableStructCaseBenchmark.tt @@ -54,7 +54,7 @@ namespace BenchmarkDotNet.IntegrationTests.InProcess.EmitTests } }<#+ - private void EmitStructCaseBenchmark(ref int counter, string type, string argValue = null) + private void EmitStructCaseBenchmark(ref int counter, string type, string? argValue = null) { #> // ---- Begin StructCase(<#=type#>) ---- diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs index cb034920a1..b7365131af 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs @@ -10,6 +10,7 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.InProcess.Emit; @@ -65,13 +66,17 @@ private void DiffEmit(Summary summary) { // .Net Core does not support assembly saving so far // SEE https://github.com/dotnet/corefx/issues/4491 + // TODO: Use new PersistedAssemblyBuilder when BDN and tests are updated to net9.0 or newer. if (!Portability.RuntimeInformation.IsFullFramework) return; - var caseName = summary.BenchmarksCases.First().Job.ToString(); + var benchmarkCase = summary.BenchmarksCases.First(); + var caseName = $"{benchmarkCase.Descriptor.Type.Assembly.GetName().Name}-{benchmarkCase.Job.FolderInfo}"; + // The benchmark config built jobs with 2 toolchains, 1 InProcessEmit and 1 Roslyn, + // so we need to subtract 1 from the partition counter to obtain the emit output. NaiveRunnableEmitDiff.RunDiff( - $@"{caseName}.exe", - $@"{caseName}Emitted.dll", + $@"{caseName}-{BuildPartition.s_partitionCounter}.exe", + $@"{caseName}-{BuildPartition.s_partitionCounter - 1}Emitted.dll", ConsoleLogger.Default); } @@ -153,6 +158,13 @@ public async Task InvokeOnceTaskAsync() Interlocked.Increment(ref Counter); } + [Benchmark] + public async ValueTask InvokeOnceValueTaskAsync() + { + await Task.Yield(); + Interlocked.Increment(ref Counter); + } + [Benchmark] public string InvokeOnceRefType() { @@ -195,6 +207,13 @@ public static async Task InvokeOnceStaticTaskAsync() Interlocked.Increment(ref Counter); } + [Benchmark] + public static async ValueTask InvokeOnceStaticValueTaskAsync() + { + await Task.Yield(); + Interlocked.Increment(ref Counter); + } + [Benchmark] public static string InvokeOnceStaticRefType() { @@ -225,35 +244,146 @@ public static ValueTask InvokeOnceStaticValueTaskOfT() } } - [Fact] - public void InProcessEmitToolchainSupportsIterationSetupAndCleanup() + [Theory] + [InlineData(typeof(IterationSetupCleanup))] + [InlineData(typeof(GlobalSetupCleanupTask))] + [InlineData(typeof(GlobalSetupCleanupValueTask))] + [InlineData(typeof(GlobalSetupCleanupValueTaskSource))] + public void InProcessEmitToolchainSupportsSetupAndCleanup(Type benchmarkType) { var logger = new OutputLogger(Output); var config = CreateInProcessConfig(logger); - WithIterationSetupAndCleanup.SetupCounter = 0; - WithIterationSetupAndCleanup.BenchmarkCounter = 0; - WithIterationSetupAndCleanup.CleanupCounter = 0; + Counters.SetupCounter = 0; + Counters.BenchmarkCounter = 0; + Counters.CleanupCounter = 0; - var summary = CanExecute(config); + var summary = CanExecute(benchmarkType, config); - Assert.Equal(1, WithIterationSetupAndCleanup.SetupCounter); - Assert.Equal(16, WithIterationSetupAndCleanup.BenchmarkCounter); - Assert.Equal(1, WithIterationSetupAndCleanup.CleanupCounter); + Assert.Equal(1, Counters.SetupCounter); + Assert.Equal(16, Counters.BenchmarkCounter); + Assert.Equal(1, Counters.CleanupCounter); } - public class WithIterationSetupAndCleanup + private static class Counters { public static int SetupCounter, BenchmarkCounter, CleanupCounter; + } + public class IterationSetupCleanup + { [IterationSetup] - public void Setup() => Interlocked.Increment(ref SetupCounter); + public void Setup() => Interlocked.Increment(ref Counters.SetupCounter); [Benchmark] - public void Benchmark() => Interlocked.Increment(ref BenchmarkCounter); + public void Benchmark() => Interlocked.Increment(ref Counters.BenchmarkCounter); [IterationCleanup] - public void Cleanup() => Interlocked.Increment(ref CleanupCounter); + public void Cleanup() => Interlocked.Increment(ref Counters.CleanupCounter); + } + + public class GlobalSetupCleanupTask + { + [GlobalSetup] + public static async Task GlobalSetup() + { + await Task.Yield(); + Interlocked.Increment(ref Counters.SetupCounter); + } + + [GlobalCleanup] + public async Task GlobalCleanup() + { + await Task.Yield(); + Interlocked.Increment(ref Counters.CleanupCounter); + return 42; + } + + [Benchmark] + public void InvokeOnceVoid() + { + Interlocked.Increment(ref Counters.BenchmarkCounter); + } + } + + public class GlobalSetupCleanupValueTask + { + [GlobalSetup] + public static async ValueTask GlobalSetup() + { + await Task.Yield(); + Interlocked.Increment(ref Counters.SetupCounter); + } + + [GlobalCleanup] + public async ValueTask GlobalCleanup() + { + await Task.Yield(); + Interlocked.Increment(ref Counters.CleanupCounter); + return 42; + } + + [Benchmark] + public void InvokeOnceVoid() + { + Interlocked.Increment(ref Counters.BenchmarkCounter); + } + } + + public class GlobalSetupCleanupValueTaskSource + { + private readonly static ValueTaskSource valueTaskSource = new (); + + [GlobalSetup] + public static ValueTask GlobalSetup() + { + valueTaskSource.Reset(); + Task.Delay(1).ContinueWith(_ => + { + Interlocked.Increment(ref Counters.SetupCounter); + valueTaskSource.SetResult(42); + }); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + + [GlobalCleanup] + public ValueTask GlobalCleanup() + { + valueTaskSource.Reset(); + Task.Delay(1).ContinueWith(_ => + { + Interlocked.Increment(ref Counters.CleanupCounter); + valueTaskSource.SetResult(42); + }); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + + [Benchmark] + public void InvokeOnceVoid() + { + Interlocked.Increment(ref Counters.BenchmarkCounter); + } + } + + +#if NET8_0_OR_GREATER + [Fact] + public void ParamsSupportRequiredProperty() + { + var config = CreateInProcessConfig(null); + CanExecute(config); + } + + public class ParamsTestRequiredProperty + { + private const string Expected = "a"; + + [Params(Expected)] + public required string ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() => Assert.Equal(Expected, ParamProperty); } +#endif } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs index b0b3b9b07c..97c467a2bd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs @@ -123,7 +123,7 @@ private void TestInvoke(Expression> methodCall, in bool isValueTask = typeof(T).IsConstructedGenericType && typeof(T).GetGenericTypeDefinition() == typeof(ValueTask<>); - object idleExpected; + object? idleExpected; if (isValueTask) idleExpected = GetDefault(typeof(T).GetGenericArguments()[0]); else if (typeof(T).GetTypeInfo().IsValueType) @@ -180,7 +180,7 @@ private void TestInvoke(BenchmarkAction benchmarkAction, int unrollFactor, bool } } - private IConfig CreateInProcessConfig(OutputLogger logger = null) + private IConfig CreateInProcessConfig(OutputLogger? logger = null) { return new ManualConfig() .AddJob(Job.Dry.WithToolchain(new InProcessNoEmitToolchain(TimeSpan.Zero, true)).WithInvocationCount(UnrollFactor).WithUnrollFactor(UnrollFactor)) @@ -273,5 +273,26 @@ public ValueTask InvokeOnceValueTaskOfT() return new ValueTask(DecimalResult); } } + + +#if NET8_0_OR_GREATER + [Fact] + public void ParamsSupportRequiredProperty() + { + var config = CreateInProcessConfig(); + CanExecute(config); + } + + public class ParamsTestRequiredProperty + { + private const string Expected = "a"; + + [Params(Expected)] + public required string ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() => Assert.Equal(Expected, ParamProperty); + } +#endif } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs b/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs index 9a4910034c..df4427bb07 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs @@ -25,7 +25,7 @@ public JitRuntimeValidationTest(ITestOutputHelper output) : base(output) { } [InlineData(Jit.LegacyJit, Platform.X64, null)] [InlineData(Jit.RyuJit, Platform.X86, RyuJitNotAvailable)] [InlineData(Jit.RyuJit, Platform.X64, null)] - public void CheckClrOnWindows(Jit jit, Platform platform, string errorMessage) + public void CheckClrOnWindows(Jit jit, Platform platform, string? errorMessage) { Verify(ClrRuntime.Net462, jit, platform, errorMessage); } @@ -51,10 +51,10 @@ public static IEnumerable CheckCore_Arguments() [MemberData(nameof(CheckCore_Arguments))] public void CheckCore(Jit jit, Platform platform, string errorMessage) { - Verify(CoreRuntime.Core70, jit, platform, errorMessage); + Verify(CoreRuntime.Core80, jit, platform, errorMessage); } - private void Verify(Runtime runtime, Jit jit, Platform platform, string errorMessage) + private void Verify(Runtime runtime, Jit jit, Platform platform, string? errorMessage) { var logger = new OutputLogger(Output); var config = ManualConfig.CreateEmpty() diff --git a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs index 89ea7b5ba2..087d146822 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs @@ -25,7 +25,7 @@ public void BenchmarkCanAllocateMoreThan2Gb() var summary = BenchmarkRunner .Run( ManualConfig.CreateEmpty() - .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core70).WithPlatform(Platform.X64).WithId("Core")) + .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core")) .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithPlatform(Platform.X86).WithGcServer(false).WithLargeAddressAware().WithId("Framework")) .AddColumnProvider(DefaultColumnProviders.Instance) .AddLogger(new OutputLogger(output))); @@ -47,7 +47,7 @@ public void BenchmarkCanAllocateMoreThan2Gb() .Any()); Assert.Contains(".NET Framework", summary.AllRuntimes); - Assert.Contains(".NET 7.0", summary.AllRuntimes); + Assert.Contains(".NET 8.0", summary.AllRuntimes); } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index c1f2c622c6..603926c2a6 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -8,6 +8,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.IntegrationTests.Xunit; @@ -34,9 +35,6 @@ public class MemoryDiagnoserTests public static IEnumerable GetToolchains() { - if (RuntimeInformation.IsOldMono) // https://github.com/mono/mono/issues/8397 - yield break; - yield return new object[] { Job.Default.GetToolchain() }; yield return new object[] { InProcessEmitToolchain.Instance }; } @@ -72,19 +70,16 @@ public void MemoryDiagnoserIsAccurate(IToolchain toolchain) [FactEnvSpecific("We don't want to test NativeAOT twice (for .NET Framework 4.6.2 and .NET 7.0)", EnvRequirement.DotNetCoreOnly)] public void MemoryDiagnoserSupportsNativeAOT() { - if (RuntimeInformation.IsMacOS()) + if (OsDetector.IsMacOS()) return; // currently not supported - MemoryDiagnoserIsAccurate( - NativeAotToolchain.CreateBuilder() - .UseNuGet("7.0.0", "https://api.nuget.org/v3/index.json") - .ToToolchain()); + MemoryDiagnoserIsAccurate(NativeAotToolchain.Net80); } - [FactEnvSpecific("We don't want to test MonoVM twice (for .NET Framework 4.6.2 and .NET 7.0)", EnvRequirement.DotNetCoreOnly)] + [FactEnvSpecific("We don't want to test MonoVM twice (for .NET Framework 4.6.2 and .NET 8.0)", EnvRequirement.DotNetCoreOnly)] public void MemoryDiagnoserSupportsModernMono() { - MemoryDiagnoserIsAccurate(MonoToolchain.Mono70); + MemoryDiagnoserIsAccurate(MonoToolchain.Mono80); } public class AllocatingGlobalSetupAndCleanup @@ -262,8 +257,7 @@ public void Allocate() } } - [TheoryEnvSpecific(".NET Core 3.0 preview6+ exposes a GC.GetTotalAllocatedBytes method which makes it possible to work", - EnvRequirement.DotNetCore30Only)] + [Theory(Skip = "Test is flaky even in latest .Net")] [MemberData(nameof(GetToolchains))] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolchain) @@ -327,7 +321,11 @@ private IConfig CreateConfig(IToolchain toolchain) .WithGcForce(false) .WithGcServer(false) .WithGcConcurrent(false) - .WithEnvironmentVariable("COMPlus_TieredCompilation", "0") // Tiered JIT can allocate some memory on a background thread, let's disable it to make our tests less flaky (#1542) + .WithEnvironmentVariables([ + // Tiered JIT can allocate some memory on a background thread, let's disable it to make our tests less flaky (#1542) + new EnvironmentVariable("DOTNET_TieredCompilation", "0"), + new EnvironmentVariable("COMPlus_TieredCompilation", "0") + ]) .WithToolchain(toolchain)) .AddColumnProvider(DefaultColumnProviders.Instance) .AddDiagnoser(MemoryDiagnoser.Default) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs index 4313dfbec1..17309832ef 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs @@ -4,16 +4,23 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; +using Xunit.Abstractions; namespace BenchmarkDotNet.IntegrationTests { public class MonoTests : BenchmarkTestExecutor { + public MonoTests(ITestOutputHelper output) : base(output) { } + [FactEnvSpecific("UseMonoRuntime option is available in .NET Core only starting from .NET 6", EnvRequirement.DotNetCoreOnly)] - public void Mono70IsSupported() + public void Mono80IsSupported() { - var config = ManualConfig.CreateEmpty().AddJob(Job.Dry.WithRuntime(MonoRuntime.Mono70)); + var logger = new OutputLogger(Output); + var config = ManualConfig.CreateEmpty() + .AddLogger(logger) + .AddJob(Job.Dry.WithRuntime(MonoRuntime.Mono80)); CanExecute(config); } @@ -27,11 +34,11 @@ public void Check() throw new Exception("This is not Mono runtime"); } - if (RuntimeInformation.GetCurrentRuntime() != MonoRuntime.Mono70) + if (RuntimeInformation.GetCurrentRuntime() != MonoRuntime.Mono80) { throw new Exception("Incorrect runtime detection"); } } } } -} +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs b/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs index 938a1e2011..50a929bd83 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs @@ -29,7 +29,7 @@ public void SingleBenchmarkCanBeExecutedForMultipleRuntimes() var summary = BenchmarkRunner .Run( ManualConfig.CreateEmpty() - .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core70).WithPlatform(Platform.X64).WithId("Core")) + .AddJob(Job.Dry.WithRuntime(CoreRuntime.Core80).WithPlatform(Platform.X64).WithId("Core")) .AddJob(Job.Dry.WithRuntime(ClrRuntime.Net462).WithId("Framework")) .AddColumnProvider(DefaultColumnProviders.Instance) .AddLogger(new OutputLogger(output))); @@ -51,7 +51,7 @@ public void SingleBenchmarkCanBeExecutedForMultipleRuntimes() .Any()); Assert.Contains(".NET Framework", summary.AllRuntimes); - Assert.Contains(".NET 7.0", summary.AllRuntimes); + Assert.Contains(".NET 8.0", summary.AllRuntimes); } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs index ce39b0572e..169fb3a0b3 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs @@ -1,6 +1,7 @@ using System; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; @@ -22,7 +23,7 @@ public void LatestNativeAotVersionIsSupported() return; if (ContinuousIntegration.IsGitHubActionsOnWindows()) // no native dependencies installed return; - if (RuntimeInformation.IsMacOS()) + if (OsDetector.IsMacOS()) return; // currently not supported var toolchain = NativeAotToolchain.CreateBuilder().UseNuGet().IlcInstructionSet(IsAvx2Supported() ? "avx2" : "").ToToolchain(); diff --git a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs index 182acc5309..4d38427aab 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs @@ -18,7 +18,7 @@ public NuGetReferenceTests(ITestOutputHelper output) : base(output) { } - [FactEnvSpecific("For some reason this test is unstable on Ubuntu for both AzureDevOps and Travis CI", EnvRequirement.NonLinux)] + [Fact] public void UserCanSpecifyCustomNuGetPackageDependency() { var toolchain = RuntimeInformation.GetCurrentRuntime().GetToolchain(preferMsBuildToolchains: true); diff --git a/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs index 0285856bcb..0f703bf1cd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs @@ -217,5 +217,28 @@ public void Test() throw new ArgumentException($"{nameof(StaticParamProperty)} has wrong value: {StaticParamProperty}!"); } } + +#if NET8_0_OR_GREATER + [Fact] + public void ParamsSupportRequiredProperty() + { + var summary = CanExecute(); + var standardOutput = GetCombinedStandardOutput(summary); + + foreach (var param in new[] { "a", "b" }) + { + Assert.Contains($"// ### New Parameter {param} ###", standardOutput); + } + } + + public class ParamsTestRequiredProperty + { + [Params("a", "b")] + public required string ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() => Console.WriteLine($"// ### New Parameter {ParamProperty} ###"); + } +#endif } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/PowerRequest.cs b/tests/BenchmarkDotNet.IntegrationTests/PowerRequest.cs new file mode 100644 index 0000000000..676ad23d40 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/PowerRequest.cs @@ -0,0 +1,9 @@ +namespace BenchmarkDotNet.IntegrationTests; + +internal class PowerRequest(string requestType, string requesterType, string requesterName, string? reason) +{ + public string RequestType { get; } = requestType; + public string RequesterType { get; } = requesterType; + public string RequesterName { get; } = requesterName; + public string? Reason { get; } = reason; +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/PowerRequestsParser.cs b/tests/BenchmarkDotNet.IntegrationTests/PowerRequestsParser.cs new file mode 100644 index 0000000000..bb634b5228 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/PowerRequestsParser.cs @@ -0,0 +1,187 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace BenchmarkDotNet.IntegrationTests; + +/// +/// Parses the output of 'powercfg /requests' command into a list of s. +/// +/// +/// +/// Not using Sprache. It is superseded by Superpower. +/// Not using Superpower. I gained more knowledge +/// implementing this class from scratch. +/// +/// Example input: +/// +/// DISPLAY: +/// [PROCESS] \Device\HarddiskVolume3\Program Files (x86)\Google\Chrome\Application\chrome.exe +/// Video Wake Lock +/// +/// SYSTEM: +/// [DRIVER] Realtek High Definition Audio(SST) ... +/// Er wordt momenteel een audiostream gebruikt. +/// [PROCESS] \Device\HarddiskVolume3\...\NoSleep.exe +/// [PROCESS] \Device\HarddiskVolume3\Program Files (x86)\Google\Chrome\Application\chrome.exe +/// Video Wake Lock +/// +/// AWAYMODE: +/// None. +/// +/// EXECUTION: +/// [PROCESS] \Device\HarddiskVolume3\Program Files (x86)\Google\Chrome\Application\chrome.exe +/// Playing audio +/// +/// PERFBOOST: +/// None. +/// +/// ACTIVELOCKSCREEN: +/// None. +/// +/// +/// +internal class PowerRequestsParser +{ + /// + /// Parses output of 'powercfg /requests' into a list of s. + /// + /// + /// + /// This method takes a list of s. Examines next token and decides how to + /// parse. + /// + /// + /// Output of 'powercfg /requests'. + public static IEnumerable Parse(string input) + { + using TokenStream tokens = new TokenStream(Tokens(input)); + while (tokens.TryPeek().HasValue) + { + foreach (PowerRequest item in ParseRequestType(tokens)) + { + yield return item; + } + } + } + + private static IEnumerable ParseRequestType(TokenStream tokens) + { + Token requestType = tokens.Take(TokenType.RequestType); + if (tokens.Peek().TokenType == TokenType.RequesterType) + { + while (tokens.Peek().TokenType == TokenType.RequesterType) + { + yield return ParseRequesterType(requestType, tokens); + } + } + else + { + _ = tokens.Take(TokenType.None); + } + _ = tokens.Take(TokenType.EmptyLine); + } + + private static PowerRequest ParseRequesterType(Token requestType, TokenStream tokens) + { + Token requesterType = tokens.Take(TokenType.RequesterType); + Token requesterName = tokens.Take(TokenType.RequesterName); + Token? reason = null; + if (tokens.Peek().TokenType == TokenType.Reason) + { + reason = tokens.Take(TokenType.Reason); + } + return new PowerRequest(requestType.Value, requesterType.Value, requesterName.Value, reason?.Value); + } + + /// + /// Converts the input into a list of s. + /// + /// + /// + /// Looking at above sample, tokenizing is made simple when done line by line. Each line + /// contains one or two s. + /// + /// + /// Output of 'powercfg /requests'. + private static IEnumerable Tokens(string input) + { + // Contrary to calling input.Split('\r', '\n'), StringReader's ReadLine method does not + // return an empty string when CR is followed by LF. + StringReader reader = new StringReader(input); + string? line; + while ((line = reader.ReadLine()) != null) + { + if (line.Length == 0) + { + yield return new Token(TokenType.EmptyLine, ""); + } + else if (line[line.Length - 1] == ':') + { + yield return new Token(TokenType.RequestType, line.Substring(0, line.Length - 1).ToString()); + } + else if (string.Equals(line, "None.", StringComparison.InvariantCulture)) + { + yield return new Token(TokenType.None, line); + } + else if (line[0] == '[') + { + int pos = line.IndexOf(']'); + yield return new Token(TokenType.RequesterType, line.Substring(1, pos - 1)); + yield return new Token(TokenType.RequesterName, line.Substring(pos + 2)); + } + else + { + yield return new Token(TokenType.Reason, line); + } + } + } + + /// + /// Adds and to an of + /// s. + /// + /// + private class TokenStream(IEnumerable tokens) : IDisposable + { + private readonly IEnumerator tokens = tokens.GetEnumerator(); + private Token? cached; + + public Token? TryPeek() => cached ??= tokens.MoveNext() ? tokens.Current : null; + + public Token Peek() => TryPeek() ?? throw new EndOfStreamException(); + + public Token Take(TokenType requestType) + { + Token peek = Peek(); + if (peek.TokenType == requestType) + { + cached = null; + return peek; + } + else + { + throw new InvalidCastException($"Unexpected Token of type '{peek.TokenType}'. Expected type '{requestType}'."); + } + } + + public void Dispose() => tokens.Dispose(); + } + + private enum TokenType + { + EmptyLine, + None, + Reason, + RequesterName, + RequesterType, + RequestType + } + + private readonly struct Token(TokenType tokenType, string value) + { + public TokenType TokenType { get; } = tokenType; + + public string Value { get; } = value; + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs index 220991796d..724a882e22 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs @@ -14,6 +14,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Portability; using Xunit; @@ -32,11 +33,9 @@ public static IEnumerable GetToolchains() yield return new object[] { Job.Default.GetToolchain() }; if (!ContinuousIntegration.IsGitHubActionsOnWindows() // no native dependencies - && !RuntimeInformation.IsMacOS()) // currently not supported + && !OsDetector.IsMacOS()) // currently not supported { - yield return new object[]{ NativeAotToolchain.CreateBuilder() - .UseNuGet("7.0.0", "https://api.nuget.org/v3/index.json") - .ToToolchain() }; + yield return new object[]{ NativeAotToolchain.Net80 }; } // TODO: Support InProcessEmitToolchain.Instance // yield return new object[] { InProcessEmitToolchain.Instance }; diff --git a/tests/BenchmarkDotNet.IntegrationTests/WakeLockTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WakeLockTests.cs new file mode 100644 index 0000000000..cf8d0d39b9 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/WakeLockTests.cs @@ -0,0 +1,179 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Tests.XUnit; +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.Versioning; +using System.Threading; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.IntegrationTests; + +public class WakeLockTests : BenchmarkTestExecutor +{ + private const string PingEventName = @"Global\WakeLockTests-ping"; + private const string PongEventName = @"Global\WakeLockTests-pong"; + private static readonly TimeSpan testTimeout = TimeSpan.FromMinutes(1); + private readonly OutputLogger logger; + + public WakeLockTests(ITestOutputHelper output) : base(output) + { + logger = new OutputLogger(Output); + } + + [Fact] + public void ConfigurationDefaultValue() + { + Assert.Equal(WakeLockType.System, DefaultConfig.Instance.WakeLock); + Assert.Equal(WakeLockType.None, new DebugBuildConfig().WakeLock); + Assert.Equal(WakeLockType.None, new DebugInProcessConfig().WakeLock); + } + + [TheoryEnvSpecific(EnvRequirement.NonWindows)] + [InlineData(WakeLockType.None)] + [InlineData(WakeLockType.System)] + [InlineData(WakeLockType.Display)] + public void WakeLockIsWindowsOnly(WakeLockType wakeLockType) + { + using IDisposable wakeLock = WakeLock.Request(wakeLockType, "dummy", logger); + Assert.Null(wakeLock); + } + + [FactEnvSpecific(EnvRequirement.WindowsOnly)] + public void WakeLockSleepOrDisplayIsAllowed() + { + using IDisposable wakeLock = WakeLock.Request(WakeLockType.None, "dummy", logger); + Assert.Null(wakeLock); + } + + [FactEnvSpecific(EnvRequirement.WindowsOnly, EnvRequirement.NeedsPrivilegedProcess)] + public void WakeLockRequireSystem() + { + using (IDisposable wakeLock = WakeLock.Request(WakeLockType.System, "WakeLockTests", logger)) + { + Assert.NotNull(wakeLock); + Assert.Equal("SYSTEM", GetPowerRequests("WakeLockTests")); + } + Assert.Equal("", GetPowerRequests()); + } + + [FactEnvSpecific(EnvRequirement.WindowsOnly, EnvRequirement.NeedsPrivilegedProcess)] + public void WakeLockRequireDisplay() + { + using (IDisposable wakeLock = WakeLock.Request(WakeLockType.Display, "WakeLockTests", logger)) + { + Assert.NotNull(wakeLock); + Assert.Equal("DISPLAY, SYSTEM", GetPowerRequests("WakeLockTests")); + } + Assert.Equal("", GetPowerRequests()); + } + + [FactEnvSpecific(EnvRequirement.NonWindows)] + public void BenchmarkRunnerIgnoresWakeLock() => + _ = CanExecute(fullValidation: false); + + [WakeLock(WakeLockType.Display)] + public class IgnoreWakeLock + { + [Benchmark] public void Sleep() { } + } + +#if !NET462 + [SupportedOSPlatform("windows")] +#endif + [TheoryEnvSpecific(EnvRequirement.WindowsOnly, EnvRequirement.NeedsPrivilegedProcess)] + [InlineData(typeof(Default), "SYSTEM")] + [InlineData(typeof(None), "")] + [InlineData(typeof(RequireSystem), "SYSTEM")] + [InlineData(typeof(RequireDisplay), "DISPLAY, SYSTEM")] + public async Task BenchmarkRunnerAcquiresWakeLock(Type type, string expected) + { + using EventWaitHandle + ping = new EventWaitHandle(false, EventResetMode.AutoReset, PingEventName), + pong = new EventWaitHandle(false, EventResetMode.AutoReset, PongEventName); + string pwrRequests = null; + Task task = WaitForBenchmarkRunningAndGetPowerRequests(); + _ = CanExecute(type, fullValidation: false); + await task; + + Assert.Equal(expected, pwrRequests); + + async Task WaitForBenchmarkRunningAndGetPowerRequests() + { + await AsTask(ping, testTimeout); + pwrRequests = GetPowerRequests("BenchmarkDotNet Running Benchmarks"); + pong.Set(); + } + } + + public class Default : Base { } + + [WakeLock(WakeLockType.None)] public class None : Base { } + + [WakeLock(WakeLockType.System)] public class RequireSystem : Base { } + + [WakeLock(WakeLockType.Display)] public class RequireDisplay : Base { } + + public class Base + { + [Benchmark] +#if !NET462 + [SupportedOSPlatform("windows")] +#endif + public void SignalBenchmarkRunningAndWaitForGetPowerRequests() + { + using EventWaitHandle + ping = EventWaitHandle.OpenExisting(PingEventName), + pong = EventWaitHandle.OpenExisting(PongEventName); + ping.Set(); + pong.WaitOne(testTimeout); + } + } + + private string GetPowerRequests(string? expectedReason = null) + { + string pwrRequests = ProcessHelper.RunAndReadOutput("powercfg", "/requests"); + Output.WriteLine(pwrRequests); // Useful to analyse failing tests. + string fileName = Process.GetCurrentProcess().MainModule.FileName; + string mustEndWith = fileName.Substring(Path.GetPathRoot(fileName).Length); + + return string.Join(", ", + from pr in PowerRequestsParser.Parse(pwrRequests) + where + pr.RequesterName.EndsWith(mustEndWith, StringComparison.InvariantCulture) && + string.Equals(pr.RequesterType, "PROCESS", StringComparison.InvariantCulture) && + (expectedReason == null || string.Equals(pr.Reason, expectedReason, StringComparison.InvariantCulture)) + select pr.RequestType); + } + + private Task AsTask(WaitHandle waitHandle, TimeSpan timeout) + { + TaskCompletionSource tcs = new TaskCompletionSource(); + RegisteredWaitHandle rwh = null; + rwh = ThreadPool.RegisterWaitForSingleObject( + waitHandle, + (object state, bool timedOut) => + { + rwh.Unregister(null); + if (timedOut) + { + tcs.SetException(new TimeoutException()); + } + else + { + tcs.SetResult(true); + } + }, + null, + timeout, + true); + return tcs.Task; + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs new file mode 100644 index 0000000000..6d3519a9c5 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs @@ -0,0 +1,57 @@ +using System; +using System.IO; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Tests.XUnit; +using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Toolchains.MonoWasm; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.IntegrationTests +{ + /// + /// In order to run WasmTests locally, the following prerequisites are required: + /// * Install wasm-tools workload: `BenchmarkDotNet/build.cmd install-wasm-tools` + /// * Install npm + /// * Install v8: `npm install jsvu -g && jsvu --os=default --engines=v8` + /// * Add `$HOME/.jsvu/bin` to PATH + /// * Run tests using .NET SDK from `BenchmarkDotNet/.dotnet/` + /// + public class WasmTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) + { + [FactEnvSpecific("WASM is only supported on Unix", EnvRequirement.NonWindows)] + public void WasmIsSupported() + { + var dotnetVersion = "net8.0"; + var logger = new OutputLogger(Output); + var netCoreAppSettings = new NetCoreAppSettings(dotnetVersion, null, "Wasm"); + var mainJsPath = Path.Combine(AppContext.BaseDirectory, "AppBundle", "test-main.js"); + + var config = ManualConfig.CreateEmpty() + .AddLogger(logger) + .AddJob(Job.Dry + .WithArguments([new MsBuildArgument($"/p:WasmMainJSPath={mainJsPath}")]) + .WithRuntime(new WasmRuntime(dotnetVersion, moniker: RuntimeMoniker.WasmNet80, javaScriptEngineArguments: "--expose_wasm --module")) + .WithToolchain(WasmToolchain.From(netCoreAppSettings))) + .WithOption(ConfigOptions.GenerateMSBuildBinLog, true); + + CanExecute(config); + } + + public class WasmBenchmark + { + [Benchmark] + public void Check() + { + if (!RuntimeInformation.IsWasm) + { + throw new Exception("Incorrect runtime detection"); + } + } + } + } +} diff --git a/tests/BenchmarkDotNet.Tests/Analysers/OutliersAnalyserTests.cs b/tests/BenchmarkDotNet.Tests/Analysers/OutliersAnalyserTests.cs index c1368adf94..e8f42d2366 100644 --- a/tests/BenchmarkDotNet.Tests/Analysers/OutliersAnalyserTests.cs +++ b/tests/BenchmarkDotNet.Tests/Analysers/OutliersAnalyserTests.cs @@ -10,12 +10,8 @@ namespace BenchmarkDotNet.Tests.Analysers { - public class OutliersAnalyserTests + public class OutliersAnalyserTests(ITestOutputHelper output) { - private readonly ITestOutputHelper output; - - public OutliersAnalyserTests(ITestOutputHelper output) => this.output = output; - [Theory] [InlineData(0, 0, "")] [InlineData(1, 1, "1 outlier was removed")] @@ -64,9 +60,9 @@ public void RangeMessageTest(int lowerOutliers, int upperOutliers, string expect var cultureInfo = TestCultureInfo.Instance; string actualMessage = OutliersAnalyser.GetMessage(s.AllOutliers, s.AllOutliers, s.LowerOutliers, s.UpperOutliers, cultureInfo).ToAscii(); output.WriteLine("Values : " + - string.Join(", ", values.Take(5).Select(x => TimeInterval.FromNanoseconds(x).ToString(cultureInfo, "N2"))) + + string.Join(", ", values.Take(5).Select(x => TimeInterval.FromNanoseconds(x).ToDefaultString("N2"))) + ", ..., " + - string.Join(", ", values.Skip(values.Count - 5).Select(x => TimeInterval.FromNanoseconds(x).ToString(cultureInfo, "N2")))); + string.Join(", ", values.Skip(values.Count - 5).Select(x => TimeInterval.FromNanoseconds(x).ToDefaultString("N2")))); output.WriteLine("Actual : " + actualMessage); output.WriteLine("Expected : " + expectedMessage); Assert.Equal(expectedMessage, actualMessage); diff --git a/tests/BenchmarkDotNet.Tests/Analysers/ZeroMeasurementHelperTests.cs b/tests/BenchmarkDotNet.Tests/Analysers/ZeroMeasurementHelperTests.cs index d0bcb15cae..9a6679e3ec 100644 --- a/tests/BenchmarkDotNet.Tests/Analysers/ZeroMeasurementHelperTests.cs +++ b/tests/BenchmarkDotNet.Tests/Analysers/ZeroMeasurementHelperTests.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using BenchmarkDotNet.Analysers; +using Perfolizer; using Xunit; namespace BenchmarkDotNet.Tests.Analysers @@ -9,7 +10,7 @@ public class ZeroMeasurementHelperTests #region OneSampleTests // let processor frequency ~ 3.30 GHz - private static double ThresholdMock = 0.2702d / 2; + private const double ThresholdMock = 0.2702d / 2; /* Test distributions inspired by data from method @@ -30,16 +31,14 @@ and RyuJit x64 [InlineData(0.27049, 0.27410, 0.27433, 0.27468, 0.27476, 0.27591, 0.27644, 0.27704, 0.27724, 0.27766, 0.27792, 0.27878, 0.28025)] public void OneSample_Around_One_CPU_Cycle_Method(params double[] distribution) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementOneSample(distribution, ThresholdMock); - Assert.False(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.IsNoticeable(new Sample(distribution), ThresholdMock)); } [Theory] [InlineData(0.2703, 0.2701, 0.2703, 0.2701, 0.2703, 0.2701, 0.2703, 0.2701, 0.2703, 0.2701, 0.2703, 0.2701)] public void OneSample_Exactly_One_CPU_Cycle_Method(params double[] distribution) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementOneSample(distribution, ThresholdMock); - Assert.False(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.IsNoticeable(new Sample(distribution), ThresholdMock)); } /* @@ -57,11 +56,11 @@ and RyuJit x64 [Theory] [InlineData(0d, 0d, 0.00191d, 0.00530d, 0.00820d, 0.01383d, 0.01617d, 0.02183d, 0.02421d, 0.03640d, 0.03726d, 0.04894d, 0.05122d, 0.05924d, 0.06183d)] [InlineData(0d, 0d, 0d, 0d, 0d, 0d, 0d, 0.00138d, 0.00482d, 0.00616d, 0.01318d, 0.02266d, 0.03048d, 0.03144d)] - [InlineData(0.02203d, 0.02523d, 0.02567d, 0.02706d, 0.03048d, 0.03461d, 0.03953d, 0.04127d, 0.04396d, 0.04939d, 0.05361d, 0.05670d, 0.06394d, 0.06812d, 0.06901d)] + [InlineData(0.02203d, 0.02523d, 0.02567d, 0.02706d, 0.03048d, 0.03461d, 0.03953d, 0.04127d, 0.04396d, 0.04939d, 0.05361d, 0.05670d, 0.06394d, 0.06812d, + 0.06901d)] public void OneSample_Less_Than_One_CPU_Cycle_Method(params double[] distribution) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementOneSample(distribution, ThresholdMock); - Assert.True(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.IsNegligible(new Sample(distribution), ThresholdMock)); } /* @@ -72,8 +71,7 @@ Sometimes appears distributions with all zero values [InlineData(0d, 0d, 0d, 0d, 0d, 0d, 0d, 0d, 0d, 0d, 0d, 0d, 0d)] public void OneSample_Exactly_Zero_ns_Method(params double[] distribution) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementOneSample(distribution, ThresholdMock); - Assert.True(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.IsNegligible(new Sample(distribution), ThresholdMock)); } /* @@ -85,18 +83,20 @@ Test distributions inspired by data from method and RyuJit x64 */ [Theory] - [InlineData(0.57079d, 0.57469d, 0.57990d, 0.58025d, 0.58532d, 0.59250d, 0.59442d, 0.59487d, 0.59522d, 0.59619d, 0.59756d, 0.59802d, 0.60120d, 0.60813d, 0.61592d)] - [InlineData(0.57106d, 0.57168d, 0.57326d, 0.57587d, 0.57958d, 0.58982d, 0.59493d, 0.59950d, 0.61413d, 0.62000d, 0.62607d, 0.63209d, 0.63730d, 0.65048d, 0.65119d)] + [InlineData(0.57079d, 0.57469d, 0.57990d, 0.58025d, 0.58532d, 0.59250d, 0.59442d, 0.59487d, 0.59522d, 0.59619d, 0.59756d, 0.59802d, 0.60120d, 0.60813d, + 0.61592d)] + [InlineData(0.57106d, 0.57168d, 0.57326d, 0.57587d, 0.57958d, 0.58982d, 0.59493d, 0.59950d, 0.61413d, 0.62000d, 0.62607d, 0.63209d, 0.63730d, 0.65048d, + 0.65119d)] [InlineData(0.57347d, 0.57483d, 0.57598d, 0.57681d, 0.57724d, 0.57906d, 0.57944d, 0.58182d, 0.58261d, 0.58300d, 0.58468d, 0.59045d, 0.59217d)] public void OneSample_Around_Two_CPU_Cycle_Method(params double[] distribution) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementOneSample(distribution, ThresholdMock); - Assert.False(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.IsNoticeable(new Sample(distribution), ThresholdMock)); } #endregion #region TwoSamplesTests + /* Data inspired by distribution of benchmark (Workload Actual and Overhead Actual measurements) @@ -115,14 +115,21 @@ public static IEnumerable LessThanOneCycleTwoSamples { return new[] { - new object[] { new[] { 2.0037, 2.0062, 2.0066, 2.0073, 2.0089, 2.0103, 2.013, 2.0169, 2.0197, 2.0224, 2.0243, 2.0271, 2.0281, 2.0514, 2.0517 }, - new[] { 2.0426, 2.046, 2.0471, 2.0506, 2.0508, 2.0555, 2.0573, 2.0653, 2.0657, 2.0659, 2.0692, 2.0717, 2.0777, 2.0856, 2.0868 } }, - - new object[] { new[] { 2.0186, 2.0196, 2.0207, 2.0208, 2.0208, 2.0211, 2.0213, 2.0215, 2.0288, 2.0315, 2.0326, 2.039, 2.049, 2.055, 2.0598 }, - new[] { 2.0151, 2.0192, 2.0226, 2.0248, 2.0271, 2.0276, 2.0298, 2.0339, 2.0411, 2.0429, 2.0458, 2.0501, 2.061, 2.0733, 2.0744 } }, - - new object[] { new[] { 2.0049, 2.0141, 2.0194, 2.0253, 2.0264, 2.0296, 2.0333, 2.0422, 2.0438, 2.044, 2.047, 2.048, 2.0494, 2.0549, 2.0675 }, - new[] { 1.9963, 2.0037, 2.0037, 2.0046, 2.0051, 2.007, 2.0166, 2.021, 2.0225, 2.0247, 2.0391, 2.0473, 2.0572, 2.0576, 2.0582 } } + new object[] + { + new[] { 2.0037, 2.0062, 2.0066, 2.0073, 2.0089, 2.0103, 2.013, 2.0169, 2.0197, 2.0224, 2.0243, 2.0271, 2.0281, 2.0514, 2.0517 }, + new[] { 2.0426, 2.046, 2.0471, 2.0506, 2.0508, 2.0555, 2.0573, 2.0653, 2.0657, 2.0659, 2.0692, 2.0717, 2.0777, 2.0856, 2.0868 } + }, + + [ + new[] { 2.0186, 2.0196, 2.0207, 2.0208, 2.0208, 2.0211, 2.0213, 2.0215, 2.0288, 2.0315, 2.0326, 2.039, 2.049, 2.055, 2.0598 }, + new[] { 2.0151, 2.0192, 2.0226, 2.0248, 2.0271, 2.0276, 2.0298, 2.0339, 2.0411, 2.0429, 2.0458, 2.0501, 2.061, 2.0733, 2.0744 } + ], + + [ + new[] { 2.0049, 2.0141, 2.0194, 2.0253, 2.0264, 2.0296, 2.0333, 2.0422, 2.0438, 2.044, 2.047, 2.048, 2.0494, 2.0549, 2.0675 }, + new[] { 1.9963, 2.0037, 2.0037, 2.0046, 2.0051, 2.007, 2.0166, 2.021, 2.0225, 2.0247, 2.0391, 2.0473, 2.0572, 2.0576, 2.0582 } + ] }; } } @@ -131,8 +138,7 @@ public static IEnumerable LessThanOneCycleTwoSamples [MemberData(nameof(LessThanOneCycleTwoSamples))] public void TwoSamples_Less_Than_One_CPU_Cycle_Method(double[] workload, double[] overhead) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workload, overhead); - Assert.True(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.AreIndistinguishable(new Sample(workload), new Sample(overhead))); } /* @@ -153,14 +159,41 @@ public static IEnumerable OneCycleTwoSamples { return new[] { - new object[] { new[] { 2.34763, 2.34861, 2.34872, 2.34953, 2.35002, 2.35614, 2.35650, 2.36323, 2.36941, 2.37376, 2.38491, 2.38619, 2.38657, 2.38902, 2.39455 }, - new[] { 2.05899, 2.06069, 2.06243, 2.06405, 2.06762, 2.06785, 2.06889, 2.06891, 2.06895, 2.07531, 2.08003, 2.08024, 2.08342, 2.08959 } }, - - new object[] { new[] { 2.36960, 2.37438, 2.37442, 2.38332, 2.38940, 2.39099, 2.39394, 2.39974, 2.40808, 2.41760, 2.41980, 2.42275, 2.42828, 2.42946, 2.43763 }, - new[] { 2.06486, 2.06599, 2.07205, 2.07660, 2.07810, 2.07841, 2.08107, 2.08714, 2.10467, 2.10469, 2.11713, 2.12078, 2.12476, 2.12858, 2.13760 } }, - - new object[] { new[] { 2.35046, 2.35630, 2.35788, 2.35801, 2.36632, 2.36841, 2.36925, 2.36980, 2.36998, 2.37153, 2.37330, 2.38491, 2.38732, 2.38853, 2.41052 }, - new[] { 2.06291, 2.06545, 2.06763, 2.07381, 2.07568, 2.07810, 2.07894, 2.08153, 2.08264, 2.09000, 2.09814, 2.10082, 2.10107, 2.10576, 2.12841 } } + new object[] + { + new[] + { + 2.34763, 2.34861, 2.34872, 2.34953, 2.35002, 2.35614, 2.35650, 2.36323, 2.36941, 2.37376, 2.38491, 2.38619, 2.38657, 2.38902, + 2.39455 + }, + new[] { 2.05899, 2.06069, 2.06243, 2.06405, 2.06762, 2.06785, 2.06889, 2.06891, 2.06895, 2.07531, 2.08003, 2.08024, 2.08342, 2.08959 } + }, + + [ + new[] + { + 2.36960, 2.37438, 2.37442, 2.38332, 2.38940, 2.39099, 2.39394, 2.39974, 2.40808, 2.41760, 2.41980, 2.42275, 2.42828, 2.42946, + 2.43763 + }, + new[] + { + 2.06486, 2.06599, 2.07205, 2.07660, 2.07810, 2.07841, 2.08107, 2.08714, 2.10467, 2.10469, 2.11713, 2.12078, 2.12476, 2.12858, + 2.13760 + } + ], + + [ + new[] + { + 2.35046, 2.35630, 2.35788, 2.35801, 2.36632, 2.36841, 2.36925, 2.36980, 2.36998, 2.37153, 2.37330, 2.38491, 2.38732, 2.38853, + 2.41052 + }, + new[] + { + 2.06291, 2.06545, 2.06763, 2.07381, 2.07568, 2.07810, 2.07894, 2.08153, 2.08264, 2.09000, 2.09814, 2.10082, 2.10107, 2.10576, + 2.12841 + } + ] }; } } @@ -169,8 +202,7 @@ public static IEnumerable OneCycleTwoSamples [MemberData(nameof(OneCycleTwoSamples))] public void TwoSamples_Around_One_CPU_Cycle_Method(double[] workload, double[] overhead) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workload, overhead); - Assert.False(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.AreDistinguishable(workload, overhead)); } /* @@ -187,14 +219,19 @@ public static IEnumerable TwoCycleTwoSamples { return new[] { - new object[] { new[] { 2.6561, 2.658, 2.6606, 2.6621, 2.6636, 2.6639, 2.6656, 2.6673, 2.6741, 2.6754, 2.6787, 2.6935, 2.6997, 2.7313, 2.7394 }, - new[] { 2.0387, 2.0436, 2.0485, 2.0485, 2.0525, 2.0584, 2.0638, 2.0678, 2.0678, 2.0698, 2.0703, 2.0715, 2.0825, 2.0864 } }, - - new object[] { new[] { 2.6368, 2.6504, 2.652, 2.6541, 2.6607, 2.6642, 2.6698, 2.6749, 2.679, 2.6847, 2.6858, 2.6883, 2.6929, 2.702, 2.7134 }, - new[] { 2.04, 2.0461, 2.0481, 2.0485, 2.0502, 2.0523, 2.0547, 2.0602, 2.0613, 2.0677, 2.069, 2.0691, 2.0699, 2.0865 } }, - - new object[] { new[] { 2.6607, 2.6691, 2.67, 2.6741, 2.6753, 2.679, 2.6856, 2.6869, 2.6893, 2.692, 2.6971, 2.7146, 2.7245, 2.7368 }, - new[] { 2.0346, 2.054, 2.0655, 2.0673, 2.0718, 2.0748, 2.0766, 2.0839, 2.0856, 2.0869, 2.0924, 2.0968, 2.1129, 2.1148, 2.1328 } } + new object[] + { + new[] { 2.6561, 2.658, 2.6606, 2.6621, 2.6636, 2.6639, 2.6656, 2.6673, 2.6741, 2.6754, 2.6787, 2.6935, 2.6997, 2.7313, 2.7394 }, + new[] { 2.0387, 2.0436, 2.0485, 2.0485, 2.0525, 2.0584, 2.0638, 2.0678, 2.0678, 2.0698, 2.0703, 2.0715, 2.0825, 2.0864 } + }, + [ + new[] { 2.6368, 2.6504, 2.652, 2.6541, 2.6607, 2.6642, 2.6698, 2.6749, 2.679, 2.6847, 2.6858, 2.6883, 2.6929, 2.702, 2.7134 }, + new[] { 2.04, 2.0461, 2.0481, 2.0485, 2.0502, 2.0523, 2.0547, 2.0602, 2.0613, 2.0677, 2.069, 2.0691, 2.0699, 2.0865 } + ], + [ + new[] { 2.6607, 2.6691, 2.67, 2.6741, 2.6753, 2.679, 2.6856, 2.6869, 2.6893, 2.692, 2.6971, 2.7146, 2.7245, 2.7368 }, + new[] { 2.0346, 2.054, 2.0655, 2.0673, 2.0718, 2.0748, 2.0766, 2.0839, 2.0856, 2.0869, 2.0924, 2.0968, 2.1129, 2.1148, 2.1328 } + ] }; } } @@ -203,9 +240,9 @@ public static IEnumerable TwoCycleTwoSamples [MemberData(nameof(TwoCycleTwoSamples))] public void TwoSamples_Around_Two_CPU_Cycle_Method(double[] workload, double[] overhead) { - var zeroMeasurement = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workload, overhead); - Assert.False(zeroMeasurement); + Assert.True(ZeroMeasurementHelper.AreDistinguishable(workload, overhead)); } + #endregion } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesVerifyTests.cs b/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesVerifyTests.cs index a1086da1de..f766f0eba3 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesVerifyTests.cs +++ b/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesVerifyTests.cs @@ -9,6 +9,7 @@ using BenchmarkDotNet.Tests.Mocks; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Infra; using BenchmarkDotNet.Validators; using JetBrains.Annotations; using VerifyXunit; @@ -53,7 +54,7 @@ public Task BenchmarkShouldProduceSummary(Type benchmarkType) foreach (var error in errors) logger.WriteLineError("* " + error.Message); - var settings = VerifySettingsFactory.Create(); + var settings = VerifyHelper.Create(); settings.UseTextForParameters(benchmarkType.Name); return Verifier.Verify(logger.GetLog(), settings); } diff --git a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfBool.verified.txt b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfBool.verified.txt index 41554d0c7c..d89bd4c091 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfBool.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfBool.verified.txt @@ -2,14 +2,14 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | ParamProperty | Mean | Error | StdDev | ---------- |-------------- |---------:|--------:|--------:| - Benchmark | False | 102.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | True | 202.0 ns | 6.09 ns | 1.58 ns | ^ + Benchmark | False | 114.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | True | 214.5 ns | 5.88 ns | 8.80 ns | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfEnum.verified.txt b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfEnum.verified.txt index c1a301f759..2e8ac5702e 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfEnum.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfEnum.verified.txt @@ -2,15 +2,15 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | ParamProperty | Mean | Error | StdDev | ---------- |-------------- |---------:|--------:|--------:| - Benchmark | A | 102.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | B | 202.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | C | 302.0 ns | 6.09 ns | 1.58 ns | ^ + Benchmark | A | 114.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | B | 214.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | C | 314.5 ns | 5.88 ns | 8.80 ns | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableBool.verified.txt b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableBool.verified.txt index 61f4eeed4f..d292990cb8 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableBool.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableBool.verified.txt @@ -2,15 +2,15 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | ParamProperty | Mean | Error | StdDev | ---------- |-------------- |---------:|--------:|--------:| - Benchmark | ? | 102.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | False | 202.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | True | 302.0 ns | 6.09 ns | 1.58 ns | ^ + Benchmark | ? | 114.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | False | 214.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | True | 314.5 ns | 5.88 ns | 8.80 ns | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableEnum.verified.txt b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableEnum.verified.txt index 0c1dd08fdc..32f13c0208 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableEnum.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithAllValuesOfNullableEnum.verified.txt @@ -2,16 +2,16 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | ParamProperty | Mean | Error | StdDev | ---------- |-------------- |---------:|--------:|--------:| - Benchmark | ? | 102.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | A | 202.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | B | 302.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | C | 402.0 ns | 6.09 ns | 1.58 ns | ^ + Benchmark | ? | 114.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | A | 214.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | B | 314.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | C | 414.5 ns | 5.88 ns | 8.80 ns | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedFlagsEnumError.verified.txt b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedFlagsEnumError.verified.txt index ba31d1bc3b..bd49ee8d35 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedFlagsEnumError.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedFlagsEnumError.verified.txt @@ -2,14 +2,14 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | ParamProperty | Mean | Error | StdDev | ---------- |-------------- |---------:|--------:|--------:| - Benchmark | 0 | 102.0 ns | 6.09 ns | 1.58 ns | + Benchmark | 0 | 114.5 ns | 5.88 ns | 8.80 ns | Errors: 1 * Unable to use TestFlagsEnum with [ParamsAllValues], because it's flags enum. diff --git a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedNullableTypeError.verified.txt b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedNullableTypeError.verified.txt index 19566395d8..aceee21a7c 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedNullableTypeError.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedNullableTypeError.verified.txt @@ -2,15 +2,15 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | ParamProperty | Mean | Error | StdDev | ---------- |-------------- |---------:|--------:|--------:| - Benchmark | ? | 102.0 ns | 6.09 ns | 1.58 ns | ^ - Benchmark | 0 | 202.0 ns | 6.09 ns | 1.58 ns | ^ + Benchmark | ? | 114.5 ns | 5.88 ns | 8.80 ns | ^ + Benchmark | 0 | 214.5 ns | 5.88 ns | 8.80 ns | ^ Errors: 1 * Type Int32 cannot be used with [ParamsAllValues], allowed types are: bool, enum types and nullable type for another allowed type. diff --git a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedTypeError.verified.txt b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedTypeError.verified.txt index 9a27385161..d53d688ab8 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedTypeError.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Attributes/VerifiedFiles/ParamsAllValuesVerifyTests.BenchmarkShouldProduceSummary_WithNotAllowedTypeError.verified.txt @@ -2,14 +2,14 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | ParamProperty | Mean | Error | StdDev | ---------- |-------------- |---------:|--------:|--------:| - Benchmark | 0 | 102.0 ns | 6.09 ns | 1.58 ns | + Benchmark | 0 | 114.5 ns | 5.88 ns | 8.80 ns | Errors: 1 * Type Int32 cannot be used with [ParamsAllValues], allowed types are: bool, enum types and nullable type for another allowed type. diff --git a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj index 5b7a5b9cdd..000a27ca2c 100755 --- a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj +++ b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj @@ -2,7 +2,7 @@ BenchmarkDotNet.Tests - net7.0;net462 + net8.0;net462 BenchmarkDotNet.Tests BenchmarkDotNet.Tests true @@ -15,18 +15,20 @@ - - - - + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + + @@ -34,12 +36,23 @@ + - + + + + + CpuInfoFormatterTests + CpuInfoFormatterTests.cs + + + + + diff --git a/tests/BenchmarkDotNet.Tests/BuildResultTests.cs b/tests/BenchmarkDotNet.Tests/BuildResultTests.cs index 978da2a516..beba98e49e 100644 --- a/tests/BenchmarkDotNet.Tests/BuildResultTests.cs +++ b/tests/BenchmarkDotNet.Tests/BuildResultTests.cs @@ -11,24 +11,24 @@ public class BuildResultTests [Fact] public void NotFullyCompatibleMsBuildErrorIsTranslatedToMoreUserFriendlyVersion() { - const string msbuildError = @"C:\Program Files\dotnet\sdk\3.0.100-preview9-013617\Microsoft.Common.CurrentVersion.targets(1653,5): warning NU1702: ProjectReference 'C:\Projects\BenchmarkDotNet\tests\BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetFramework\BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetFramework.csproj' was resolved using '.NETFramework,Version=v4.6.2' instead of the project target framework '.NETCoreApp,Version=v2.1'. This project may not be fully compatible with your project. [C:\Projects\BenchmarkDotNet\tests\BenchmarkDotNet.IntegrationTests\bin\Release\net462\Job-VUALUD\BenchmarkDotNet.Autogenerated.csproj]"; + const string msbuildError = @"C:\Program Files\dotnet\sdk\3.0.100-preview9-013617\Microsoft.Common.CurrentVersion.targets(1653,5): warning NU1702: ProjectReference 'C:\Projects\BenchmarkDotNet\tests\BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetFramework\BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetFramework.csproj' was resolved using '.NETFramework,Version=v4.6.2' instead of the project target framework '.NETCoreApp,Version=v8.0'. This project may not be fully compatible with your project. [C:\Projects\BenchmarkDotNet\tests\BenchmarkDotNet.IntegrationTests\bin\Release\net462\Job-VUALUD\BenchmarkDotNet.Autogenerated.csproj]"; - string expected = $@"The project which defines benchmarks does not target 'netcoreapp2.1'." + Environment.NewLine + - $"You need to add 'netcoreapp2.1' to in your project file " + + string expected = $@"The project which defines benchmarks does not target 'net8.0'." + Environment.NewLine + + $"You need to add 'net8.0' to in your project file " + @"('C:\Projects\BenchmarkDotNet\tests\BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetFramework\BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetFramework.csproj')." + Environment.NewLine + - "Example: net462;netcoreapp2.1"; + "Example: net462;net8.0"; Verify(msbuildError, true, expected); } [Fact] public void NotCompatibleMsBuildErrorIsTranslatedToMoreUserFriendlyVersion() { - const string msbuildError = @"error NU1201: Project BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetCore is not compatible with net462 (.NETFramework,Version=v4.6.2) / win7-x64. Project BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetCore supports: netcoreapp2.1 (.NETCoreApp,Version=v2.1)"; + const string msbuildError = @"error NU1201: Project BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetCore is not compatible with net462 (.NETFramework,Version=v4.6.2) / win7-x64. Project BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetCore supports: net8.0 (.NETCoreApp,Version=v8.0)"; string expected = $@"The project which defines benchmarks does not target 'net462'." + Environment.NewLine + $"You need to add 'net462' to in your project file " + @"('BenchmarkDotNet.IntegrationTests.SingleRuntime.DotNetCore')." + Environment.NewLine + - "Example: netcoreapp2.1;net462"; + "Example: net8.0;net462"; Verify(msbuildError, true, expected); } @@ -36,7 +36,7 @@ public void NotCompatibleMsBuildErrorIsTranslatedToMoreUserFriendlyVersion() [Fact] public void MissingSdkIsTranslatedToMoreUserFriendlyVersion() { - const string msbuildError = @"C:\Program Files\dotnet\sdk\3.0.100-preview9-013617\Sdks\Microsoft.NET.Sdk\targets\Microsoft.NET.TargetFrameworkInference.targets(134,5): error NETSDK1045: The current .NET SDK does not support targeting .NET Core 5.0. Either target .NET Core 3.0 or lower, or use a version of the .NET SDK that supports .NET Core 5.0. [C:\Projects\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\netcoreapp2.1\bad57fca-694a-41ad-b630-9e5317a782ab\BenchmarkDotNet.Autogenerated.csproj]"; + const string msbuildError = @"C:\Program Files\dotnet\sdk\3.0.100-preview9-013617\Sdks\Microsoft.NET.Sdk\targets\Microsoft.NET.TargetFrameworkInference.targets(134,5): error NETSDK1045: The current .NET SDK does not support targeting .NET Core 5.0. Either target .NET Core 3.0 or lower, or use a version of the .NET SDK that supports .NET Core 5.0. [C:\Projects\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\net8.0\bad57fca-694a-41ad-b630-9e5317a782ab\BenchmarkDotNet.Autogenerated.csproj]"; string expected = "The current .NET SDK does not support targeting .NET Core 5.0. You need to install it or pass the path to dotnet cli via the `--cli` console line argument."; diff --git a/tests/BenchmarkDotNet.Tests/Builders/HostEnvironmentInfoBuilder.cs b/tests/BenchmarkDotNet.Tests/Builders/HostEnvironmentInfoBuilder.cs index e77ee1abd9..0c0737ecbb 100644 --- a/tests/BenchmarkDotNet.Tests/Builders/HostEnvironmentInfoBuilder.cs +++ b/tests/BenchmarkDotNet.Tests/Builders/HostEnvironmentInfoBuilder.cs @@ -1,18 +1,21 @@ using System; +using System.Diagnostics.CodeAnalysis; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Portability.Cpu; using Perfolizer.Horology; +using Perfolizer.Phd.Dto; namespace BenchmarkDotNet.Tests.Builders { + [SuppressMessage("ReSharper", "InconsistentNaming")] public class HostEnvironmentInfoBuilder { private string architecture = "64mock"; private string benchmarkDotNetVersion = "0.10.x-mock"; private Frequency chronometerFrequency = new Frequency(2531248); private string configuration = "CONFIGURATION"; - private string dotNetSdkVersion = "1.0.x.mock"; + private string? dotNetSdkVersion = "1.0.x.mock"; private HardwareTimerKind hardwareTimerKind = HardwareTimerKind.Tsc; private bool hasAttachedDebugger = false; private bool hasRyuJit = true; @@ -20,18 +23,20 @@ public class HostEnvironmentInfoBuilder private bool isServerGC = false; private string jitInfo = "RyuJIT-v4.6.x.mock"; private string jitModules = "clrjit-v4.6.x.mock"; - private string osVersion = "Microsoft Windows NT 10.0.x.mock"; + private PhdOs os = new () { Display = "Microsoft Windows NT 10.0.x.mock" }; private string runtimeVersion = "Clr 4.0.x.mock"; - private CpuInfo cpuInfo = new CpuInfo("MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", - physicalProcessorCount: 1, - physicalCoreCount: 4, - logicalCoreCount: 8, - nominalFrequency: Frequency.FromMHz(3100), - maxFrequency: Frequency.FromMHz(3100), - minFrequency: Frequency.FromMHz(3100)); + private readonly PhdCpu cpu = new () + { + ProcessorName = "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 4, + LogicalCoreCount = 8, + NominalFrequencyHz = Frequency.FromMHz(3100).Hertz.RoundToLong(), + MaxFrequencyHz = Frequency.FromMHz(3100).Hertz.RoundToLong() + }; - private VirtualMachineHypervisor virtualMachineHypervisor = HyperV.Default; + private VirtualMachineHypervisor? virtualMachineHypervisor = HyperV.Default; public HostEnvironmentInfoBuilder WithVMHypervisor(VirtualMachineHypervisor hypervisor) { @@ -55,7 +60,7 @@ public HostEnvironmentInfo Build() { return new MockHostEnvironmentInfo(architecture, benchmarkDotNetVersion, chronometerFrequency, configuration, dotNetSdkVersion, hardwareTimerKind, hasAttachedDebugger, hasRyuJit, isConcurrentGC, isServerGC, - jitInfo, jitModules, osVersion, cpuInfo, runtimeVersion, virtualMachineHypervisor); + jitInfo, jitModules, os, cpu, runtimeVersion, virtualMachineHypervisor); } } @@ -64,7 +69,7 @@ internal class MockHostEnvironmentInfo : HostEnvironmentInfo public MockHostEnvironmentInfo( string architecture, string benchmarkDotNetVersion, Frequency chronometerFrequency, string configuration, string dotNetSdkVersion, HardwareTimerKind hardwareTimerKind, bool hasAttachedDebugger, bool hasRyuJit, bool isConcurrentGC, bool isServerGC, - string jitInfo, string jitModules, string osVersion, CpuInfo cpuInfo, + string jitInfo, string jitModules, PhdOs os, PhdCpu cpu, string runtimeVersion, VirtualMachineHypervisor virtualMachineHypervisor) { Architecture = architecture; @@ -79,8 +84,8 @@ public MockHostEnvironmentInfo( IsServerGC = isServerGC; JitInfo = jitInfo; HardwareIntrinsicsShort = ""; - OsVersion = new Lazy(() => osVersion); - CpuInfo = new Lazy(() => cpuInfo); + Os = new Lazy(() => os); + Cpu = new Lazy(() => cpu); RuntimeVersion = runtimeVersion; VirtualMachineHypervisor = new Lazy(() => virtualMachineHypervisor); } diff --git a/tests/BenchmarkDotNet.Tests/Builders/VerifySettingsFactory.cs b/tests/BenchmarkDotNet.Tests/Builders/VerifySettingsFactory.cs deleted file mode 100644 index 027b5272a7..0000000000 --- a/tests/BenchmarkDotNet.Tests/Builders/VerifySettingsFactory.cs +++ /dev/null @@ -1,15 +0,0 @@ -using VerifyTests; - -namespace BenchmarkDotNet.Tests.Builders -{ - public static class VerifySettingsFactory - { - public static VerifySettings Create() - { - var result = new VerifySettings(); - result.UseDirectory("VerifiedFiles"); - result.DisableDiff(); - return result; - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Columns/MetricColumnTests.cs b/tests/BenchmarkDotNet.Tests/Columns/MetricColumnTests.cs index f16c4bd7f6..4c7678ba67 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/MetricColumnTests.cs +++ b/tests/BenchmarkDotNet.Tests/Columns/MetricColumnTests.cs @@ -11,6 +11,7 @@ using BenchmarkDotNet.Parameters; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Mocks; using BenchmarkDotNet.Validators; using Perfolizer.Horology; using Xunit; @@ -37,7 +38,7 @@ private static Summary CreateMockSummary(bool printUnitsInContent, bool printUni var summaryStyle = new SummaryStyle(TestCultureInfo.Instance, printUnitsInHeader, null, timeUnit, printUnitsInContent); var config = new ManualConfig().WithSummaryStyle(summaryStyle); var benchmarkCase = new BenchmarkCase( - new Descriptor(null, null), + new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo), Job.Dry, new ParameterInstances(ImmutableArray.Empty), ImmutableConfigBuilder.Create(config)); diff --git a/tests/BenchmarkDotNet.Tests/Columns/StatisticalTestColumnTests.cs b/tests/BenchmarkDotNet.Tests/Columns/StatisticalTestColumnTests.cs index ee45b61347..7a0d4bb1fd 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/StatisticalTestColumnTests.cs +++ b/tests/BenchmarkDotNet.Tests/Columns/StatisticalTestColumnTests.cs @@ -2,9 +2,9 @@ using System.Linq; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Mathematics; -using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; using BenchmarkDotNet.Reports; +using Perfolizer.Mathematics.Distributions.ContinuousDistributions; +using Perfolizer.Metrology; using Xunit; namespace BenchmarkDotNet.Tests.Columns @@ -12,60 +12,56 @@ namespace BenchmarkDotNet.Tests.Columns public class StatisticalTestColumnTests { [Theory] - [InlineData(StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.05)] - [InlineData(StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.05)] - [InlineData(StatisticalTestKind.Welch, ThresholdUnit.Milliseconds, 10)] - [InlineData(StatisticalTestKind.MannWhitney, ThresholdUnit.Milliseconds, 10)] - public void NoDifferenceIfValuesAreTheSame(StatisticalTestKind statisticalTestKind, ThresholdUnit thresholdUnit, double thresholdValue) + [InlineData("5%")] + [InlineData("10ms")] + public void NoDifferenceIfValuesAreTheSame(string threshold) { - var values = Enumerable.Repeat(100.0, 20).ToArray(); - - Compare(statisticalTestKind, thresholdUnit, thresholdValue, values, values, "Base"); + double[] values = Enumerable.Repeat(100.0, 20).ToArray(); + Compare(threshold, values, values, "Baseline"); } [Theory] - [InlineData(StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.02)] - [InlineData(StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.02)] - public void RegressionsAreDetected(StatisticalTestKind statisticalTestKind, ThresholdUnit thresholdUnit, double thresholdValue) + [InlineData("2%")] + public void RegressionsAreDetected(string threshold) { - var baseline = new[] { 10.0, 10.01, 10.02, 10.0, 10.03, 10.02, 9.99, 9.98, 10.0, 10.02 }; - var current = baseline.Select(value => value * 1.03).ToArray(); + double[]? baseline = [10.0, 10.01, 10.02, 10.0, 10.03, 10.02, 9.99, 9.98, 10.0, 10.02]; + double[]? current = baseline.Select(value => value * 1.03).ToArray(); - Compare(statisticalTestKind, thresholdUnit, thresholdValue, baseline, current, "Slower"); + Compare(threshold, baseline, current, "Slower"); } [Theory] - [InlineData(StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.02)] - [InlineData(StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.02)] - public void CanCompareDifferentSampleSizes(StatisticalTestKind statisticalTestKind, ThresholdUnit thresholdUnit, double thresholdValue) + [InlineData("2%")] + public void CanCompareDifferentSampleSizes(string threshold) { - var baseline = new[] { 10.0, 10.01, 10.02, 10.0, 10.03, 10.02, 9.99, 9.98, 10.0, 10.02 }; - var current = baseline + double[] baseline = new NormalDistribution(10, 0.01).Random(1729).Next(30); + double[] current = baseline .Skip(1) // we skip one element to make sure the sample size is different .Select(value => value * 1.03).ToArray(); - Compare(statisticalTestKind, thresholdUnit, thresholdValue, baseline, current, "Slower"); + Compare(threshold, baseline, current, "Slower"); } [Theory] - [InlineData(StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.02)] - [InlineData(StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.02)] - public void ImprovementsDetected(StatisticalTestKind statisticalTestKind, ThresholdUnit thresholdUnit, double thresholdValue) + [InlineData("2%")] + public void ImprovementsDetected(string threshold) { var baseline = new[] { 10.0, 10.01, 10.02, 10.0, 10.03, 10.02, 9.99, 9.98, 10.0, 10.02 }; var current = baseline.Select(value => value * 0.97).ToArray(); - Compare(statisticalTestKind, thresholdUnit, thresholdValue, baseline, current, "Faster"); + Compare(threshold, baseline, current, "Faster"); } - private static void Compare(StatisticalTestKind statisticalTestKind, ThresholdUnit thresholdUnit, double thresholdValue, double[] baseline, double[] current, string expectedResult) + private static void Compare(string threshold, double[] baseline, double[] current, string expectedResult) { - var sut = new StatisticalTestColumn(statisticalTestKind, Threshold.Create(thresholdUnit, thresholdValue)); + var sut = new StatisticalTestColumn(Threshold.Parse(threshold)); var emptyMetrics = new Dictionary(); - Assert.Equal(expectedResult, sut.GetValue(null, null, new Statistics(baseline), emptyMetrics, new Statistics(current), emptyMetrics, isBaseline: true)); - Assert.Equal(expectedResult, sut.GetValue(null, null, new Statistics(baseline), emptyMetrics, new Statistics(current), emptyMetrics, isBaseline: false)); + Assert.Equal(expectedResult, + sut.GetValue(null, null, new Statistics(baseline), emptyMetrics, new Statistics(current), emptyMetrics, isBaseline: true)); + Assert.Equal(expectedResult, + sut.GetValue(null, null, new Statistics(baseline), emptyMetrics, new Statistics(current), emptyMetrics, isBaseline: false)); } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Common/AbsoluteEqualityComparer.cs b/tests/BenchmarkDotNet.Tests/Common/AbsoluteEqualityComparer.cs new file mode 100644 index 0000000000..80a477d348 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Common/AbsoluteEqualityComparer.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; + +namespace BenchmarkDotNet.Tests.Common; + +public class AbsoluteEqualityComparer(double eps) : IEqualityComparer +{ + public static readonly IEqualityComparer E4 = new AbsoluteEqualityComparer(1e-4); + public static readonly IEqualityComparer E5 = new AbsoluteEqualityComparer(1e-5); + public static readonly IEqualityComparer E9 = new AbsoluteEqualityComparer(1e-9); + + public bool Equals(double x, double y) + { + if (double.IsPositiveInfinity(x) && double.IsPositiveInfinity(y)) + return true; + if (double.IsNegativeInfinity(x) && double.IsNegativeInfinity(y)) + return true; + if (double.IsNaN(x) && double.IsNaN(y)) + return true; + return Math.Abs(x - y) < eps; + } + + public int GetHashCode(double x) => x.GetHashCode(); +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index 538bad8a91..dd25527e8a 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -23,8 +23,6 @@ using Xunit.Abstractions; using BenchmarkDotNet.Portability; using Perfolizer.Horology; -using Perfolizer.Mathematics.SignificanceTesting; -using Perfolizer.Mathematics.Thresholds; namespace BenchmarkDotNet.Tests { @@ -76,7 +74,7 @@ public void UserCanSpecifyHowManyTimesTheBenchmarkShouldBeExecuted() var config = ConfigParser.Parse(new[] { "--LaunchCount", launchCount.ToString(), - "--warmupCount", warmupCount.ToString(), + "--warmupCount", warmupCount.ToString(), "--iterationTime", iterationTime.ToString(), "--iterationCount", iterationCount.ToString() }, new OutputLogger(Output)).config; @@ -141,12 +139,14 @@ public void CoreRunConfigParsedCorrectlyWhenRuntimeNotSpecified() var fakeDotnetCliPath = typeof(object).Assembly.Location; var fakeCoreRunPath = typeof(ConfigParserTests).Assembly.Location; var fakeRestorePackages = Path.GetTempPath(); - var config = ConfigParser.Parse(new[] { "--job=Dry", "--coreRun", fakeCoreRunPath, "--cli", fakeDotnetCliPath, "--packages", fakeRestorePackages }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "--job=Dry", "--coreRun", fakeCoreRunPath, "--cli", fakeDotnetCliPath, "--packages", fakeRestorePackages }, + new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); CoreRunToolchain toolchain = config.GetJobs().Single().GetToolchain() as CoreRunToolchain; Assert.NotNull(toolchain); - Assert.Equal(RuntimeInformation.GetCurrentRuntime().MsBuildMoniker, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); // runtime was not specified so the current was used + Assert.Equal(RuntimeInformation.GetCurrentRuntime().MsBuildMoniker, + ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); // runtime was not specified so the current was used Assert.Equal(fakeCoreRunPath, toolchain.SourceCoreRun.FullName); Assert.Equal(fakeDotnetCliPath, toolchain.CustomDotNetCliPath.FullName); Assert.Equal(fakeRestorePackages, toolchain.RestorePath.FullName); @@ -162,17 +162,19 @@ public void SpecifyingCoreRunWithFullFrameworkTargetsMostRecentTfm() CoreRunToolchain coreRunToolchain = (CoreRunToolchain)coreRunJob.GetToolchain(); DotNetCliGenerator generator = (DotNetCliGenerator)coreRunToolchain.Generator; - Assert.Equal("net8.0", generator.TargetFrameworkMoniker); + Assert.Equal("net10.0", generator.TargetFrameworkMoniker); } [FactEnvSpecific("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process", EnvRequirement.DotNetCoreOnly)] public void SpecifyingCoreRunAndRuntimeCreatesTwoJobs() { - const string runtime = "net7.0"; + const string runtime = "net8.0"; var fakeDotnetCliPath = typeof(object).Assembly.Location; var fakeCoreRunPath = typeof(ConfigParserTests).Assembly.Location; var fakeRestorePackages = Path.GetTempPath(); - var config = ConfigParser.Parse(new[] { "--job=Dry", "--coreRun", fakeCoreRunPath, "--cli", fakeDotnetCliPath, "--packages", fakeRestorePackages, "-r", runtime }, new OutputLogger(Output)).config; + var config = ConfigParser + .Parse(new[] { "--job=Dry", "--coreRun", fakeCoreRunPath, "--cli", fakeDotnetCliPath, "--packages", fakeRestorePackages, "-r", runtime }, + new OutputLogger(Output)).config; Assert.Equal(2, config.GetJobs().Count()); @@ -231,9 +233,8 @@ public void UserCanSpecifyMultipleCoreRunPaths() var jobs = config.GetJobs().ToArray(); Assert.Equal(2, jobs.Length); - Assert.Single(jobs.Where(job => job.GetToolchain() is CoreRunToolchain toolchain && toolchain.SourceCoreRun.FullName == fakeCoreRunPath_1)); - Assert.Single(jobs.Where(job => job.GetToolchain() is CoreRunToolchain toolchain && toolchain.SourceCoreRun.FullName == fakeCoreRunPath_2)); - Assert.Equal(2, jobs.Select(job => job.Id).Distinct().Count()); // each job must have a unique ID + Assert.Single(jobs, job => job.GetToolchain() is CoreRunToolchain toolchain && toolchain.SourceCoreRun.FullName == fakeCoreRunPath_1); + Assert.Single(jobs, job => job.GetToolchain() is CoreRunToolchain toolchain && toolchain.SourceCoreRun.FullName == fakeCoreRunPath_2); } [Fact] @@ -243,7 +244,7 @@ public void MonoPathParsedCorrectly() var config = ConfigParser.Parse(new[] { "-r", "mono", "--monoPath", fakeMonoPath }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is MonoRuntime mono && mono.CustomPath == fakeMonoPath)); + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is MonoRuntime mono && mono.CustomPath == fakeMonoPath); } [FactEnvSpecific("Testing local builds of Full .NET Framework is supported only on Windows", EnvRequirement.WindowsOnly)] @@ -253,13 +254,13 @@ public void ClrVersionParsedCorrectly() var config = ConfigParser.Parse(new[] { "--clrVersion", clrVersion }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is ClrRuntime clr && clr.Version == clrVersion)); + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is ClrRuntime clr && clr.Version == clrVersion); } [Fact] public void IlCompilerPathParsedCorrectly() { - var fakePath = new FileInfo(typeof(ConfigParserTests).Assembly.Location).Directory; + var fakePath = new FileInfo(typeof(ConfigParserTests).Assembly.Location).Directory; var config = ConfigParser.Parse(new[] { "-r", "nativeaot60", "--ilcPackages", fakePath.FullName }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); @@ -274,6 +275,12 @@ public void IlCompilerPathParsedCorrectly() [InlineData("netcoreapp2.1", true)] [InlineData("netcoreapp2.2", true)] [InlineData("netcoreapp3.0", true)] + [InlineData("netcoreapp3.1", true)] + [InlineData("net5.0", true)] + [InlineData("net6.0", true)] + [InlineData("net7.0", true)] + [InlineData("net8.0", true)] + [InlineData("net9.0", true)] [InlineData("net462", false)] [InlineData("net48", false)] public void DotNetCliParsedCorrectly(string tfm, bool isCore) @@ -286,12 +293,12 @@ public void DotNetCliParsedCorrectly(string tfm, bool isCore) if (isCore) { Assert.True(toolchain is CsProjCoreToolchain); - Assert.Equal(fakeDotnetCliPath, ((CsProjCoreToolchain) toolchain).CustomDotNetCliPath); + Assert.Equal(fakeDotnetCliPath, ((CsProjCoreToolchain)toolchain).CustomDotNetCliPath); } else { Assert.True(toolchain is CsProjClassicNetToolchain); - Assert.Equal(fakeDotnetCliPath, ((CsProjClassicNetToolchain) toolchain).CustomDotNetCliPath); + Assert.Equal(fakeDotnetCliPath, ((CsProjClassicNetToolchain)toolchain).CustomDotNetCliPath); } Assert.Equal(tfm, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); } @@ -301,7 +308,7 @@ public void DotNetCliParsedCorrectly(string tfm, bool isCore) [InlineData(ConfigOptions.KeepBenchmarkFiles, "--keepFiles")] [InlineData(ConfigOptions.DontOverwriteResults, "--noOverwrite")] [InlineData(ConfigOptions.StopOnFirstError, "--stopOnFirstError")] - [InlineData(ConfigOptions.DisableLogFile, "--disableLogFile" )] + [InlineData(ConfigOptions.DisableLogFile, "--disableLogFile")] [InlineData(ConfigOptions.LogBuildOutput, "--logBuildOutput")] [InlineData(ConfigOptions.GenerateMSBuildBinLog | ConfigOptions.KeepBenchmarkFiles, "--generateBinLog")] [InlineData( @@ -332,7 +339,7 @@ public void WhenConfigOptionsFlagsAreNotSpecifiedTheyAreNotSet() public void PackagesPathParsedCorrectly() { var fakeRestoreDirectory = new FileInfo(typeof(object).Assembly.Location).Directory.FullName; - var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.0", "--packages", fakeRestoreDirectory }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.1", "--packages", fakeRestoreDirectory }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); CsProjCoreToolchain toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain; @@ -344,7 +351,7 @@ public void PackagesPathParsedCorrectly() public void UserCanSpecifyBuildTimeout() { const int timeoutInSeconds = 10; - var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.0", "--buildTimeout", timeoutInSeconds.ToString() }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.1", "--buildTimeout", timeoutInSeconds.ToString() }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); CsProjCoreToolchain toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain; @@ -355,7 +362,7 @@ public void UserCanSpecifyBuildTimeout() [Fact] public void WhenUserDoesNotSpecifyTimeoutTheDefaultValueIsUsed() { - var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.0" }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.1" }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); CsProjCoreToolchain toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain; @@ -363,6 +370,22 @@ public void WhenUserDoesNotSpecifyTimeoutTheDefaultValueIsUsed() Assert.Equal(DefaultConfig.Instance.BuildTimeout, config.BuildTimeout); } + [Fact] + public void UserCanSpecifyWakeLock() + { + var config = ConfigParser.Parse(["--wakeLock", "Display"], new OutputLogger(Output)).config; + + Assert.Equal(WakeLockType.Display, config.WakeLock); + } + + [Fact] + public void WhenUserDoesNotSpecifyWakeLockTheDefaultValueIsUsed() + { + var config = ConfigParser.Parse([], new OutputLogger(Output)).config; + + Assert.Equal(DefaultConfig.Instance.WakeLock, config.WakeLock); + } + [Theory] [InlineData("net461")] [InlineData("net462")] @@ -383,9 +406,17 @@ public void NetFrameworkMonikerParsedCorrectly(string tfm) [Theory] [InlineData("net50")] + [InlineData("net5.0")] [InlineData("net60")] + [InlineData("net6.0")] [InlineData("net70")] + [InlineData("net7.0")] [InlineData("net80")] + [InlineData("net8.0")] + [InlineData("net90")] + [InlineData("net9.0")] + [InlineData("net10_0")] + [InlineData("net10.0")] public void NetMonikersAreRecognizedAsNetCoreMonikers(string tfm) { var config = ConfigParser.Parse(new[] { "-r", tfm }, new OutputLogger(Output)).config; @@ -412,56 +443,63 @@ public void PlatformSpecificMonikersAreSupported(string msBuildMoniker) [Fact] public void CanCompareFewDifferentRuntimes() { - var config = ConfigParser.Parse(new[] { "--runtimes", "net462", "MONO", "netcoreapp3.0", "nativeaot6.0", "nativeAOT7.0", "nativeAOT8.0" }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(["--runtimes", "net462", "MONO", "netcoreapp2.0", "nativeaot6.0", "nativeAOT7.0", "nativeAOT8.0"], + new OutputLogger(Output)).config; Assert.True(config.GetJobs().First().Meta.Baseline); // when the user provides multiple runtimes the first one should be marked as baseline - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is ClrRuntime clrRuntime && clrRuntime.MsBuildMoniker == "net462")); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is MonoRuntime)); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is CoreRuntime coreRuntime && coreRuntime.MsBuildMoniker == "netcoreapp3.0" && coreRuntime.RuntimeMoniker == RuntimeMoniker.NetCoreApp30)); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net6.0" && nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot60)); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net7.0" && nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot70)); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net8.0" && nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot80)); + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is ClrRuntime clrRuntime && clrRuntime.MsBuildMoniker == "net462"); + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is MonoRuntime); + Assert.Single(config.GetJobs(), job => + job.Environment.Runtime is CoreRuntime coreRuntime && coreRuntime.MsBuildMoniker == "netcoreapp2.0" && + coreRuntime.RuntimeMoniker == RuntimeMoniker.NetCoreApp20); + Assert.Single(config.GetJobs(), job => + job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net6.0" && + nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot60); + Assert.Single(config.GetJobs(), job => + job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net7.0" && + nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot70); + Assert.Single(config.GetJobs(), job => + job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net8.0" && + nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot80); } [Theory] - [InlineData(ThresholdUnit.Ratio, 5)] - [InlineData(ThresholdUnit.Milliseconds, 10)] - public void CanUseStatisticalTestsToCompareFewDifferentRuntimes(ThresholdUnit thresholdUnit, double thresholdValue) + [InlineData("5%")] + [InlineData("10ms")] + public void CanUseStatisticalTestsToCompareFewDifferentRuntimes(string threshold) { - var config = ConfigParser.Parse(new[] - { - "--runtimes", "netcoreapp2.1", "netcoreapp2.2", - "--statisticalTest", $"{thresholdValue.ToString(CultureInfo.InvariantCulture)}{thresholdUnit.ToShortName()}" - }, new OutputLogger(Output)).config; + string[] arguments = ["--runtimes", "net6.0", "net8.0", "--statisticalTest", threshold]; + var config = ConfigParser.Parse(arguments, new OutputLogger(Output)).config; var mockSummary = MockFactory.CreateSummary(config); Assert.True(config.GetJobs().First().Meta.Baseline); // when the user provides multiple runtimes the first one should be marked as baseline Assert.False(config.GetJobs().Last().Meta.Baseline); - var statisticalTestColumn = config.GetColumnProviders().SelectMany(columnProvider => columnProvider.GetColumns(mockSummary)).OfType().Single(); + var statisticalTestColumn = config.GetColumnProviders().SelectMany(columnProvider => columnProvider.GetColumns(mockSummary)) + .OfType().Single(); - Assert.Equal(StatisticalTestKind.MannWhitney, statisticalTestColumn.Kind); - Assert.Equal(Threshold.Create(thresholdUnit, thresholdUnit == ThresholdUnit.Ratio ? thresholdValue / 100.0 : thresholdValue), statisticalTestColumn.Threshold); + Assert.Equal(threshold, statisticalTestColumn.Threshold.ToString()); } [Fact] public void SpecifyingInvalidStatisticalTestsThresholdMeansFailure() { - Assert.False(ConfigParser.Parse(new[] {"--statisticalTest", "not a number" }, new OutputLogger(Output)).isSuccess); - Assert.False(ConfigParser.Parse(new[] {"--statisticalTest", "1unknownUnit" }, new OutputLogger(Output)).isSuccess); - Assert.False(ConfigParser.Parse(new[] {"--statisticalTest", "1 unknownUnit" }, new OutputLogger(Output)).isSuccess); - Assert.False(ConfigParser.Parse(new[] {"--statisticalTest", "%1" }, new OutputLogger(Output)).isSuccess); // reverse order - a typo + Assert.False(ConfigParser.Parse(["--statisticalTest", "not a number"], new OutputLogger(Output)).isSuccess); + Assert.False(ConfigParser.Parse(["--statisticalTest", "1unknownUnit"], new OutputLogger(Output)).isSuccess); + Assert.False(ConfigParser.Parse(["--statisticalTest", "1 unknownUnit"], new OutputLogger(Output)).isSuccess); + Assert.False(ConfigParser.Parse(["--statisticalTest", "%1"], new OutputLogger(Output)).isSuccess); // reverse order - a typo } [Fact] public void CanParseHardwareCounters() { - var config = ConfigParser.Parse(new[] { "--counters", $"{nameof(HardwareCounter.CacheMisses)}+{nameof(HardwareCounter.InstructionRetired)}"}, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "--counters", $"{nameof(HardwareCounter.CacheMisses)}+{nameof(HardwareCounter.InstructionRetired)}" }, + new OutputLogger(Output)).config; Assert.Equal(2, config.GetHardwareCounters().Count()); - Assert.Single(config.GetHardwareCounters().Where(counter => counter == HardwareCounter.CacheMisses)); - Assert.Single(config.GetHardwareCounters().Where(counter => counter == HardwareCounter.InstructionRetired)); + Assert.Single(config.GetHardwareCounters(), counter => counter == HardwareCounter.CacheMisses); + Assert.Single(config.GetHardwareCounters(), counter => counter == HardwareCounter.InstructionRetired); } [Fact] @@ -481,7 +519,7 @@ public void CanParseDisassemblerWithCustomRecursiveDepth() { const int depth = 123; - var config = ConfigParser.Parse(new[] { "--disasm", "--disasmDepth", depth.ToString()}, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "--disasm", "--disasmDepth", depth.ToString() }, new OutputLogger(Output)).config; var diagnoser = config.GetDiagnosers().OfType().Single(); @@ -517,7 +555,7 @@ public void UserCanSpecifyCustomDefaultJobAndOverwriteItsSettingsViaConsoleArgs( .WithWarmupCount(1) .AsDefault()); - var parsedConfig = ConfigParser.Parse(new[] { "--warmupCount", "2"}, new OutputLogger(Output), globalConfig).config; + var parsedConfig = ConfigParser.Parse(new[] { "--warmupCount", "2" }, new OutputLogger(Output), globalConfig).config; Assert.Equal(2, parsedConfig.GetJobs().Single().Run.WarmupCount); Assert.False(parsedConfig.GetJobs().Single().Meta.IsDefault); // after the merge the job is not "default" anymore @@ -672,4 +710,4 @@ public void CheckUpdateInvalidArgs(string strArgs) Assert.False(isSuccess); } } -} +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs index 2eae58c7c1..d77c5202ed 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs @@ -249,10 +249,10 @@ public void MissingDependencyIsNotAddedWhenItIsAlreadyPresent() [Fact] public void WhenTwoConfigsAreAddedTheRegularJobsAreJustAdded() { - var configWithClrJob = CreateConfigFromJobs(Job.Default.WithRuntime(CoreRuntime.Core21)); - var configWithCoreJob = CreateConfigFromJobs(Job.Default.WithRuntime(ClrRuntime.Net462)); + var configWithClrJob = CreateConfigFromJobs(Job.Default.WithRuntime(ClrRuntime.Net462)); + var configWithCoreJob = CreateConfigFromJobs(Job.Default.WithRuntime(CoreRuntime.Core80)); - foreach (var added in AddLeftToTheRightAndRightToTheLef(configWithClrJob, configWithCoreJob)) + foreach (var added in AddLeftToTheRightAndRightToTheLef(configWithCoreJob, configWithClrJob)) { var runnableJobs = added.GetJobs(); @@ -269,7 +269,7 @@ public void WhenTwoConfigsAreAddedTheMutatorJobsAreAppliedToAllOtherJobs() var configWithMutatorJob = CreateConfigFromJobs(Job.Default.WithWarmupCount(warmupCount).AsMutator()); var configWithTwoStandardJobs = CreateConfigFromJobs( Job.Default.WithRuntime(ClrRuntime.Net462), - Job.Default.WithRuntime(CoreRuntime.Core21)); + Job.Default.WithRuntime(CoreRuntime.Core80)); foreach (var added in AddLeftToTheRightAndRightToTheLef(configWithTwoStandardJobs, configWithMutatorJob)) { @@ -376,6 +376,45 @@ public void WhenTwoCustomTimeoutsAreProvidedTheLongerOneIsUsed(bool direction) Assert.Equal(TimeSpan.FromSeconds(2), final.BuildTimeout); } + [Fact] + public void WhenWakeLockIsNotSpecifiedTheDefaultValueIsUsed() + { + var mutable = ManualConfig.CreateEmpty(); + var final = ImmutableConfigBuilder.Create(mutable); + Assert.Equal(DefaultConfig.Instance.WakeLock, final.WakeLock); + } + + [Fact] + public void CustomWakeLockHasPrecedenceOverDefaultWakeLock() + { + WakeLockType customWakeLock = WakeLockType.Display; + var mutable = ManualConfig.CreateEmpty().WithWakeLock(customWakeLock); + var final = ImmutableConfigBuilder.Create(mutable); + Assert.Equal(customWakeLock, final.WakeLock); + } + + [Theory] + [InlineData(WakeLockType.None, WakeLockType.None, WakeLockType.None)] + [InlineData(WakeLockType.System, WakeLockType.None, WakeLockType.None)] + [InlineData(WakeLockType.Display, WakeLockType.None, WakeLockType.Display)] + [InlineData(WakeLockType.None, WakeLockType.System, WakeLockType.None)] + [InlineData(WakeLockType.System, WakeLockType.System, WakeLockType.System)] + [InlineData(WakeLockType.Display, WakeLockType.System, WakeLockType.Display)] + [InlineData(WakeLockType.None, WakeLockType.Display, WakeLockType.Display)] + [InlineData(WakeLockType.System, WakeLockType.Display, WakeLockType.Display)] + [InlineData(WakeLockType.Display, WakeLockType.Display, WakeLockType.Display)] + public void WhenTwoCustomWakeLocksAreProvidedDisplayBeatsNoneBeatsDefault( + WakeLockType left, WakeLockType right, WakeLockType expected) + { + var l = ManualConfig.CreateEmpty().WithWakeLock(left); + var r = ManualConfig.CreateEmpty().WithWakeLock(right); + + l.Add(r); + + var final = ImmutableConfigBuilder.Create(l); + Assert.Equal(expected, final.WakeLock); + } + private static ManualConfig CreateConfigFromJobs(params Job[] jobs) { var config = ManualConfig.CreateEmpty(); diff --git a/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs b/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs index 96f671e60d..fd47e9fd83 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs @@ -60,11 +60,11 @@ public static void Test01Create() Assert.False(j.Environment.Gc.AllowVeryLargeObjects); Assert.Equal(Platform.AnyCpu, j.Environment.Platform); Assert.Equal(RunStrategy.Throughput, j.Run.RunStrategy); // set by default - Assert.Equal("Default", j.Id); // id reset - Assert.True(j.DisplayInfo == "DefaultJob", "DisplayInfo = " + j.DisplayInfo); - Assert.True(j.ResolvedId == "DefaultJob", "ResolvedId = " + j.ResolvedId); + Assert.Equal("CustomId", j.Id); // id remains after unfreeze + Assert.Equal("CustomId", j.DisplayInfo); + Assert.Equal("CustomId", j.ResolvedId); Assert.Equal(j.ResolvedId, j.FolderInfo); - Assert.Equal("Default", j.Environment.Id); + Assert.Equal("CustomId", j.Environment.Id); // id remains after unfreeze // new job j = new Job(j.Freeze()); @@ -160,7 +160,7 @@ public static void Test02Modify() // 4. Freeze-unfreeze: j = j.Freeze().UnfreezeCopy(); - Assert.Equal("Platform=X86, LaunchCount=1", j.Id); + Assert.Equal("SomeId", j.Id); // id not lost Assert.Equal(Platform.X86, j.Environment.Platform); Assert.Equal(1, j.Run.LaunchCount); @@ -204,15 +204,15 @@ public static void Test03IdDoesNotFlow() Assert.False(j.HasValue(CharacteristicObject.IdCharacteristic)); Assert.False(j.Environment.HasValue(CharacteristicObject.IdCharacteristic)); - Job.EnvironmentCharacteristic[j] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will not flow - Assert.False(j.HasValue(CharacteristicObject.IdCharacteristic)); - Assert.False(j.Environment.HasValue(CharacteristicObject.IdCharacteristic)); + Job.EnvironmentCharacteristic[j] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will flow + Assert.True(j.HasValue(CharacteristicObject.IdCharacteristic)); + Assert.True(j.Environment.HasValue(CharacteristicObject.IdCharacteristic)); var c = new CharacteristicSet(EnvironmentMode.LegacyJitX64, RunMode.Long); // id will not flow, new CharacteristicSet Assert.False(c.HasValue(CharacteristicObject.IdCharacteristic)); - Job.EnvironmentCharacteristic[c] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will not flow - Assert.False(c.HasValue(CharacteristicObject.IdCharacteristic)); + Job.EnvironmentCharacteristic[c] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will flow + Assert.True(c.HasValue(CharacteristicObject.IdCharacteristic)); CharacteristicObject.IdCharacteristic[c] = "MyId"; // id set explicitly Assert.Equal("MyId", c.Id); @@ -221,12 +221,12 @@ public static void Test03IdDoesNotFlow() Assert.Equal("MyId", j.Id); Assert.Equal("MyId", j.Environment.Id); - Job.EnvironmentCharacteristic[j] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); // id will not flow - Assert.Equal("MyId", j.Id); - Assert.Equal("MyId", j.Environment.Id); + Job.EnvironmentCharacteristic[j] = EnvironmentMode.LegacyJitX86.UnfreezeCopy(); + Assert.Equal("LegacyJitX86", j.Id); + Assert.Equal("LegacyJitX86", j.Environment.Id); - j = j.WithJit(Jit.RyuJit); // custom id will flow - Assert.Equal("MyId", j.Id); + j = j.WithJit(Jit.RyuJit); + Assert.Equal("LegacyJitX86", j.Id); } [Fact] @@ -403,7 +403,7 @@ public static void Test06CharacteristicHacks() [Fact] public static void MutatorAppliedToOtherJobOverwritesOnlyTheConfiguredSettings() { - var jobBefore = Job.Default.WithRuntime(CoreRuntime.Core30); // this is a default job with Runtime set to Core + var jobBefore = Job.Default.WithRuntime(CoreRuntime.Core80); // this is a default job with Runtime set to Core var copy = jobBefore.UnfreezeCopy(); Assert.False(copy.HasValue(RunMode.MaxIterationCountCharacteristic)); @@ -474,6 +474,20 @@ public static void WithNuGet() Assert.Equal(expected, j.Infrastructure.NuGetReferences); // ensure that the list's equality operator returns true when the contents are the same } + [Fact] + public static void UnfreezeCopy_PreservesIdCharacteristic() + { + // Arrange + var original = new Job(); + original.SetValue(Job.IdCharacteristic, "TestID"); + + // Act + var copy = original.UnfreezeCopy(); + + // Assert + Assert.Equal("TestID", copy.GetValue(Job.IdCharacteristic)); + } + private static bool IsSubclassOfobModeOfItself(Type type) { Type jobModeOfT; diff --git a/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs b/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs index 715d7dad64..ef1225432e 100644 --- a/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs +++ b/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs @@ -18,10 +18,15 @@ namespace BenchmarkDotNet.Tests public class CsProjGeneratorTests { private FileInfo TestAssemblyFileInfo = new FileInfo(typeof(CsProjGeneratorTests).Assembly.Location); + private const string runtimeHostConfigurationOptionChunk = """ + + + +"""; [Theory] [InlineData("net471", false)] - [InlineData("netcoreapp3.0", true)] + [InlineData("netcoreapp3.1", true)] public void ItsPossibleToCustomizeProjectSdkBasedOnProjectSdkFromTheProjectFile(string targetFrameworkMoniker, bool isNetCore) { const string withCustomProjectSdk = @" @@ -36,7 +41,7 @@ public void ItsImpossibleToCustomizeProjectSdkForFullFrameworkAppsBasedOnTheImpo { const string withCustomProjectImport = @" - + "; AssertParsedSdkName(withCustomProjectImport, "net471", "Microsoft.NET.Sdk", false); @@ -47,10 +52,10 @@ public void ItsPossibleToCustomizeProjectSdkForNetCoreAppsBasedOnTheImportOfSdk( { const string withCustomProjectImport = @" - + "; - AssertParsedSdkName(withCustomProjectImport, "netcoreapp3.0", "Microsoft.NET.Sdk.WindowsDesktop", true); + AssertParsedSdkName(withCustomProjectImport, "netcoreapp3.1", "Microsoft.NET.Sdk.WindowsDesktop", true); } [AssertionMethod] @@ -68,7 +73,7 @@ private void AssertParsedSdkName(string csProjContent, string targetFrameworkMon private static void AssertCustomProperties(string expected, string actual) { - Assert.Equal(expected.Replace(Environment.NewLine, "\n").Replace("\n", Environment.NewLine), actual); + Assert.Equal(expected.Replace("\r", "").Replace("\n", Environment.NewLine), actual); } [Fact] @@ -82,7 +87,7 @@ public void UseWpfSettingGetsCopied() "; - var sut = new CsProjGenerator("netcoreapp3.0", null, null, null, true); + var sut = new CsProjGenerator("netcoreapp3.1", null, null, null, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(withUseWpfTrue); @@ -112,7 +117,7 @@ public void SettingsFromPropsFileImportedUsingAbsolutePathGetCopies() "; - var sut = new CsProjGenerator("netcoreapp3.0", null, null, null, true); + var sut = new CsProjGenerator("netcoreapp3.1", null, null, null, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(importingAbsolutePath); @@ -144,7 +149,7 @@ public void SettingsFromPropsFileImportedUsingRelativePathGetCopies() "; - var sut = new CsProjGenerator("netcoreapp3.0", null, null, null, true); + var sut = new CsProjGenerator("netcoreapp3.1", null, null, null, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(importingRelativePath); @@ -158,28 +163,40 @@ public void SettingsFromPropsFileImportedUsingRelativePathGetCopies() File.Delete(propsFilePath); } + [Fact] + public void RuntimeHostConfigurationOptionIsCopied() + { + string source = $@" + +{runtimeHostConfigurationOptionChunk} +"; + + var sut = new CsProjGenerator("netcoreapp3.1", null, null, null, true); + + var xmlDoc = new XmlDocument(); + xmlDoc.LoadXml(source); + var (customProperties, sdkName) = sut.GetSettingsThatNeedToBeCopied(xmlDoc, TestAssemblyFileInfo); + + AssertCustomProperties(runtimeHostConfigurationOptionChunk, customProperties); + Assert.Equal("Microsoft.NET.Sdk", sdkName); + } + [Fact] public void TheDefaultFilePathShouldBeUsedWhenAnAssemblyLocationIsEmpty() { const string programName = "testProgram"; var config = ManualConfig.CreateEmpty().CreateImmutableConfig(); - var benchmarkMethod = - typeof(MockFactory.MockBenchmarkClass) - .GetTypeInfo() - .GetMethods() - .Single(method => method.Name == nameof(MockFactory.MockBenchmarkClass.Foo)); - //Simulate loading an assembly from a stream var benchmarkDotNetAssembly = typeof(MockFactory.MockBenchmarkClass).GetTypeInfo().Assembly; var streamLoadedAssembly = Assembly.Load(File.ReadAllBytes(benchmarkDotNetAssembly.Location)); - var assemblyType = streamLoadedAssembly.GetRunnableBenchmarks().Select(type => type).FirstOrDefault(); + var assemblyType = streamLoadedAssembly.GetRunnableBenchmarks().Select(type => type).First(); - var target = new Descriptor(assemblyType, benchmarkMethod); + var target = new Descriptor(assemblyType, MockFactory.MockMethodInfo); var benchmarkCase = BenchmarkCase.Create(target, Job.Default, null, config); var benchmarks = new[] { new BenchmarkBuildInfo(benchmarkCase, config.CreateImmutableConfig(), 999) }; - var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.0", null, null, null, true); + var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.1", null, null, null, true); string binariesPath = projectGenerator.ResolvePathForBinaries(new BuildPartition(benchmarks, new Resolver()), programName); string expectedPath = Path.Combine(Path.Combine(Directory.GetCurrentDirectory(), "BenchmarkDotNet.Bin"), programName); @@ -190,15 +207,10 @@ public void TheDefaultFilePathShouldBeUsedWhenAnAssemblyLocationIsEmpty() public void TestAssemblyFilePathIsUsedWhenTheAssemblyLocationIsNotEmpty() { const string programName = "testProgram"; - var benchmarkMethod = - typeof(MockFactory.MockBenchmarkClass) - .GetTypeInfo() - .GetMethods() - .Single(method => method.Name == nameof(MockFactory.MockBenchmarkClass.Foo)); - var target = new Descriptor(typeof(MockFactory.MockBenchmarkClass), benchmarkMethod); + var target = new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo); var benchmarkCase = BenchmarkCase.Create(target, Job.Default, null, ManualConfig.CreateEmpty().CreateImmutableConfig()); var benchmarks = new[] { new BenchmarkBuildInfo(benchmarkCase, ManualConfig.CreateEmpty().CreateImmutableConfig(), 0) }; - var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.0", null, null, null, true); + var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.1", null, null, null, true); var buildPartition = new BuildPartition(benchmarks, new Resolver()); string binariesPath = projectGenerator.ResolvePathForBinaries(buildPartition, programName); diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/CpuInfoFormatterTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/CpuInfoFormatterTests.cs new file mode 100644 index 0000000000..342c38a1e8 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/CpuInfoFormatterTests.cs @@ -0,0 +1,39 @@ +using System.Text; +using System.Threading.Tasks; +using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Infra; +using Perfolizer.Helpers; +using Perfolizer.Phd.Dto; +using VerifyXunit; +using Xunit; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +[Collection("VerifyTests")] +[UsesVerify] +public class CpuInfoFormatterTests +{ + [Fact] + public Task FormatTest() + { + var captions = new StringBuilder(); + foreach (var processorName in new[] { null, "", "Intel" }) + foreach (var physicalProcessorCount in new int?[] { null, 0, 1, 2 }) + foreach (var physicalCoreCount in new int?[] { null, 0, 1, 2 }) + foreach (var logicalCoreCount in new int?[] { null, 0, 1, 2 }) + { + var cpu = new PhdCpu + { + ProcessorName = processorName, + PhysicalProcessorCount = physicalProcessorCount, + PhysicalCoreCount = physicalCoreCount, + LogicalCoreCount = logicalCoreCount, + }; + + captions.AppendLine(cpu.ToFullBrandName()); + } + + var settings = VerifyHelper.Create(); + return Verifier.Verify(captions.ToString(), settings); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/LinuxCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/LinuxCpuInfoParserTests.cs new file mode 100644 index 0000000000..596f54aba1 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/LinuxCpuInfoParserTests.cs @@ -0,0 +1,173 @@ +using BenchmarkDotNet.Detectors.Cpu.Linux; +using Perfolizer.Phd.Dto; +using Xunit; +using Xunit.Abstractions; +using static Perfolizer.Horology.Frequency; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +// ReSharper disable StringLiteralTypo +public class LinuxCpuInfoParserTests(ITestOutputHelper output) +{ + private ITestOutputHelper Output { get; } = output; + + [Fact] + public void EmptyTest() + { + var actual = LinuxCpuInfoParser.Parse("", ""); + var expected = new PhdCpu(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void MalformedTest() + { + var actual = LinuxCpuInfoParser.Parse("malformedkey: malformedvalue\n\nmalformedkey2: malformedvalue2", null); + var expected = new PhdCpu(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void TwoProcessorWithDifferentCoresCountTest() + { + string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoProcessorWithDifferentCoresCount.txt"); + var actual = LinuxCpuInfoParser.Parse(cpuInfo, null); + var expected = new PhdCpu + { + ProcessorName = "Unknown processor with 2 cores and hyper threading, Unknown processor with 4 cores", + PhysicalProcessorCount = 2, + PhysicalCoreCount = 6, + LogicalCoreCount = 8, + NominalFrequencyHz = 2_500_000_000, + MaxFrequencyHz = 2_500_000_000 + }; + Output.AssertEqual(expected, actual); + } + + + [Fact] + public void RealOneProcessorTwoCoresTest() + { + string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoRealOneProcessorTwoCores.txt"); + var actual = LinuxCpuInfoParser.Parse(cpuInfo, null); + var expected = new PhdCpu + { + ProcessorName = "Intel(R) Core(TM) i5-6200U CPU @ 2.30GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 2, + LogicalCoreCount = 4, + NominalFrequencyHz = 2_300_000_000, + MaxFrequencyHz = 2_300_000_000 + }; + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealOneProcessorFourCoresTest() + { + string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoRealOneProcessorFourCores.txt"); + var actual = LinuxCpuInfoParser.Parse(cpuInfo, null); + var expected = new PhdCpu + { + ProcessorName = "Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 4, + LogicalCoreCount = 8, + NominalFrequencyHz = 2_500_000_000, + MaxFrequencyHz = 2_500_000_000 + }; + Output.AssertEqual(expected, actual); + } + + // https://github.com/dotnet/BenchmarkDotNet/issues/2577 + [Fact] + public void Issue2577Test() + { + const string cpuInfo = + """ + processor : 0 + BogoMIPS : 50.00 + Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm lrcpc dcpop asimddp + CPU implementer : 0x41 + CPU architecture: 8 + CPU variant : 0x3 + CPU part : 0xd0c + CPU revision : 1 + """; + const string lscpu = + """ + Architecture: aarch64 + CPU op-mode(s): 32-bit, 64-bit + Byte Order: Little Endian + CPU(s): 16 + On-line CPU(s) list: 0-15 + Vendor ID: ARM + Model name: Neoverse-N1 + Model: 1 + Thread(s) per core: 1 + Core(s) per socket: 16 + Socket(s): 1 + Stepping: r3p1 + BogoMIPS: 50.00 + Flags: fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm lrcpc dcpop asimddp + """; + var actual = LinuxCpuInfoParser.Parse(cpuInfo, lscpu); + var expected = new PhdCpu { ProcessorName = "Neoverse-N1", PhysicalCoreCount = 16 }; + Output.AssertEqual(expected, actual); + } + + [Fact] + public void AmdRyzen9_7950X() + { + string cpuInfo = TestHelper.ReadTestFile("ryzen9-cpuinfo.txt"); + const string lscpu = + """ + Architecture: x86_64 + CPU op-mode(s): 32-bit, 64-bit + Address sizes: 48 bits physical, 48 bits virtual + Byte Order: Little Endian + CPU(s): 32 + On-line CPU(s) list: 0-31 + Vendor ID: AuthenticAMD + Model name: AMD Ryzen 9 7950X 16-Core Processor + CPU family: 25 + Model: 97 + Thread(s) per core: 2 + Core(s) per socket: 16 + Socket(s): 1 + Stepping: 2 + CPU(s) scaling MHz: 41% + CPU max MHz: 5881.0000 + CPU min MHz: 400.0000 + BogoMIPS: 8983.23 + Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl + pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb + bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512 + cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean + flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succo + r smca fsrm flush_l1d + """; + var actual = LinuxCpuInfoParser.Parse(cpuInfo, lscpu); + var expected = new PhdCpu + { + ProcessorName = "AMD Ryzen 9 7950X 16-Core Processor", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 16, + LogicalCoreCount = 32, + NominalFrequencyHz = 5_881_000_000, + MaxFrequencyHz = 5_881_000_000 + }; + Output.AssertEqual(expected, actual); + } + + [Theory] + [InlineData("Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", 2.50)] + [InlineData("Intel(R) Core(TM) i5-6200U CPU @ 2.30GHz", 2.30)] + [InlineData("Unknown processor with 2 cores and hyper threading, Unknown processor with 4 cores", 0)] + [InlineData("Intel(R) Core(TM) i5-2500 CPU @ 3.30GHz", 3.30)] + public void ParseFrequencyFromBrandStringTests(string brandString, double expectedGHz) + { + var frequency = LinuxCpuInfoParser.ParseFrequencyFromBrandString(brandString) ?? Zero; + Assert.Equal(FromGHz(expectedGHz), frequency); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/SysctlCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/SysctlCpuInfoParserTests.cs new file mode 100644 index 0000000000..208a7c8373 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/SysctlCpuInfoParserTests.cs @@ -0,0 +1,45 @@ +using BenchmarkDotNet.Detectors.Cpu.macOS; +using Perfolizer.Phd.Dto; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +// ReSharper disable StringLiteralTypo +public class SysctlCpuInfoParserTests(ITestOutputHelper output) +{ + private ITestOutputHelper Output { get; } = output; + + [Fact] + public void EmptyTest() + { + var actual = SysctlCpuInfoParser.Parse(string.Empty); + var expected = new PhdCpu(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void MalformedTest() + { + var actual = SysctlCpuInfoParser.Parse("malformedkey=malformedvalue\n\nmalformedkey2=malformedvalue2"); + var expected = new PhdCpu(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealOneProcessorFourCoresTest() + { + string cpuInfo = TestHelper.ReadTestFile("SysctlRealOneProcessorFourCores.txt"); + var actual = SysctlCpuInfoParser.Parse(cpuInfo); + var expected = new PhdCpu + { + ProcessorName = "Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 4, + LogicalCoreCount = 8, + NominalFrequencyHz = 2_200_000_000, + MaxFrequencyHz = 2_200_000_000 + }; + Output.AssertEqual(expected, actual); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoProcessorWithDifferentCoresCount.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoProcessorWithDifferentCoresCount.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoProcessorWithDifferentCoresCount.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoProcessorWithDifferentCoresCount.txt diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoRealOneProcessorFourCores.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoRealOneProcessorFourCores.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoRealOneProcessorFourCores.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoRealOneProcessorFourCores.txt diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoRealOneProcessorTwoCores.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoRealOneProcessorTwoCores.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoRealOneProcessorTwoCores.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoRealOneProcessorTwoCores.txt diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/SysctlRealOneProcessorFourCores.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/SysctlRealOneProcessorFourCores.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/SysctlRealOneProcessorFourCores.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/SysctlRealOneProcessorFourCores.txt diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ryzen9-cpuinfo.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ryzen9-cpuinfo.txt new file mode 100644 index 0000000000..d942a8d484 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ryzen9-cpuinfo.txt @@ -0,0 +1,896 @@ +processor : 0 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5346.955 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 0 +cpu cores : 16 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 1 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5487.348 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 1 +cpu cores : 16 +apicid : 2 +initial apicid : 2 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 2 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5479.603 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 2 +cpu cores : 16 +apicid : 4 +initial apicid : 4 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 3 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5601.456 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 3 +cpu cores : 16 +apicid : 6 +initial apicid : 6 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 4 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5410.513 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 4 +cpu cores : 16 +apicid : 8 +initial apicid : 8 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 5 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5493.255 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 5 +cpu cores : 16 +apicid : 10 +initial apicid : 10 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 6 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4422.161 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 6 +cpu cores : 16 +apicid : 12 +initial apicid : 12 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 7 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5505.019 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 7 +cpu cores : 16 +apicid : 14 +initial apicid : 14 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 8 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5414.920 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 8 +cpu cores : 16 +apicid : 16 +initial apicid : 16 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 9 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5060.967 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 9 +cpu cores : 16 +apicid : 18 +initial apicid : 18 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 10 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 3352.555 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 10 +cpu cores : 16 +apicid : 20 +initial apicid : 20 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 11 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5400.772 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 11 +cpu cores : 16 +apicid : 22 +initial apicid : 22 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 12 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5415.333 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 12 +cpu cores : 16 +apicid : 24 +initial apicid : 24 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 13 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 3236.400 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 13 +cpu cores : 16 +apicid : 26 +initial apicid : 26 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 14 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4519.257 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 14 +cpu cores : 16 +apicid : 28 +initial apicid : 28 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 15 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5414.841 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 15 +cpu cores : 16 +apicid : 30 +initial apicid : 30 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 16 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4738.471 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 0 +cpu cores : 16 +apicid : 1 +initial apicid : 1 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 17 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5599.000 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 1 +cpu cores : 16 +apicid : 3 +initial apicid : 3 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 18 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5477.067 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 2 +cpu cores : 16 +apicid : 5 +initial apicid : 5 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 19 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5507.543 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 3 +cpu cores : 16 +apicid : 7 +initial apicid : 7 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 20 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4401.379 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 4 +cpu cores : 16 +apicid : 9 +initial apicid : 9 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 21 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 400.000 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 5 +cpu cores : 16 +apicid : 11 +initial apicid : 11 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 22 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5607.945 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 6 +cpu cores : 16 +apicid : 13 +initial apicid : 13 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 23 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5280.720 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 7 +cpu cores : 16 +apicid : 15 +initial apicid : 15 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 24 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5414.893 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 8 +cpu cores : 16 +apicid : 17 +initial apicid : 17 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 25 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5414.888 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 9 +cpu cores : 16 +apicid : 19 +initial apicid : 19 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 26 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5385.641 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 10 +cpu cores : 16 +apicid : 21 +initial apicid : 21 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 27 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 3130.265 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 11 +cpu cores : 16 +apicid : 23 +initial apicid : 23 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 28 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5400.876 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 12 +cpu cores : 16 +apicid : 25 +initial apicid : 25 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 29 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4279.794 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 13 +cpu cores : 16 +apicid : 27 +initial apicid : 27 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 30 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 3124.980 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 14 +cpu cores : 16 +apicid : 29 +initial apicid : 29 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 31 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5389.561 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 15 +cpu cores : 16 +apicid : 31 +initial apicid : 31 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestHelper.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestHelper.cs new file mode 100644 index 0000000000..806035da29 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestHelper.cs @@ -0,0 +1,38 @@ +using System.IO; +using System.Reflection; +using JetBrains.Annotations; +using Perfolizer.Helpers; +using Perfolizer.Phd.Dto; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +public static class TestHelper +{ + public static string ReadTestFile(string name) + { + var assembly = typeof(TestHelper).GetTypeInfo().Assembly; + string resourceName = $"{typeof(TestHelper).Namespace}.TestFiles.{name}"; + + using var stream = assembly.GetManifestResourceStream(resourceName); + if (stream == null) + throw new FileNotFoundException($"Resource {resourceName} not found in {assembly.FullName}"); + + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } + + [AssertionMethod] + public static void AssertEqual(this ITestOutputHelper output, PhdCpu expected, PhdCpu actual) + { + output.WriteLine($"Expected : {expected.ToFullBrandName()}"); + output.WriteLine($"Actual : {actual.ToFullBrandName()}"); + Assert.Equal(expected.ProcessorName, actual.ProcessorName); + Assert.Equal(expected.PhysicalProcessorCount, actual.PhysicalProcessorCount); + Assert.Equal(expected.PhysicalCoreCount, actual.PhysicalCoreCount); + Assert.Equal(expected.LogicalCoreCount, actual.LogicalCoreCount); + Assert.Equal(expected.NominalFrequency(), actual.NominalFrequency()); + Assert.Equal(expected.MaxFrequency(), actual.MaxFrequency()); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/VerifiedFiles/CpuInfoFormatterTests.FormatTest.verified.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/VerifiedFiles/CpuInfoFormatterTests.FormatTest.verified.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/VerifiedFiles/CpuInfoFormatterTests.FormatTest.verified.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/VerifiedFiles/CpuInfoFormatterTests.FormatTest.verified.txt diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/WmicCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/WmicCpuInfoParserTests.cs new file mode 100644 index 0000000000..3ee84a8e44 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/WmicCpuInfoParserTests.cs @@ -0,0 +1,116 @@ +using BenchmarkDotNet.Detectors.Cpu.Windows; +using Perfolizer.Phd.Dto; +using Xunit; +using Xunit.Abstractions; +using static Perfolizer.Horology.Frequency; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +// ReSharper disable StringLiteralTypo +public class WmicCpuInfoParserTests(ITestOutputHelper output) +{ + private ITestOutputHelper Output { get; } = output; + + [Fact] + public void EmptyTest() + { + var actual = WmicCpuInfoParser.Parse(string.Empty); + var expected = new PhdCpu(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void MalformedTest() + { + var actual = WmicCpuInfoParser.Parse("malformedkey=malformedvalue\n\nmalformedkey2=malformedvalue2"); + var expected = new PhdCpu(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealTwoProcessorEightCoresTest() + { + const string cpuInfo = @" + +MaxClockSpeed=2400 +Name=Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz +NumberOfCores=8 +NumberOfLogicalProcessors=16 + + +MaxClockSpeed=2400 +Name=Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz +NumberOfCores=8 +NumberOfLogicalProcessors=16 + +"; + var actual = WmicCpuInfoParser.Parse(cpuInfo); + var expected = new PhdCpu + { + ProcessorName = "Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz", + PhysicalProcessorCount = 2, + PhysicalCoreCount = 16, + LogicalCoreCount = 32, + NominalFrequencyHz = 2_400_000_000, + MaxFrequencyHz = 2_400_000_000, + }; + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealTwoProcessorEightCoresWithWmicBugTest() + { + const string cpuInfo = + "\r\r\n" + + "\r\r\n" + + "MaxClockSpeed=3111\r\r\n" + + "Name=Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz\r\r\n" + + "NumberOfCores=8\r\r\n" + + "NumberOfLogicalProcessors=16\r\r\n" + + "\r\r\n" + + "\r\r\n" + + "MaxClockSpeed=3111\r\r\n" + + "Name=Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz\r\r\n" + + "NumberOfCores=8\r\r\n" + + "NumberOfLogicalProcessors=16\r\r\n" + + "\r\r\n" + + "\r\r\n" + + "\r\r\n"; + var actual = WmicCpuInfoParser.Parse(cpuInfo); + var expected = new PhdCpu + { + ProcessorName = "Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz", + PhysicalProcessorCount = 2, + PhysicalCoreCount = 16, + LogicalCoreCount = 32, + NominalFrequencyHz = 3_111_000_000, + MaxFrequencyHz = 3_111_000_000, + }; + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealOneProcessorFourCoresTest() + { + const string cpuInfo = @" + +MaxClockSpeed=2500 +Name=Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz +NumberOfCores=4 +NumberOfLogicalProcessors=8 + +"; + + var actual = WmicCpuInfoParser.Parse(cpuInfo); + var expected = new PhdCpu + { + ProcessorName = "Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 4, + LogicalCoreCount = 8, + NominalFrequencyHz = 2_500_000_000, + MaxFrequencyHz = 2_500_000_000, + }; + Output.AssertEqual(expected, actual); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs index 52136089cb..1f966fbcaa 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs @@ -81,7 +81,7 @@ public class WithForceAutoWarmup public void Method() { } } - private void AutoTest(Func measure, int min, int max = -1, IterationMode mode = IterationMode.Workload, Job job = null) + private void AutoTest(Func measure, int min, int max = -1, IterationMode mode = IterationMode.Workload, Job? job = null) { if (max == -1) max = min; diff --git a/tests/BenchmarkDotNet.Tests/Environments/HostEnvironmentInfoTests.cs b/tests/BenchmarkDotNet.Tests/Environments/HostEnvironmentInfoTests.cs index 977be28869..a49d9ea2e7 100644 --- a/tests/BenchmarkDotNet.Tests/Environments/HostEnvironmentInfoTests.cs +++ b/tests/BenchmarkDotNet.Tests/Environments/HostEnvironmentInfoTests.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Portability; using BenchmarkDotNet.Tests.Builders; using JetBrains.Annotations; +using Perfolizer.Helpers; using Xunit; namespace BenchmarkDotNet.Tests.Environments @@ -22,7 +23,7 @@ public void ReturnsHypervisorNameWhenItsDetected(string hypervisorName) string line = info.ToFormattedString().First(); string expected = $"{HostEnvironmentInfo.BenchmarkDotNetCaption} v{info.BenchmarkDotNetVersion}, " + - $"{info.OsVersion.Value} ({hypervisor.Name})"; + $"{info.Os.Value.ToBrandString()} ({hypervisor.Name})"; Assert.Equal(expected, line); } @@ -46,7 +47,7 @@ public void DoesntReturnHypervisorNameWhenItsNotDetected() string line = info.ToFormattedString().First(); string expected = $"{HostEnvironmentInfo.BenchmarkDotNetCaption} v{info.BenchmarkDotNetVersion}, " + - $"{info.OsVersion.Value}"; + $"{info.Os.Value.ToBrandString()}"; Assert.Equal(expected, line); } } diff --git a/tests/BenchmarkDotNet.Tests/Environments/OsBrandStringTests.cs b/tests/BenchmarkDotNet.Tests/Environments/OsBrandStringTests.cs deleted file mode 100644 index 8aafe03fba..0000000000 --- a/tests/BenchmarkDotNet.Tests/Environments/OsBrandStringTests.cs +++ /dev/null @@ -1,69 +0,0 @@ -using BenchmarkDotNet.Environments; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Environments -{ - public class OsBrandStringTests - { - private readonly ITestOutputHelper output; - - public OsBrandStringTests(ITestOutputHelper output) => this.output = output; - - private void Check(string actual, string expected) - { - output.WriteLine("LENGTH : " + actual.Length); - output.WriteLine("ACTUAL : " + actual); - output.WriteLine("EXPECTED : " + expected); - Assert.Equal(expected, actual); - - // The line with OsBrandString is one of the longest lines in the summary. - // When people past in on GitHub, it can be a reason of an ugly horizontal scrollbar. - // To avoid this, we are trying to minimize this line and use the minimum possible number of characters. - // In this test, we check that the length of the OS brand string for typical Windows versions - // is less than 61 characters. - const int maxLength = 61; - Assert.True(actual.Length <= maxLength, $"The brand string name length should be <= {maxLength}"); - } - - [Theory] - [InlineData("6.3.9600", "Windows 8.1 (6.3.9600)")] - [InlineData("10.0.14393", "Windows 10 (10.0.14393/1607/AnniversaryUpdate/Redstone1)")] - public void WindowsIsPrettified(string originalVersion, string prettifiedName) - => Check(OsBrandStringHelper.Prettify("Windows", originalVersion), prettifiedName); - - [Theory] - [InlineData("10.0.10240", 17797, "Windows 10 (10.0.10240.17797/1507/RTM/Threshold1)")] - [InlineData("10.0.10586", 1478, "Windows 10 (10.0.10586.1478/1511/NovemberUpdate/Threshold2)")] - [InlineData("10.0.14393", 2156, "Windows 10 (10.0.14393.2156/1607/AnniversaryUpdate/Redstone1)")] - [InlineData("10.0.15063", 997, "Windows 10 (10.0.15063.997/1703/CreatorsUpdate/Redstone2)")] - [InlineData("10.0.16299", 334, "Windows 10 (10.0.16299.334/1709/FallCreatorsUpdate/Redstone3)")] - [InlineData("10.0.17134", 48, "Windows 10 (10.0.17134.48/1803/April2018Update/Redstone4)")] - [InlineData("10.0.17763", 1, "Windows 10 (10.0.17763.1/1809/October2018Update/Redstone5)")] - [InlineData("10.0.18362", 693, "Windows 10 (10.0.18362.693/1903/May2019Update/19H1)")] - [InlineData("10.0.18363", 657, "Windows 10 (10.0.18363.657/1909/November2019Update/19H2)")] - [InlineData("10.0.19041", 1, "Windows 10 (10.0.19041.1/2004/May2020Update/20H1)")] - [InlineData("10.0.19042", 746, "Windows 10 (10.0.19042.746/20H2/October2020Update)")] - [InlineData("10.0.19043", 964, "Windows 10 (10.0.19043.964/21H1/May2021Update)")] - [InlineData("10.0.19044", 1147, "Windows 10 (10.0.19044.1147/21H2/November2021Update)")] - [InlineData("10.0.19099", 1729, "Windows 10 (10.0.19099.1729)")] - [InlineData("10.0.19045", 0, "Windows 10 (10.0.19045.0/22H2/2022Update)")] - [InlineData("10.0.22000", 348, "Windows 11 (10.0.22000.348/21H2/SunValley)")] - [InlineData("10.0.22518", 1012, "Windows 11 (10.0.22518.1012)")] - [InlineData("10.0.22621", 0, "Windows 11 (10.0.22621.0/22H2/2022Update/SunValley2)")] - public void WindowsWithUbrIsPrettified(string originalVersion, int ubr, string prettifiedName) - => Check(OsBrandStringHelper.Prettify("Windows", originalVersion, ubr), prettifiedName); - - [Theory] - [InlineData("macOS 10.12.6 (16G29)", "Darwin 16.7.0", "macOS Sierra 10.12.6 (16G29) [Darwin 16.7.0]")] - [InlineData("macOS 10.13.4 (17E199)", "Darwin 17.5.0", "macOS High Sierra 10.13.4 (17E199) [Darwin 17.5.0]")] - [InlineData("macOS 10.15.4 (19E266)", "Darwin 19.4.0", "macOS Catalina 10.15.4 (19E266) [Darwin 19.4.0]")] - [InlineData("macOS 11.1 (20C69)", "Darwin 20.2.0", "macOS Big Sur 11.1 (20C69) [Darwin 20.2.0]")] - [InlineData("macOS 11.3.1 (20E241)", "Darwin 20.4.0", "macOS Big Sur 11.3.1 (20E241) [Darwin 20.4.0]")] - [InlineData("macOS 12.1 (21C52)", "Darwin 21.2.0", "macOS Monterey 12.1 (21C52) [Darwin 21.2.0]")] - [InlineData("macOS 13.0.1 (22A400)", "Darwin 22.1.0", "macOS Ventura 13.0.1 (22A400) [Darwin 22.1.0]")] - [InlineData("macOS 14.0.0", "Darwin 23.0.0", "macOS Sonoma 14.0.0 [Darwin 23.0.0]")] - public void MacOSXIsPrettified(string systemVersion, string kernelVersion, string prettifiedName) - => Check(OsBrandStringHelper.PrettifyMacOSX(systemVersion, kernelVersion), prettifiedName); - } -} diff --git a/tests/BenchmarkDotNet.Tests/Environments/ProcessorBrandStringTests.cs b/tests/BenchmarkDotNet.Tests/Environments/ProcessorBrandStringTests.cs deleted file mode 100644 index 7911d5a135..0000000000 --- a/tests/BenchmarkDotNet.Tests/Environments/ProcessorBrandStringTests.cs +++ /dev/null @@ -1,73 +0,0 @@ -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; -using Xunit; - -namespace BenchmarkDotNet.Tests.Environments -{ - public class ProcessorBrandStringTests - { - [Theory] - [InlineData("Intel(R) Pentium(TM) G4560 CPU @ 3.50GHz", "Intel Pentium G4560 CPU 3.50GHz")] - [InlineData("Intel(R) Core(TM) i7 CPU 970 @ 3.20GHz", "Intel Core i7 CPU 970 3.20GHz (Nehalem)")] // Nehalem/Westmere/Gulftown - [InlineData("Intel(R) Core(TM) i7-920 CPU @ 2.67GHz", "Intel Core i7-920 CPU 2.67GHz (Nehalem)")] - [InlineData("Intel(R) Core(TM) i7-2600 CPU @ 3.40GHz", "Intel Core i7-2600 CPU 3.40GHz (Sandy Bridge)")] - [InlineData("Intel(R) Core(TM) i7-3770 CPU @ 3.40GHz", "Intel Core i7-3770 CPU 3.40GHz (Ivy Bridge)")] - [InlineData("Intel(R) Core(TM) i7-4770K CPU @ 3.50GHz", "Intel Core i7-4770K CPU 3.50GHz (Haswell)")] - [InlineData("Intel(R) Core(TM) i7-5775R CPU @ 3.30GHz", "Intel Core i7-5775R CPU 3.30GHz (Broadwell)")] - [InlineData("Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz", "Intel Core i7-6700HQ CPU 2.60GHz (Skylake)")] - [InlineData("Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz", "Intel Core i7-7700 CPU 3.60GHz (Kaby Lake)")] - [InlineData("Intel(R) Core(TM) i7-8650U CPU @ 1.90GHz ", "Intel Core i7-8650U CPU 1.90GHz (Kaby Lake R)")] - [InlineData("Intel(R) Core(TM) i7-8700K CPU @ 3.70GHz", "Intel Core i7-8700K CPU 3.70GHz (Coffee Lake)")] - public void IntelCoreIsPrettified(string originalName, string prettifiedName) => - Assert.Equal(prettifiedName, ProcessorBrandStringHelper.Prettify(new CpuInfo(originalName, nominalFrequency: null))); - - [Theory] - [InlineData("Intel(R) Pentium(TM) G4560 CPU @ 3.50GHz", "Intel Pentium G4560 CPU 3.50GHz (Max: 3.70GHz)", 3.7)] - [InlineData("Intel(R) Core(TM) i5-2500 CPU @ 3.30GHz", "Intel Core i5-2500 CPU 3.30GHz (Max: 3.70GHz) (Sandy Bridge)", 3.7)] - [InlineData("Intel(R) Core(TM) i7-2600 CPU @ 3.40GHz", "Intel Core i7-2600 CPU 3.40GHz (Max: 3.70GHz) (Sandy Bridge)", 3.7)] - [InlineData("Intel(R) Core(TM) i7-3770 CPU @ 3.40GHz", "Intel Core i7-3770 CPU 3.40GHz (Max: 3.50GHz) (Ivy Bridge)", 3.5)] - [InlineData("Intel(R) Core(TM) i7-4770K CPU @ 3.50GHz", "Intel Core i7-4770K CPU 3.50GHz (Max: 3.60GHz) (Haswell)", 3.6)] - [InlineData("Intel(R) Core(TM) i7-5775R CPU @ 3.30GHz", "Intel Core i7-5775R CPU 3.30GHz (Max: 3.40GHz) (Broadwell)", 3.4)] - public void CoreIsPrettifiedWithDiffFrequencies(string originalName, string prettifiedName, double actualFrequency) - { - Assert.Equal(prettifiedName, ProcessorBrandStringHelper.Prettify(new CpuInfo(originalName, nominalFrequency: Frequency.FromGHz(actualFrequency)), includeMaxFrequency: true)); - } - - [Theory] - [InlineData("AMD Ryzen 7 2700X Eight-Core Processor", "AMD Ryzen 7 2700X 4.10GHz", 4.1, 8, 16)] - [InlineData("AMD Ryzen 7 2700X Eight-Core Processor", "AMD Ryzen 7 2700X Eight-Core Processor 4.10GHz", 4.1, null, null)] - public void AmdIsPrettifiedWithDiffFrequencies(string originalName, string prettifiedName, double actualFrequency, int? physicalCoreCount, int? logicalCoreCount) - { - var cpuInfo = new CpuInfo( - originalName, - physicalProcessorCount: null, - physicalCoreCount, - logicalCoreCount, - Frequency.FromGHz(actualFrequency), - minFrequency: null, - maxFrequency: null); - - Assert.Equal(prettifiedName, ProcessorBrandStringHelper.Prettify(cpuInfo, includeMaxFrequency: true)); - } - - [Theory] - [InlineData("8130U", "Kaby Lake")] - [InlineData("9900K", "Coffee Lake")] - [InlineData("8809G", "Kaby Lake G")] - public void IntelCoreMicroarchitecture(string modelNumber, string microarchitecture) - { - Assert.Equal(microarchitecture, ProcessorBrandStringHelper.ParseIntelCoreMicroarchitecture(modelNumber)); - } - - [Theory] - [InlineData("", "Unknown processor")] - [InlineData(null, "Unknown processor")] - public void UnknownProcessorDoesNotThrow(string originalName, string prettifiedName) - { - var cpuInfo = new CpuInfo(originalName, nominalFrequency: null); - - Assert.Equal(prettifiedName, ProcessorBrandStringHelper.Prettify(cpuInfo, includeMaxFrequency: true)); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs b/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs index b3e142c6c5..5ab48b7a30 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs +++ b/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs @@ -5,12 +5,16 @@ using System.Threading.Tasks; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Exporters.Json; using BenchmarkDotNet.Exporters.Xml; using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Infra; using BenchmarkDotNet.Tests.Mocks; +using BenchmarkDotNet.Tests.Reports; using JetBrains.Annotations; using VerifyXunit; using Xunit; @@ -51,10 +55,15 @@ public Task Exporters(string cultureInfoName) foreach (var exporter in exporters) { PrintTitle(logger, exporter); - exporter.ExportToLog(MockFactory.CreateSummary(config.WithCultureInfo(cultureInfo)), logger); + exporter.ExportToLog( + MockFactory.CreateSummary( + config.WithCultureInfo(cultureInfo), + hugeSd: false, + [new Metric(new FakeMetricDescriptor("CacheMisses", "Hardware counter 'CacheMisses' per single operation", "N0"), 7)]), + logger); } - var settings = VerifySettingsFactory.Create(); + var settings = VerifyHelper.Create(); settings.UseTextForParameters(GetName(cultureInfo)); return Verifier.Verify(logger.GetLog(), settings); } @@ -98,6 +107,8 @@ private static IEnumerable GetExporters() .AddColumn(StatisticColumn.Mean) .AddColumn(StatisticColumn.StdDev) .AddColumn(StatisticColumn.P67) + .AddHardwareCounters(HardwareCounter.CacheMisses) + .AddColumnProvider(DefaultColumnProviders.Metrics) .AddDiagnoser(Diagnosers.MemoryDiagnoser.Default); public void Dispose() diff --git a/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterVerifyTests.cs b/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterVerifyTests.cs index 950f8498f8..8da0a0e24e 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterVerifyTests.cs +++ b/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterVerifyTests.cs @@ -10,6 +10,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Infra; using BenchmarkDotNet.Validators; using JetBrains.Annotations; using VerifyXunit; @@ -21,9 +22,7 @@ namespace BenchmarkDotNet.Tests.Exporters [UsesVerify] public class MarkdownExporterVerifyTests : IDisposable { - private readonly CultureInfo initCulture; - - public MarkdownExporterVerifyTests() => initCulture = Thread.CurrentThread.CurrentCulture; + private readonly CultureInfo initCulture = Thread.CurrentThread.CurrentCulture; [UsedImplicitly] public static TheoryData GetGroupBenchmarkTypes() @@ -54,7 +53,7 @@ public Task GroupExporterTest(Type benchmarkType) foreach (var error in errors) logger.WriteLineError("* " + error.Message); - var settings = VerifySettingsFactory.Create(); + var settings = VerifyHelper.Create(); settings.UseTextForParameters(benchmarkType.Name); return Verifier.Verify(logger.GetLog(), settings); } diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt index cd3482c04f..b406e20dd1 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt @@ -4,18 +4,18 @@ AsciiDocExporter .... BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 .... [options="header"] |=== -|Method |Mean |Error |StdDev |P67 -|Foo | 1.000 ns| NA| 0.000 ns| 1.000 ns -|Bar | 1.000 ns| NA| 0.000 ns| 1.000 ns +|Method |Mean |Error |StdDev |P67 |CacheMisses +|Foo | 1.000 ns| 0.000 ns| 0.000 ns| 1.000 ns| 7 +|Bar | 1.000 ns| 0.000 ns| 0.000 ns| 1.000 ns| 7 |=== ############################################ HtmlExporter @@ -37,19 +37,19 @@ HtmlExporter 

 BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V)
 MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
-Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC
-  [Host]  : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
-  LongRun : extra output line
+Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC
+.NET Core SDK 1.0.x.mock
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
 
Job=LongRun  IterationCount=100  LaunchCount=3  
 WarmupCount=15
- + - - + +
MethodMeanErrorStdDevP67
MethodMeanErrorStdDevP67CacheMisses
Foo1.000 nsNA0.000 ns1.000 ns
Bar1.000 nsNA0.000 ns1.000 ns
Foo1.000 ns0.000 ns0.000 ns1.000 ns7
Bar1.000 ns0.000 ns0.000 ns1.000 ns7
@@ -71,7 +71,7 @@ JsonExporter-brief "HasAttachedDebugger":false, "HasRyuJit":true, "Configuration":"CONFIGURATION", - "DotNetCliVersion":null, + "DotNetCliVersion":"1.0.x.mock", "ChronometerFrequency":{ "Hertz":2531248 }, @@ -89,9 +89,9 @@ JsonExporter-brief "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -116,13 +116,13 @@ JsonExporter-brief "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -154,9 +154,9 @@ JsonExporter-brief "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -181,13 +181,13 @@ JsonExporter-brief "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -214,7 +214,7 @@ JsonExporter-brief ############################################ JsonExporter-brief-compressed ############################################ -{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":null,"ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}}]} +{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":"1.0.x.mock","ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}}]} ############################################ JsonExporter-full ############################################ @@ -233,7 +233,7 @@ JsonExporter-full "HasAttachedDebugger":false, "HasRyuJit":true, "Configuration":"CONFIGURATION", - "DotNetCliVersion":null, + "DotNetCliVersion":"1.0.x.mock", "ChronometerFrequency":{ "Hertz":2531248 }, @@ -251,9 +251,9 @@ JsonExporter-full "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -278,13 +278,13 @@ JsonExporter-full "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -313,6 +313,56 @@ JsonExporter-full "IterationIndex":1, "Operations":1, "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":2, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":3, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":4, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":5, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":6, + "Operations":1, + "Nanoseconds":1 + } + ], + "Metrics":[ + { + "Value":7, + "Descriptor":{ + "Id":"CacheMisses", + "DisplayName":"CacheMisses", + "Legend":"Hardware counter 'CacheMisses' per single operation", + "NumberFormat":"N0", + "UnitType":0, + "Unit":null, + "TheGreaterTheBetter":false, + "PriorityInCategory":0 + } } ] },{ @@ -326,9 +376,9 @@ JsonExporter-full "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -353,13 +403,13 @@ JsonExporter-full "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -388,6 +438,56 @@ JsonExporter-full "IterationIndex":1, "Operations":1, "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":2, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":3, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":4, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":5, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":6, + "Operations":1, + "Nanoseconds":1 + } + ], + "Metrics":[ + { + "Value":7, + "Descriptor":{ + "Id":"CacheMisses", + "DisplayName":"CacheMisses", + "Legend":"Hardware counter 'CacheMisses' per single operation", + "NumberFormat":"N0", + "UnitType":0, + "Unit":null, + "TheGreaterTheBetter":false, + "PriorityInCategory":0 + } } ] } @@ -396,24 +496,24 @@ JsonExporter-full ############################################ JsonExporter-full-compressed ############################################ -{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":null,"ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1}]},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1}]}]} +{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":"1.0.x.mock","ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":2,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":3,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":4,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":5,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":6,"Operations":1,"Nanoseconds":1}],"Metrics":[{"Value":7,"Descriptor":{"Id":"CacheMisses","DisplayName":"CacheMisses","Legend":"Hardware counter 'CacheMisses' per single operation","NumberFormat":"N0","UnitType":0,"Unit":null,"TheGreaterTheBetter":false,"PriorityInCategory":0}}]},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":2,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":3,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":4,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":5,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":6,"Operations":1,"Nanoseconds":1}],"Metrics":[{"Value":7,"Descriptor":{"Id":"CacheMisses","DisplayName":"CacheMisses","Legend":"Hardware counter 'CacheMisses' per single operation","NumberFormat":"N0","UnitType":0,"Unit":null,"TheGreaterTheBetter":false,"PriorityInCategory":0}}]}]} ############################################ MarkdownExporter-default ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 - Method | Mean | Error | StdDev | P67 | -------- |---------:|------:|---------:|---------:| - Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | - Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | + Method | Mean | Error | StdDev | P67 | CacheMisses | +------- |---------:|---------:|---------:|---------:|------------:| + Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | + Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-atlassian ############################################ @@ -421,34 +521,34 @@ MarkdownExporter-atlassian BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 {noformat} -||Method ||Mean ||Error ||StdDev ||P67 || -| Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | -| Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | +||Method ||Mean ||Error ||StdDev ||P67 ||CacheMisses || +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-console ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 -| Method | Mean | Error | StdDev | P67 | -|------- |---------:|------:|---------:|---------:| -| Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | -| Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | +| Method | Mean | Error | StdDev | P67 | CacheMisses | +|------- |---------:|---------:|---------:|---------:|------------:| +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-github ############################################ @@ -456,63 +556,73 @@ MarkdownExporter-github BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 ``` -| Method | Mean | Error | StdDev | P67 | -|------- |---------:|------:|---------:|---------:| -| Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | -| Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | +| Method | Mean | Error | StdDev | P67 | CacheMisses | +|------- |---------:|---------:|---------:|---------:|------------:| +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-stackoverflow ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores - Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line + Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC + .NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 - Method | Mean | Error | StdDev | P67 | - ------- |---------:|------:|---------:|---------:| - Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | - Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | + Method | Mean | Error | StdDev | P67 | CacheMisses | + ------- |---------:|---------:|---------:|---------:|------------:| + Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | + Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ PlainExporter ############################################ *** MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15) *** * Raw * WorkloadResult 1: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 2: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 3: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 4: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 5: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 6: 1 op, 1.00 ns, 1.0000 ns/op * Statistics for WorkloadResult -Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 1, StdDev = 0.000 ns +Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 6, StdDev = 0.000 ns Min = 1.000 ns, Q1 = 1.000 ns, Median = 1.000 ns, Q3 = 1.000 ns, Max = 1.000 ns IQR = 0.000 ns, LowerFence = 1.000 ns, UpperFence = 1.000 ns -ConfidenceInterval = [NaN ns; NaN ns] (CI 99.9%), Margin = NaN ns (NaN% of Mean) +ConfidenceInterval = [1.000 ns; 1.000 ns] (CI 99.9%), Margin = 0.000 ns (0.00% of Mean) Skewness = NaN, Kurtosis = NaN, MValue = 2 -------------------- Histogram -------------------- -[0.500 ns ; 1.500 ns) | @ +[0.500 ns ; 1.500 ns) | @@@@@@ --------------------------------------------------- *** MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15) *** * Raw * WorkloadResult 1: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 2: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 3: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 4: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 5: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 6: 1 op, 1.00 ns, 1.0000 ns/op * Statistics for WorkloadResult -Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 1, StdDev = 0.000 ns +Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 6, StdDev = 0.000 ns Min = 1.000 ns, Q1 = 1.000 ns, Median = 1.000 ns, Q3 = 1.000 ns, Max = 1.000 ns IQR = 0.000 ns, LowerFence = 1.000 ns, UpperFence = 1.000 ns -ConfidenceInterval = [NaN ns; NaN ns] (CI 99.9%), Margin = NaN ns (NaN% of Mean) +ConfidenceInterval = [1.000 ns; 1.000 ns] (CI 99.9%), Margin = 0.000 ns (0.00% of Mean) Skewness = NaN, Kurtosis = NaN, MValue = 2 -------------------- Histogram -------------------- -[0.500 ns ; 1.500 ns) | @ +[0.500 ns ; 1.500 ns) | @@@@@@ --------------------------------------------------- ############################################ XmlExporter-brief @@ -533,6 +643,7 @@ XmlExporter-brief False True CONFIGURATION + 1.0.x.mock 2531248 @@ -548,8 +659,13 @@ XmlExporter-brief 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -565,13 +681,13 @@ XmlExporter-brief NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -585,6 +701,20 @@ XmlExporter-brief 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -602,8 +732,13 @@ XmlExporter-brief 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -619,13 +754,13 @@ XmlExporter-brief NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -639,6 +774,20 @@ XmlExporter-brief 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -652,7 +801,7 @@ XmlExporter-brief ############################################ XmlExporter-brief-compressed ############################################ -Codestin Search AppBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo11111111110000NaNNaN110L999NaNNaNNaN1111111110000MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar11111111110000NaNNaN110L999NaNNaNNaN1111111110000 +Codestin Search AppBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION1.0.x.mock2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000 ############################################ XmlExporter-full ############################################ @@ -672,6 +821,7 @@ XmlExporter-full False True CONFIGURATION + 1.0.x.mock 2531248 @@ -687,8 +837,13 @@ XmlExporter-full 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -704,13 +859,13 @@ XmlExporter-full NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -724,6 +879,20 @@ XmlExporter-full 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -740,6 +909,46 @@ XmlExporter-full 1 1 + + Workload + Result + 1 + 2 + 1 + 1 + + + Workload + Result + 1 + 3 + 1 + 1 + + + Workload + Result + 1 + 4 + 1 + 1 + + + Workload + Result + 1 + 5 + 1 + 1 + + + Workload + Result + 1 + 6 + 1 + 1 + @@ -751,8 +960,13 @@ XmlExporter-full 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -768,13 +982,13 @@ XmlExporter-full NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -788,6 +1002,20 @@ XmlExporter-full 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -804,6 +1032,46 @@ XmlExporter-full 1 1 + + Workload + Result + 1 + 2 + 1 + 1 + + + Workload + Result + 1 + 3 + 1 + 1 + + + Workload + Result + 1 + 4 + 1 + 1 + + + Workload + Result + 1 + 5 + 1 + 1 + + + Workload + Result + 1 + 6 + 1 + 1 + @@ -811,4 +1079,4 @@ XmlExporter-full ############################################ XmlExporter-full-compressed ############################################ -Codestin Search AppBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo11111111110000NaNNaN110L999NaNNaNNaN1111111110000WorkloadResult1111MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar11111111110000NaNNaN110L999NaNNaNNaN1111111110000WorkloadResult1111 +Codestin Search AppBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION1.0.x.mock2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000WorkloadResult1111WorkloadResult1211WorkloadResult1311WorkloadResult1411WorkloadResult1511WorkloadResult1611MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000WorkloadResult1111WorkloadResult1211WorkloadResult1311WorkloadResult1411WorkloadResult1511WorkloadResult1611 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt index cd3482c04f..b406e20dd1 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt @@ -4,18 +4,18 @@ AsciiDocExporter .... BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 .... [options="header"] |=== -|Method |Mean |Error |StdDev |P67 -|Foo | 1.000 ns| NA| 0.000 ns| 1.000 ns -|Bar | 1.000 ns| NA| 0.000 ns| 1.000 ns +|Method |Mean |Error |StdDev |P67 |CacheMisses +|Foo | 1.000 ns| 0.000 ns| 0.000 ns| 1.000 ns| 7 +|Bar | 1.000 ns| 0.000 ns| 0.000 ns| 1.000 ns| 7 |=== ############################################ HtmlExporter @@ -37,19 +37,19 @@ HtmlExporter 

 BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V)
 MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
-Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC
-  [Host]  : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
-  LongRun : extra output line
+Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC
+.NET Core SDK 1.0.x.mock
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
 
Job=LongRun  IterationCount=100  LaunchCount=3  
 WarmupCount=15
- + - - + +
MethodMeanErrorStdDevP67
MethodMeanErrorStdDevP67CacheMisses
Foo1.000 nsNA0.000 ns1.000 ns
Bar1.000 nsNA0.000 ns1.000 ns
Foo1.000 ns0.000 ns0.000 ns1.000 ns7
Bar1.000 ns0.000 ns0.000 ns1.000 ns7
@@ -71,7 +71,7 @@ JsonExporter-brief "HasAttachedDebugger":false, "HasRyuJit":true, "Configuration":"CONFIGURATION", - "DotNetCliVersion":null, + "DotNetCliVersion":"1.0.x.mock", "ChronometerFrequency":{ "Hertz":2531248 }, @@ -89,9 +89,9 @@ JsonExporter-brief "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -116,13 +116,13 @@ JsonExporter-brief "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -154,9 +154,9 @@ JsonExporter-brief "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -181,13 +181,13 @@ JsonExporter-brief "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -214,7 +214,7 @@ JsonExporter-brief ############################################ JsonExporter-brief-compressed ############################################ -{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":null,"ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}}]} +{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":"1.0.x.mock","ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}}]} ############################################ JsonExporter-full ############################################ @@ -233,7 +233,7 @@ JsonExporter-full "HasAttachedDebugger":false, "HasRyuJit":true, "Configuration":"CONFIGURATION", - "DotNetCliVersion":null, + "DotNetCliVersion":"1.0.x.mock", "ChronometerFrequency":{ "Hertz":2531248 }, @@ -251,9 +251,9 @@ JsonExporter-full "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -278,13 +278,13 @@ JsonExporter-full "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -313,6 +313,56 @@ JsonExporter-full "IterationIndex":1, "Operations":1, "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":2, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":3, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":4, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":5, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":6, + "Operations":1, + "Nanoseconds":1 + } + ], + "Metrics":[ + { + "Value":7, + "Descriptor":{ + "Id":"CacheMisses", + "DisplayName":"CacheMisses", + "Legend":"Hardware counter 'CacheMisses' per single operation", + "NumberFormat":"N0", + "UnitType":0, + "Unit":null, + "TheGreaterTheBetter":false, + "PriorityInCategory":0 + } } ] },{ @@ -326,9 +376,9 @@ JsonExporter-full "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -353,13 +403,13 @@ JsonExporter-full "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -388,6 +438,56 @@ JsonExporter-full "IterationIndex":1, "Operations":1, "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":2, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":3, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":4, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":5, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":6, + "Operations":1, + "Nanoseconds":1 + } + ], + "Metrics":[ + { + "Value":7, + "Descriptor":{ + "Id":"CacheMisses", + "DisplayName":"CacheMisses", + "Legend":"Hardware counter 'CacheMisses' per single operation", + "NumberFormat":"N0", + "UnitType":0, + "Unit":null, + "TheGreaterTheBetter":false, + "PriorityInCategory":0 + } } ] } @@ -396,24 +496,24 @@ JsonExporter-full ############################################ JsonExporter-full-compressed ############################################ -{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":null,"ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1}]},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1}]}]} +{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":"1.0.x.mock","ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":2,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":3,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":4,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":5,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":6,"Operations":1,"Nanoseconds":1}],"Metrics":[{"Value":7,"Descriptor":{"Id":"CacheMisses","DisplayName":"CacheMisses","Legend":"Hardware counter 'CacheMisses' per single operation","NumberFormat":"N0","UnitType":0,"Unit":null,"TheGreaterTheBetter":false,"PriorityInCategory":0}}]},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":2,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":3,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":4,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":5,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":6,"Operations":1,"Nanoseconds":1}],"Metrics":[{"Value":7,"Descriptor":{"Id":"CacheMisses","DisplayName":"CacheMisses","Legend":"Hardware counter 'CacheMisses' per single operation","NumberFormat":"N0","UnitType":0,"Unit":null,"TheGreaterTheBetter":false,"PriorityInCategory":0}}]}]} ############################################ MarkdownExporter-default ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 - Method | Mean | Error | StdDev | P67 | -------- |---------:|------:|---------:|---------:| - Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | - Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | + Method | Mean | Error | StdDev | P67 | CacheMisses | +------- |---------:|---------:|---------:|---------:|------------:| + Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | + Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-atlassian ############################################ @@ -421,34 +521,34 @@ MarkdownExporter-atlassian BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 {noformat} -||Method ||Mean ||Error ||StdDev ||P67 || -| Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | -| Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | +||Method ||Mean ||Error ||StdDev ||P67 ||CacheMisses || +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-console ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 -| Method | Mean | Error | StdDev | P67 | -|------- |---------:|------:|---------:|---------:| -| Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | -| Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | +| Method | Mean | Error | StdDev | P67 | CacheMisses | +|------- |---------:|---------:|---------:|---------:|------------:| +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-github ############################################ @@ -456,63 +556,73 @@ MarkdownExporter-github BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 ``` -| Method | Mean | Error | StdDev | P67 | -|------- |---------:|------:|---------:|---------:| -| Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | -| Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | +| Method | Mean | Error | StdDev | P67 | CacheMisses | +|------- |---------:|---------:|---------:|---------:|------------:| +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-stackoverflow ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores - Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line + Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC + .NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 - Method | Mean | Error | StdDev | P67 | - ------- |---------:|------:|---------:|---------:| - Foo | 1.000 ns | NA | 0.000 ns | 1.000 ns | - Bar | 1.000 ns | NA | 0.000 ns | 1.000 ns | + Method | Mean | Error | StdDev | P67 | CacheMisses | + ------- |---------:|---------:|---------:|---------:|------------:| + Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | + Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ PlainExporter ############################################ *** MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15) *** * Raw * WorkloadResult 1: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 2: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 3: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 4: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 5: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 6: 1 op, 1.00 ns, 1.0000 ns/op * Statistics for WorkloadResult -Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 1, StdDev = 0.000 ns +Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 6, StdDev = 0.000 ns Min = 1.000 ns, Q1 = 1.000 ns, Median = 1.000 ns, Q3 = 1.000 ns, Max = 1.000 ns IQR = 0.000 ns, LowerFence = 1.000 ns, UpperFence = 1.000 ns -ConfidenceInterval = [NaN ns; NaN ns] (CI 99.9%), Margin = NaN ns (NaN% of Mean) +ConfidenceInterval = [1.000 ns; 1.000 ns] (CI 99.9%), Margin = 0.000 ns (0.00% of Mean) Skewness = NaN, Kurtosis = NaN, MValue = 2 -------------------- Histogram -------------------- -[0.500 ns ; 1.500 ns) | @ +[0.500 ns ; 1.500 ns) | @@@@@@ --------------------------------------------------- *** MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15) *** * Raw * WorkloadResult 1: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 2: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 3: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 4: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 5: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 6: 1 op, 1.00 ns, 1.0000 ns/op * Statistics for WorkloadResult -Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 1, StdDev = 0.000 ns +Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 6, StdDev = 0.000 ns Min = 1.000 ns, Q1 = 1.000 ns, Median = 1.000 ns, Q3 = 1.000 ns, Max = 1.000 ns IQR = 0.000 ns, LowerFence = 1.000 ns, UpperFence = 1.000 ns -ConfidenceInterval = [NaN ns; NaN ns] (CI 99.9%), Margin = NaN ns (NaN% of Mean) +ConfidenceInterval = [1.000 ns; 1.000 ns] (CI 99.9%), Margin = 0.000 ns (0.00% of Mean) Skewness = NaN, Kurtosis = NaN, MValue = 2 -------------------- Histogram -------------------- -[0.500 ns ; 1.500 ns) | @ +[0.500 ns ; 1.500 ns) | @@@@@@ --------------------------------------------------- ############################################ XmlExporter-brief @@ -533,6 +643,7 @@ XmlExporter-brief False True CONFIGURATION + 1.0.x.mock 2531248 @@ -548,8 +659,13 @@ XmlExporter-brief 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -565,13 +681,13 @@ XmlExporter-brief NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -585,6 +701,20 @@ XmlExporter-brief 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -602,8 +732,13 @@ XmlExporter-brief 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -619,13 +754,13 @@ XmlExporter-brief NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -639,6 +774,20 @@ XmlExporter-brief 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -652,7 +801,7 @@ XmlExporter-brief ############################################ XmlExporter-brief-compressed ############################################ -Codestin Search AppBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo11111111110000NaNNaN110L999NaNNaNNaN1111111110000MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar11111111110000NaNNaN110L999NaNNaNNaN1111111110000 +Codestin Search AppBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION1.0.x.mock2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000 ############################################ XmlExporter-full ############################################ @@ -672,6 +821,7 @@ XmlExporter-full False True CONFIGURATION + 1.0.x.mock 2531248 @@ -687,8 +837,13 @@ XmlExporter-full 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -704,13 +859,13 @@ XmlExporter-full NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -724,6 +879,20 @@ XmlExporter-full 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -740,6 +909,46 @@ XmlExporter-full 1 1 + + Workload + Result + 1 + 2 + 1 + 1 + + + Workload + Result + 1 + 3 + 1 + 1 + + + Workload + Result + 1 + 4 + 1 + 1 + + + Workload + Result + 1 + 5 + 1 + 1 + + + Workload + Result + 1 + 6 + 1 + 1 + @@ -751,8 +960,13 @@ XmlExporter-full 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -768,13 +982,13 @@ XmlExporter-full NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -788,6 +1002,20 @@ XmlExporter-full 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -804,6 +1032,46 @@ XmlExporter-full 1 1 + + Workload + Result + 1 + 2 + 1 + 1 + + + Workload + Result + 1 + 3 + 1 + 1 + + + Workload + Result + 1 + 4 + 1 + 1 + + + Workload + Result + 1 + 5 + 1 + 1 + + + Workload + Result + 1 + 6 + 1 + 1 + @@ -811,4 +1079,4 @@ XmlExporter-full ############################################ XmlExporter-full-compressed ############################################ -Codestin Search AppBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo11111111110000NaNNaN110L999NaNNaNNaN1111111110000WorkloadResult1111MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar11111111110000NaNNaN110L999NaNNaNNaN1111111110000WorkloadResult1111 +Codestin Search AppBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION1.0.x.mock2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000WorkloadResult1111WorkloadResult1211WorkloadResult1311WorkloadResult1411WorkloadResult1511WorkloadResult1611MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000WorkloadResult1111WorkloadResult1211WorkloadResult1311WorkloadResult1411WorkloadResult1511WorkloadResult1611 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_ru-RU.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_ru-RU.verified.txt index bbfaff1da8..b406e20dd1 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_ru-RU.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_ru-RU.verified.txt @@ -4,18 +4,18 @@ AsciiDocExporter .... BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 .... [options="header"] |=== -|Method |Mean |Error |StdDev |P67 -|Foo | 1,000 ns| NA| 0,000 ns| 1,000 ns -|Bar | 1,000 ns| NA| 0,000 ns| 1,000 ns +|Method |Mean |Error |StdDev |P67 |CacheMisses +|Foo | 1.000 ns| 0.000 ns| 0.000 ns| 1.000 ns| 7 +|Bar | 1.000 ns| 0.000 ns| 0.000 ns| 1.000 ns| 7 |=== ############################################ HtmlExporter @@ -37,19 +37,19 @@ HtmlExporter 

 BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V)
 MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores
-Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC
-  [Host]  : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
-  LongRun : extra output line
+Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC
+.NET Core SDK 1.0.x.mock
+  [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION
 
Job=LongRun  IterationCount=100  LaunchCount=3  
 WarmupCount=15
- + - - + +
MethodMeanErrorStdDevP67
MethodMeanErrorStdDevP67CacheMisses
Foo1,000 nsNA0,000 ns1,000 ns
Bar1,000 nsNA0,000 ns1,000 ns
Foo1.000 ns0.000 ns0.000 ns1.000 ns7
Bar1.000 ns0.000 ns0.000 ns1.000 ns7
@@ -71,7 +71,7 @@ JsonExporter-brief "HasAttachedDebugger":false, "HasRyuJit":true, "Configuration":"CONFIGURATION", - "DotNetCliVersion":null, + "DotNetCliVersion":"1.0.x.mock", "ChronometerFrequency":{ "Hertz":2531248 }, @@ -89,9 +89,9 @@ JsonExporter-brief "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -116,13 +116,13 @@ JsonExporter-brief "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -154,9 +154,9 @@ JsonExporter-brief "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -181,13 +181,13 @@ JsonExporter-brief "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -214,7 +214,7 @@ JsonExporter-brief ############################################ JsonExporter-brief-compressed ############################################ -{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":null,"ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}}]} +{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":"1.0.x.mock","ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null}}]} ############################################ JsonExporter-full ############################################ @@ -233,7 +233,7 @@ JsonExporter-full "HasAttachedDebugger":false, "HasRyuJit":true, "Configuration":"CONFIGURATION", - "DotNetCliVersion":null, + "DotNetCliVersion":"1.0.x.mock", "ChronometerFrequency":{ "Hertz":2531248 }, @@ -251,9 +251,9 @@ JsonExporter-full "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -278,13 +278,13 @@ JsonExporter-full "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -313,6 +313,56 @@ JsonExporter-full "IterationIndex":1, "Operations":1, "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":2, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":3, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":4, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":5, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":6, + "Operations":1, + "Nanoseconds":1 + } + ], + "Metrics":[ + { + "Value":7, + "Descriptor":{ + "Id":"CacheMisses", + "DisplayName":"CacheMisses", + "Legend":"Hardware counter 'CacheMisses' per single operation", + "NumberFormat":"N0", + "UnitType":0, + "Unit":null, + "TheGreaterTheBetter":false, + "PriorityInCategory":0 + } } ] },{ @@ -326,9 +376,9 @@ JsonExporter-full "HardwareIntrinsics":"", "Statistics":{ "OriginalValues":[ - 1 + 1,1,1,1,1,1 ], - "N":1, + "N":6, "Min":1, "LowerFence":1, "Q1":1, @@ -353,13 +403,13 @@ JsonExporter-full "Skewness":"", "Kurtosis":"", "ConfidenceInterval":{ - "N":1, + "N":6, "Mean":1, "StandardError":0, "Level":12, - "Margin":"", - "Lower":"", - "Upper":"" + "Margin":0, + "Lower":1, + "Upper":1 }, "Percentiles":{ "P0":1, @@ -388,6 +438,56 @@ JsonExporter-full "IterationIndex":1, "Operations":1, "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":2, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":3, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":4, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":5, + "Operations":1, + "Nanoseconds":1 + },{ + "IterationMode":"Workload", + "IterationStage":"Result", + "LaunchIndex":1, + "IterationIndex":6, + "Operations":1, + "Nanoseconds":1 + } + ], + "Metrics":[ + { + "Value":7, + "Descriptor":{ + "Id":"CacheMisses", + "DisplayName":"CacheMisses", + "Legend":"Hardware counter 'CacheMisses' per single operation", + "NumberFormat":"N0", + "UnitType":0, + "Unit":null, + "TheGreaterTheBetter":false, + "PriorityInCategory":0 + } } ] } @@ -396,24 +496,24 @@ JsonExporter-full ############################################ JsonExporter-full-compressed ############################################ -{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":null,"ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1}]},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1],"N":1,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":1,"Mean":1,"StandardError":0,"Level":12,"Margin":"","Lower":"","Upper":""},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1}]}]} +{"Title":"MockSummary","HostEnvironmentInfo":{"BenchmarkDotNetCaption":"BenchmarkDotNet","BenchmarkDotNetVersion":"0.10.x-mock","OsVersion":"Microsoft Windows NT 10.0.x.mock","ProcessorName":"MockIntel Core i7-6700HQ CPU 2.60GHz","PhysicalProcessorCount":1,"PhysicalCoreCount":4,"LogicalCoreCount":8,"RuntimeVersion":"Clr 4.0.x.mock","Architecture":"64mock","HasAttachedDebugger":false,"HasRyuJit":true,"Configuration":"CONFIGURATION","DotNetCliVersion":"1.0.x.mock","ChronometerFrequency":{"Hertz":2531248},"HardwareTimerKind":"Tsc"},"Benchmarks":[{"DisplayInfo":"MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Foo","MethodTitle":"Foo","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Foo","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":2,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":3,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":4,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":5,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":6,"Operations":1,"Nanoseconds":1}],"Metrics":[{"Value":7,"Descriptor":{"Id":"CacheMisses","DisplayName":"CacheMisses","Legend":"Hardware counter 'CacheMisses' per single operation","NumberFormat":"N0","UnitType":0,"Unit":null,"TheGreaterTheBetter":false,"PriorityInCategory":0}}]},{"DisplayInfo":"MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)","Namespace":"BenchmarkDotNet.Tests.Mocks","Type":"MockBenchmarkClass","Method":"Bar","MethodTitle":"Bar","Parameters":"","FullName":"BenchmarkDotNet.Tests.Mocks.MockFactory+MockBenchmarkClass.Bar","HardwareIntrinsics":"","Statistics":{"OriginalValues":[1,1,1,1,1,1],"N":6,"Min":1,"LowerFence":1,"Q1":1,"Median":1,"Mean":1,"Q3":1,"UpperFence":1,"Max":1,"InterquartileRange":0,"LowerOutliers":[],"UpperOutliers":[],"AllOutliers":[],"StandardError":0,"Variance":0,"StandardDeviation":0,"Skewness":"","Kurtosis":"","ConfidenceInterval":{"N":6,"Mean":1,"StandardError":0,"Level":12,"Margin":0,"Lower":1,"Upper":1},"Percentiles":{"P0":1,"P25":1,"P50":1,"P67":1,"P80":1,"P85":1,"P90":1,"P95":1,"P100":1}},"Memory":{"Gen0Collections":0,"Gen1Collections":0,"Gen2Collections":0,"TotalOperations":0,"BytesAllocatedPerOperation":null},"Measurements":[{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":1,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":2,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":3,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":4,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":5,"Operations":1,"Nanoseconds":1},{"IterationMode":"Workload","IterationStage":"Result","LaunchIndex":1,"IterationIndex":6,"Operations":1,"Nanoseconds":1}],"Metrics":[{"Value":7,"Descriptor":{"Id":"CacheMisses","DisplayName":"CacheMisses","Legend":"Hardware counter 'CacheMisses' per single operation","NumberFormat":"N0","UnitType":0,"Unit":null,"TheGreaterTheBetter":false,"PriorityInCategory":0}}]}]} ############################################ MarkdownExporter-default ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 - Method | Mean | Error | StdDev | P67 | -------- |---------:|------:|---------:|---------:| - Foo | 1,000 ns | NA | 0,000 ns | 1,000 ns | - Bar | 1,000 ns | NA | 0,000 ns | 1,000 ns | + Method | Mean | Error | StdDev | P67 | CacheMisses | +------- |---------:|---------:|---------:|---------:|------------:| + Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | + Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-atlassian ############################################ @@ -421,34 +521,34 @@ MarkdownExporter-atlassian BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 {noformat} -||Method ||Mean ||Error ||StdDev ||P67 || -| Foo | 1,000 ns | NA | 0,000 ns | 1,000 ns | -| Bar | 1,000 ns | NA | 0,000 ns | 1,000 ns | +||Method ||Mean ||Error ||StdDev ||P67 ||CacheMisses || +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-console ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 -| Method | Mean | Error | StdDev | P67 | -|------- |---------:|------:|---------:|---------:| -| Foo | 1,000 ns | NA | 0,000 ns | 1,000 ns | -| Bar | 1,000 ns | NA | 0,000 ns | 1,000 ns | +| Method | Mean | Error | StdDev | P67 | CacheMisses | +|------- |---------:|---------:|---------:|---------:|------------:| +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-github ############################################ @@ -456,63 +556,73 @@ MarkdownExporter-github BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC +.NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 ``` -| Method | Mean | Error | StdDev | P67 | -|------- |---------:|------:|---------:|---------:| -| Foo | 1,000 ns | NA | 0,000 ns | 1,000 ns | -| Bar | 1,000 ns | NA | 0,000 ns | 1,000 ns | +| Method | Mean | Error | StdDev | P67 | CacheMisses | +|------- |---------:|---------:|---------:|---------:|------------:| +| Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | +| Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ MarkdownExporter-stackoverflow ############################################ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores - Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC - [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION - LongRun : extra output line + Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC + .NET Core SDK 1.0.x.mock + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job=LongRun IterationCount=100 LaunchCount=3 WarmupCount=15 - Method | Mean | Error | StdDev | P67 | - ------- |---------:|------:|---------:|---------:| - Foo | 1,000 ns | NA | 0,000 ns | 1,000 ns | - Bar | 1,000 ns | NA | 0,000 ns | 1,000 ns | + Method | Mean | Error | StdDev | P67 | CacheMisses | + ------- |---------:|---------:|---------:|---------:|------------:| + Foo | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | + Bar | 1.000 ns | 0.000 ns | 0.000 ns | 1.000 ns | 7 | ############################################ PlainExporter ############################################ *** MockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15) *** * Raw * WorkloadResult 1: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 2: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 3: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 4: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 5: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 6: 1 op, 1.00 ns, 1.0000 ns/op * Statistics for WorkloadResult -Mean = 1,000 ns; StdErr = 0,000 ns (0,00%); N = 1; StdDev = 0,000 ns -Min = 1,000 ns; Q1 = 1,000 ns; Median = 1,000 ns; Q3 = 1,000 ns; Max = 1,000 ns -IQR = 0,000 ns; LowerFence = 1,000 ns; UpperFence = 1,000 ns -ConfidenceInterval = [не число ns; не число ns] (CI 99.9%); Margin = не число ns (не число% of Mean) -Skewness = не число; Kurtosis = не число; MValue = 2 +Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 6, StdDev = 0.000 ns +Min = 1.000 ns, Q1 = 1.000 ns, Median = 1.000 ns, Q3 = 1.000 ns, Max = 1.000 ns +IQR = 0.000 ns, LowerFence = 1.000 ns, UpperFence = 1.000 ns +ConfidenceInterval = [1.000 ns; 1.000 ns] (CI 99.9%), Margin = 0.000 ns (0.00% of Mean) +Skewness = NaN, Kurtosis = NaN, MValue = 2 -------------------- Histogram -------------------- -[0,500 ns ; 1,500 ns) | @ +[0.500 ns ; 1.500 ns) | @@@@@@ --------------------------------------------------- *** MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15) *** * Raw * WorkloadResult 1: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 2: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 3: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 4: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 5: 1 op, 1.00 ns, 1.0000 ns/op +WorkloadResult 6: 1 op, 1.00 ns, 1.0000 ns/op * Statistics for WorkloadResult -Mean = 1,000 ns; StdErr = 0,000 ns (0,00%); N = 1; StdDev = 0,000 ns -Min = 1,000 ns; Q1 = 1,000 ns; Median = 1,000 ns; Q3 = 1,000 ns; Max = 1,000 ns -IQR = 0,000 ns; LowerFence = 1,000 ns; UpperFence = 1,000 ns -ConfidenceInterval = [не число ns; не число ns] (CI 99.9%); Margin = не число ns (не число% of Mean) -Skewness = не число; Kurtosis = не число; MValue = 2 +Mean = 1.000 ns, StdErr = 0.000 ns (0.00%), N = 6, StdDev = 0.000 ns +Min = 1.000 ns, Q1 = 1.000 ns, Median = 1.000 ns, Q3 = 1.000 ns, Max = 1.000 ns +IQR = 0.000 ns, LowerFence = 1.000 ns, UpperFence = 1.000 ns +ConfidenceInterval = [1.000 ns; 1.000 ns] (CI 99.9%), Margin = 0.000 ns (0.00% of Mean) +Skewness = NaN, Kurtosis = NaN, MValue = 2 -------------------- Histogram -------------------- -[0,500 ns ; 1,500 ns) | @ +[0.500 ns ; 1.500 ns) | @@@@@@ --------------------------------------------------- ############################################ XmlExporter-brief @@ -533,6 +643,7 @@ XmlExporter-brief False True CONFIGURATION + 1.0.x.mock 2531248 @@ -548,8 +659,13 @@ XmlExporter-brief 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -565,13 +681,13 @@ XmlExporter-brief NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -585,6 +701,20 @@ XmlExporter-brief 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -602,8 +732,13 @@ XmlExporter-brief 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -619,13 +754,13 @@ XmlExporter-brief NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -639,6 +774,20 @@ XmlExporter-brief 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -652,7 +801,7 @@ XmlExporter-brief ############################################ XmlExporter-brief-compressed ############################################ -Codestin Search AppBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo11111111110000NaNNaN110L999NaNNaNNaN1111111110000MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar11111111110000NaNNaN110L999NaNNaNNaN1111111110000 +Codestin Search AppBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION1.0.x.mock2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000 ############################################ XmlExporter-full ############################################ @@ -672,6 +821,7 @@ XmlExporter-full False True CONFIGURATION + 1.0.x.mock 2531248 @@ -687,8 +837,13 @@ XmlExporter-full 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -704,13 +859,13 @@ XmlExporter-full NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -724,6 +879,20 @@ XmlExporter-full 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -740,6 +909,46 @@ XmlExporter-full 1 1 + + Workload + Result + 1 + 2 + 1 + 1 + + + Workload + Result + 1 + 3 + 1 + 1 + + + Workload + Result + 1 + 4 + 1 + 1 + + + Workload + Result + 1 + 5 + 1 + 1 + + + Workload + Result + 1 + 6 + 1 + 1 + @@ -751,8 +960,13 @@ XmlExporter-full 1 + 1 + 1 + 1 + 1 + 1 - 1 + 6 1 1 1 @@ -768,13 +982,13 @@ XmlExporter-full NaN NaN - 1 + 6 1 0 L999 - NaN - NaN - NaN + 0 + 1 + 1 1 @@ -788,6 +1002,20 @@ XmlExporter-full 1 + + + 7 + + CacheMisses + CacheMisses + Hardware counter 'CacheMisses' per single operation + N0 + Dimensionless + False + 0 + + + 0 0 @@ -804,6 +1032,46 @@ XmlExporter-full 1 1 + + Workload + Result + 1 + 2 + 1 + 1 + + + Workload + Result + 1 + 3 + 1 + 1 + + + Workload + Result + 1 + 4 + 1 + 1 + + + Workload + Result + 1 + 5 + 1 + 1 + + + Workload + Result + 1 + 6 + 1 + 1 + @@ -811,4 +1079,4 @@ XmlExporter-full ############################################ XmlExporter-full-compressed ############################################ -Codestin Search AppBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo11111111110000NaNNaN110L999NaNNaNNaN1111111110000WorkloadResult1111MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar11111111110000NaNNaN110L999NaNNaNNaN1111111110000WorkloadResult1111 +Codestin Search AppBenchmarkDotNet0.10.x-mockMicrosoft Windows NT 10.0.x.mockMockIntel Core i7-6700HQ CPU 2.60GHz148Clr 4.0.x.mock64mockFalseTrueCONFIGURATION1.0.x.mock2531248TscMockBenchmarkClass.Foo: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassFooFoo1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000WorkloadResult1111WorkloadResult1211WorkloadResult1311WorkloadResult1411WorkloadResult1511WorkloadResult1611MockBenchmarkClass.Bar: LongRun(IterationCount=100, LaunchCount=3, WarmupCount=15)BenchmarkDotNet.Tests.MocksMockBenchmarkClassBarBar1111116111111110000NaNNaN610L9990111111111117CacheMissesCacheMissesHardware counter 'CacheMisses' per single operationN0DimensionlessFalse00000WorkloadResult1111WorkloadResult1211WorkloadResult1311WorkloadResult1411WorkloadResult1511WorkloadResult1611 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Escape_ParamsAndArguments.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Escape_ParamsAndArguments.verified.txt index 78bde45ccb..715b73166b 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Escape_ParamsAndArguments.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Escape_ParamsAndArguments.verified.txt @@ -2,18 +2,18 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | StringParam | charArg | Mean | Error | StdDev | ------- |------------ |-------- |---------:|--------:|--------:| - Foo | \t | \t | 102.0 ns | 6.09 ns | 1.58 ns | ^ - Foo | \t | \n | 202.0 ns | 6.09 ns | 1.58 ns | ^ - Bar | \t | ? | 302.0 ns | 6.09 ns | 1.58 ns | ^ - Foo | \n | \t | 402.0 ns | 6.09 ns | 1.58 ns | ^ - Foo | \n | \n | 502.0 ns | 6.09 ns | 1.58 ns | ^ - Bar | \n | ? | 602.0 ns | 6.09 ns | 1.58 ns | ^ + Foo | \t | \t | 114.5 ns | 5.88 ns | 8.80 ns | ^ + Foo | \t | \n | 214.5 ns | 5.88 ns | 8.80 ns | ^ + Bar | \t | ? | 314.5 ns | 5.88 ns | 8.80 ns | ^ + Foo | \n | \t | 414.5 ns | 5.88 ns | 8.80 ns | ^ + Foo | \n | \n | 514.5 ns | 5.88 ns | 8.80 ns | ^ + Bar | \n | ? | 614.5 ns | 5.88 ns | 8.80 ns | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_HideColumns_TableMarkDown.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_HideColumns_TableMarkDown.verified.txt new file mode 100644 index 0000000000..5b932e42da --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_HideColumns_TableMarkDown.verified.txt @@ -0,0 +1,15 @@ +=== HideColumns_TableMarkDown === + +BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC + [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION + DefaultJob : extra output line + +StdDev=8.80 ns + + Method | Mean | Error | +------- |---------:|--------:| + Foo | 114.5 ns | 5.88 ns | + +Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt index b690775469..314e1f16bd 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,11 +10,11 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |---------:|--------:|--------:|------:|--------:|-----:|---------------------------- |--------- | - Foo | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Invalid_TwoJobBaselines.Foo | Yes | - Foo | Job2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 2 | Invalid_TwoJobBaselines.Foo | Yes | + Foo | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | Invalid_TwoJobBaselines.Foo | Yes | + Foo | Job2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 2 | Invalid_TwoJobBaselines.Foo | Yes | | | | | | | | | | | - Bar | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Invalid_TwoJobBaselines.Bar | Yes | - Bar | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | Invalid_TwoJobBaselines.Bar | Yes | + Bar | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | Invalid_TwoJobBaselines.Bar | Yes | + Bar | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.94 | 0.09 | 2 | Invalid_TwoJobBaselines.Bar | Yes | Errors: 2 * Only 1 job in a group can have "Baseline = true" applied to it, group Invalid_TwoJobBaselines.Foo in class Invalid_TwoJobBaselines has 2 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt index aaba6592c2..11fbd70a0b 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt @@ -2,15 +2,15 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Foo | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | DefaultJob | Yes | - Bar | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | DefaultJob | Yes | + Foo | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DefaultJob | Yes | + Bar | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DefaultJob | Yes | Errors: 1 * Only 1 benchmark method in a group can have "Baseline = true" applied to it, group DefaultJob in class Invalid_TwoMethodBaselines has 2 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt index e8793a9409..5c8cbaad01 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,13 +10,13 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |---------:|--------:|--------:|------:|--------:|-----:|----------------------------- |--------- | - Base | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsJobs.Base | Yes | - Base | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | JobBaseline_MethodsJobs.Base | No | + Base | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | JobBaseline_MethodsJobs.Base | Yes | + Base | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 2 | JobBaseline_MethodsJobs.Base | No | | | | | | | | | | | - Foo | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsJobs.Foo | Yes | - Foo | Job2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | JobBaseline_MethodsJobs.Foo | No | + Foo | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | JobBaseline_MethodsJobs.Foo | Yes | + Foo | Job2 | 514.5 ns | 5.88 ns | 8.80 ns | 2.40 | 0.11 | 2 | JobBaseline_MethodsJobs.Foo | No | | | | | | | | | | | - Bar | Job1 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsJobs.Bar | Yes | - Bar | Job2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | JobBaseline_MethodsJobs.Bar | No | + Bar | Job1 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | JobBaseline_MethodsJobs.Bar | Yes | + Bar | Job2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.96 | 0.06 | 2 | JobBaseline_MethodsJobs.Bar | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt index 479192b41e..4f5f37b6c7 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,22 +10,22 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------- |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ - Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | No | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ + Base | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | No | | | | | | | | | | | | - Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | Yes | - Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | No | + Foo | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | Yes | + Foo | Job2 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 2.40 | 0.11 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | No | | | | | | | | | | | | - Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | Yes | - Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | No | + Bar | Job1 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | Yes | + Bar | Job2 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.96 | 0.06 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | No | | | | | | | | | | | | - Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ - Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.43 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | No | + Base | Job1 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ + Base | Job2 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.42 | 0.02 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | No | | | | | | | | | | | | - Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | Yes | - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.37 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | No | + Foo | Job1 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | Yes | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.37 | 0.02 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | No | | | | | | | | | | | | - Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | Yes | - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | No | + Bar | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | Yes | + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.33 | 0.02 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt index 2355618257..aa22c9caab 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt @@ -2,15 +2,15 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Base | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | DefaultJob | Yes | - Foo | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | DefaultJob | No | - Bar | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | DefaultJob | No | + Base | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DefaultJob | Yes | + Foo | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DefaultJob | No | + Bar | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt index 1096235ebb..297f782355 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,12 +10,12 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Base | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Job1 | Yes | - Foo | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | Job1 | No | - Bar | Job1 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | Job1 | No | + Base | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | Job1 | Yes | + Foo | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | Job1 | No | + Bar | Job1 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | Job1 | No | | | | | | | | | | | - Base | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Job2 | Yes | - Foo | Job2 | 502.0 ns | 6.09 ns | 1.58 ns | 1.25 | 0.00 | 2 | Job2 | No | - Bar | Job2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.50 | 0.00 | 3 | Job2 | No | + Base | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | Job2 | Yes | + Foo | Job2 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | Job2 | No | + Bar | Job2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt index 2178ec1d94..e30ce8166a 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt @@ -2,19 +2,19 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION DefaultJob : extra output line Method | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |------ |---------:|--------:|--------:|------:|--------:|-----:|---------------------- |--------- | - Base | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-DefaultJob | Yes | ^ - Foo | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | [Param=2]-DefaultJob | No | - Bar | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | [Param=2]-DefaultJob | No | + Base | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2]-DefaultJob | Yes | ^ + Foo | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | [Param=2]-DefaultJob | No | + Bar | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | [Param=2]-DefaultJob | No | | | | | | | | | | | - Base | 10 | 402.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-DefaultJob | Yes | ^ - Foo | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.25 | 0.00 | 2 | [Param=10]-DefaultJob | No | - Bar | 10 | 602.0 ns | 6.09 ns | 1.58 ns | 1.50 | 0.00 | 3 | [Param=10]-DefaultJob | No | + Base | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | [Param=10]-DefaultJob | Yes | ^ + Foo | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | [Param=10]-DefaultJob | No | + Bar | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | [Param=10]-DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt index 879a228733..f85ce9e878 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,20 +10,20 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------- |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-Job1 | Yes | ^ - Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | [Param=2]-Job1 | No | - Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | [Param=2]-Job1 | No | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2]-Job1 | Yes | ^ + Foo | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | [Param=2]-Job1 | No | + Bar | Job1 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | [Param=2]-Job1 | No | | | | | | | | | | | | - Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-Job2 | Yes | - Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 1.25 | 0.00 | 2 | [Param=2]-Job2 | No | - Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.50 | 0.00 | 3 | [Param=2]-Job2 | No | + Base | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | [Param=2]-Job2 | Yes | + Foo | Job2 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | [Param=2]-Job2 | No | + Bar | Job2 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | [Param=2]-Job2 | No | | | | | | | | | | | | - Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-Job1 | Yes | ^ - Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.14 | 0.00 | 2 | [Param=10]-Job1 | No | - Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.28 | 0.00 | 3 | [Param=10]-Job1 | No | + Base | Job1 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10]-Job1 | Yes | ^ + Foo | Job1 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.14 | 0.02 | 2 | [Param=10]-Job1 | No | + Bar | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 1.28 | 0.02 | 3 | [Param=10]-Job1 | No | | | | | | | | | | | | - Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-Job2 | Yes | - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.10 | 0.00 | 2 | [Param=10]-Job2 | No | - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | 3 | [Param=10]-Job2 | No | + Base | Job2 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | [Param=10]-Job2 | Yes | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.10 | 0.01 | 2 | [Param=10]-Job2 | No | + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.20 | 0.01 | 3 | [Param=10]-Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt index 2e9f774250..e71eeca6b6 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,9 +10,9 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Foo | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | * | Yes | - Bar | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | * | No | - Foo | Job2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | * | No | - Bar | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 4 | * | No | + Foo | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | * | Yes | + Bar | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | * | No | + Foo | Job2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | * | No | + Bar | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 4 | * | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt index fac0a1a063..30d92f0b9e 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,14 +10,14 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |------ |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Foo | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2] | Yes | ^ - Bar | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | [Param=2] | No | - Foo | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | [Param=2] | No | - Bar | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 4 | [Param=2] | No | + Foo | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2] | Yes | ^ + Bar | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | [Param=2] | No | + Foo | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | [Param=2] | No | + Bar | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 4 | [Param=2] | No | | | | | | | | | | | | - Foo | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10] | Yes | ^ - Bar | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | 2 | [Param=10] | No | - Foo | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.40 | 0.00 | 3 | [Param=10] | No | - Bar | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.60 | 0.00 | 4 | [Param=10] | No | + Foo | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10] | Yes | ^ + Bar | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.19 | 0.03 | 2 | [Param=10] | No | + Foo | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.39 | 0.03 | 3 | [Param=10] | No | + Bar | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.58 | 0.03 | 4 | [Param=10] | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs.verified.txt index 52a50e2c03..892e8d3538 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,17 +10,17 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|-----:|------------- |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1 | * | No | ^ - Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 2 | * | No | - Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 3 | * | No | - Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 4 | * | No | - Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 5 | * | No | - Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 6 | * | No | - Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 7 | * | No | ^ - Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 8 | * | No | - Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 9 | * | No | - Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 10 | * | No | - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 11 | * | No | - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 12 | * | No | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1 | * | No | ^ + Foo | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 2 | * | No | + Bar | Job1 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 3 | * | No | + Base | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 4 | * | No | + Foo | Job2 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 5 | * | No | + Bar | Job2 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 6 | * | No | + Base | Job1 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 7 | * | No | ^ + Foo | Job1 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 8 | * | No | + Bar | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 9 | * | No | + Base | Job2 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 10 | * | No | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 11 | * | No | + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 12 | * | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt index 355aa0ebd1..a7a7350881 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,36 +10,36 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------------------------- |--------- | - A1 | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=2]-CatA | Yes | ^ + A1 | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=2]-CatA | Yes | ^ | | | | | | | | | | | - A1 | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=10]-CatA | Yes | ^ + A1 | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=10]-CatA | Yes | ^ | | | | | | | | | | | - A1 | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=2]-CatA | Yes | ^ + A1 | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=2]-CatA | Yes | ^ | | | | | | | | | | | - A1 | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=10]-CatA | Yes | ^ + A1 | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=10]-CatA | Yes | ^ | | | | | | | | | | | - A2 | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=2]-CatA | No | ^ + A2 | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=2]-CatA | No | ^ | | | | | | | | | | | - A2 | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=10]-CatA | No | ^ + A2 | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=10]-CatA | No | ^ | | | | | | | | | | | - A2 | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=2]-CatA | No | ^ + A2 | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=2]-CatA | No | ^ | | | | | | | | | | | - A2 | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=10]-CatA | No | ^ + A2 | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=10]-CatA | No | ^ | | | | | | | | | | | - B1 | Job1 | 2 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=2]-CatB | Yes | ^ + B1 | Job1 | 2 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=2]-CatB | Yes | ^ | | | | | | | | | | | - B1 | Job1 | 10 | 1,302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=10]-CatB | Yes | ^ + B1 | Job1 | 10 | 1,314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=10]-CatB | Yes | ^ | | | | | | | | | | | - B1 | Job2 | 2 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=2]-CatB | Yes | ^ + B1 | Job2 | 2 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=2]-CatB | Yes | ^ | | | | | | | | | | | - B1 | Job2 | 10 | 1,502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=10]-CatB | Yes | ^ + B1 | Job2 | 10 | 1,514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=10]-CatB | Yes | ^ | | | | | | | | | | | - B2 | Job1 | 2 | 1,002.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=2]-CatB | No | ^ + B2 | Job1 | 2 | 1,014.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=2]-CatB | No | ^ | | | | | | | | | | | - B2 | Job1 | 10 | 1,402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=10]-CatB | No | ^ + B2 | Job1 | 10 | 1,414.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=10]-CatB | No | ^ | | | | | | | | | | | - B2 | Job2 | 2 | 1,202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=2]-CatB | No | ^ + B2 | Job2 | 2 | 1,214.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=2]-CatB | No | ^ | | | | | | | | | | | - B2 | Job2 | 10 | 1,602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=10]-CatB | No | ^ + B2 | Job2 | 10 | 1,614.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=10]-CatB | No | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt index b586e21cd9..bb98037dd3 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,28 +10,28 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|--------------------- |--------- | - A1 | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=2]-Job1 | Yes | ^ - A2 | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | CatA-[Param=2]-Job1 | No | + A1 | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | CatA-[Param=2]-Job1 | Yes | ^ + A2 | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | CatA-[Param=2]-Job1 | No | | | | | | | | | | | | - A1 | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=2]-Job2 | Yes | - A2 | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | CatA-[Param=2]-Job2 | No | + A1 | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | CatA-[Param=2]-Job2 | Yes | + A2 | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.32 | 0.05 | 2 | CatA-[Param=2]-Job2 | No | | | | | | | | | | | | - A1 | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=10]-Job1 | Yes | ^ - A2 | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | 2 | CatA-[Param=10]-Job1 | No | + A1 | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | CatA-[Param=10]-Job1 | Yes | ^ + A2 | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.19 | 0.03 | 2 | CatA-[Param=10]-Job1 | No | | | | | | | | | | | | - A1 | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=10]-Job2 | Yes | - A2 | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.14 | 0.00 | 2 | CatA-[Param=10]-Job2 | No | + A1 | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | CatA-[Param=10]-Job2 | Yes | + A2 | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.14 | 0.02 | 2 | CatA-[Param=10]-Job2 | No | | | | | | | | | | | | - B1 | Job1 | 2 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=2]-Job1 | Yes | ^ - B2 | Job1 | 2 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.11 | 0.00 | 2 | CatB-[Param=2]-Job1 | No | + B1 | Job1 | 2 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=2]-Job1 | Yes | ^ + B2 | Job1 | 2 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.11 | 0.01 | 2 | CatB-[Param=2]-Job1 | No | | | | | | | | | | | | - B1 | Job2 | 2 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=2]-Job2 | Yes | - B2 | Job2 | 2 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.09 | 0.00 | 2 | CatB-[Param=2]-Job2 | No | + B1 | Job2 | 2 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=2]-Job2 | Yes | + B2 | Job2 | 2 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.09 | 0.01 | 2 | CatB-[Param=2]-Job2 | No | | | | | | | | | | | | - B1 | Job1 | 10 | 1,302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=10]-Job1 | Yes | ^ - B2 | Job1 | 10 | 1,402.0 ns | 6.09 ns | 1.58 ns | 1.08 | 0.00 | 2 | CatB-[Param=10]-Job1 | No | + B1 | Job1 | 10 | 1,314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=10]-Job1 | Yes | ^ + B2 | Job1 | 10 | 1,414.5 ns | 5.88 ns | 8.80 ns | 1.08 | 0.01 | 2 | CatB-[Param=10]-Job1 | No | | | | | | | | | | | | - B1 | Job2 | 10 | 1,502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=10]-Job2 | Yes | - B2 | Job2 | 10 | 1,602.0 ns | 6.09 ns | 1.58 ns | 1.07 | 0.00 | 2 | CatB-[Param=10]-Job2 | No | + B1 | Job2 | 10 | 1,514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=10]-Job2 | Yes | + B2 | Job2 | 10 | 1,614.5 ns | 5.88 ns | 8.80 ns | 1.07 | 0.01 | 2 | CatB-[Param=10]-Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByJob.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByJob.verified.txt index 9944b6dac1..d1330fd6c7 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByJob.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByJob.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,18 +10,18 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|-----:|------------- |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1 | Job1 | No | ^ - Base | Job1 | 10 | 302.0 ns | 6.09 ns | 1.58 ns | 2 | Job1 | No | ^ - Foo | Job1 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 3 | Job1 | No | ^ - Bar | Job1 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 4 | Job1 | No | - Foo | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 5 | Job1 | No | ^ - Bar | Job1 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 6 | Job1 | No | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1 | Job1 | No | ^ + Base | Job1 | 10 | 314.5 ns | 5.88 ns | 8.80 ns | 2 | Job1 | No | ^ + Foo | Job1 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 3 | Job1 | No | ^ + Bar | Job1 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 4 | Job1 | No | + Foo | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 5 | Job1 | No | ^ + Bar | Job1 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 6 | Job1 | No | | | | | | | | | | - Base | Job2 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1 | Job2 | No | ^ - Base | Job2 | 10 | 402.0 ns | 6.09 ns | 1.58 ns | 2 | Job2 | No | ^ - Foo | Job2 | 2 | 702.0 ns | 6.09 ns | 1.58 ns | 3 | Job2 | No | ^ - Bar | Job2 | 2 | 802.0 ns | 6.09 ns | 1.58 ns | 4 | Job2 | No | - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 5 | Job2 | No | ^ - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 6 | Job2 | No | + Base | Job2 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1 | Job2 | No | ^ + Base | Job2 | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 2 | Job2 | No | ^ + Foo | Job2 | 2 | 714.5 ns | 5.88 ns | 8.80 ns | 3 | Job2 | No | ^ + Bar | Job2 | 2 | 814.5 ns | 5.88 ns | 8.80 ns | 4 | Job2 | No | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 5 | Job2 | No | ^ + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 6 | Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByMethod.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByMethod.verified.txt index fd34bb1c63..8b5b634e07 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByMethod.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByMethod.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,19 +10,19 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|-----:|------------------------------------------------ |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | ^ - Base | Job2 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | - Base | Job1 | 10 | 302.0 ns | 6.09 ns | 1.58 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | ^ - Base | Job2 | 10 | 402.0 ns | 6.09 ns | 1.58 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | ^ + Base | Job2 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | + Base | Job1 | 10 | 314.5 ns | 5.88 ns | 8.80 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | ^ + Base | Job2 | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Base | No | | | | | | | | | | - Foo | Job1 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | ^ - Foo | Job2 | 2 | 702.0 ns | 6.09 ns | 1.58 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | - Foo | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | ^ - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | + Foo | Job1 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | ^ + Foo | Job2 | 2 | 714.5 ns | 5.88 ns | 8.80 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | + Foo | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | ^ + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Foo | No | | | | | | | | | | - Bar | Job1 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | ^ - Bar | Job2 | 2 | 802.0 ns | 6.09 ns | 1.58 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | - Bar | Job1 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | ^ - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | + Bar | Job1 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 1 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | ^ + Bar | Job2 | 2 | 814.5 ns | 5.88 ns | 8.80 ns | 2 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | + Bar | Job1 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 3 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | ^ + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 4 | NoBaseline_MethodsParamsJobs_GroupByMethod.Bar | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt index 83dac489bd..6fa6a6534a 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt @@ -2,7 +2,7 @@ BenchmarkDotNet v0.10.x-mock, Microsoft Windows NT 10.0.x.mock (Hyper-V) MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC +Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC [Host] : Clr 4.0.x.mock, 64mock RyuJIT-v4.6.x.mock CONFIGURATION Job1 : extra output line Job2 : extra output line @@ -10,18 +10,18 @@ Frequency: 2531248 Hz, Resolution: 395.0620 ns, Timer: TSC Method | Job | Param | Mean | Error | StdDev | Rank | LogicalGroup | Baseline | ------- |----- |------ |-----------:|--------:|--------:|-----:|------------- |--------- | - Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1 | [Param=2] | No | ^ - Base | Job2 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 2 | [Param=2] | No | - Foo | Job1 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 3 | [Param=2] | No | - Bar | Job1 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 4 | [Param=2] | No | - Foo | Job2 | 2 | 702.0 ns | 6.09 ns | 1.58 ns | 5 | [Param=2] | No | - Bar | Job2 | 2 | 802.0 ns | 6.09 ns | 1.58 ns | 6 | [Param=2] | No | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1 | [Param=2] | No | ^ + Base | Job2 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 2 | [Param=2] | No | + Foo | Job1 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 3 | [Param=2] | No | + Bar | Job1 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 4 | [Param=2] | No | + Foo | Job2 | 2 | 714.5 ns | 5.88 ns | 8.80 ns | 5 | [Param=2] | No | + Bar | Job2 | 2 | 814.5 ns | 5.88 ns | 8.80 ns | 6 | [Param=2] | No | | | | | | | | | | - Base | Job1 | 10 | 302.0 ns | 6.09 ns | 1.58 ns | 1 | [Param=10] | No | ^ - Base | Job2 | 10 | 402.0 ns | 6.09 ns | 1.58 ns | 2 | [Param=10] | No | - Foo | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 3 | [Param=10] | No | - Bar | Job1 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 4 | [Param=10] | No | - Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 5 | [Param=10] | No | - Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 6 | [Param=10] | No | + Base | Job1 | 10 | 314.5 ns | 5.88 ns | 8.80 ns | 1 | [Param=10] | No | ^ + Base | Job2 | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 2 | [Param=10] | No | + Foo | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 3 | [Param=10] | No | + Bar | Job1 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 4 | [Param=10] | No | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 5 | [Param=10] | No | + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 6 | [Param=10] | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/XmlSerializerTests.cs b/tests/BenchmarkDotNet.Tests/Exporters/XmlSerializerTests.cs index 902a6339bd..b254b7622b 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/XmlSerializerTests.cs +++ b/tests/BenchmarkDotNet.Tests/Exporters/XmlSerializerTests.cs @@ -101,7 +101,7 @@ public void CtorThrowsWhenParameterIsNull() [InlineData(null, typeof(ArgumentException))] [InlineData(" ", typeof(ArgumentException))] [InlineData("", typeof(ArgumentException))] - public void WithRootNameThrowsGivenNameIsNullOrWhiteSpace(string name, Type exception) + public void WithRootNameThrowsGivenNameIsNullOrWhiteSpace(string? name, Type exception) { Assert.Throws(exception, () => XmlSerializer.GetBuilder(typeof(MockSource)) .WithRootName(name) @@ -116,7 +116,7 @@ public void WithRootNameThrowsGivenNameIsNullOrWhiteSpace(string name, Type exce [InlineData(null, "MockItem", typeof(ArgumentException))] [InlineData(" ", "MockItem", typeof(ArgumentException))] [InlineData("", "MockItem", typeof(ArgumentException))] - public void WithCollectionItemNameThrowsGivenInvalidArguments(string collectionName, string itemName, Type exception) + public void WithCollectionItemNameThrowsGivenInvalidArguments(string? collectionName, string? itemName, Type exception) { Assert.Throws(exception, () => XmlSerializer.GetBuilder(typeof(MockSource)) .WithCollectionItemName(collectionName, itemName) @@ -127,7 +127,7 @@ public void WithCollectionItemNameThrowsGivenInvalidArguments(string collectionN [InlineData(null, typeof(ArgumentException))] [InlineData(" ", typeof(ArgumentException))] [InlineData("", typeof(ArgumentException))] - public void WithExcludedPropertyThrowsGivenNameIsNullOrWhiteSpace(string name, Type exception) + public void WithExcludedPropertyThrowsGivenNameIsNullOrWhiteSpace(string? name, Type exception) { Assert.Throws(exception, () => XmlSerializer.GetBuilder(typeof(MockSource)) .WithExcludedProperty(name) diff --git a/tests/BenchmarkDotNet.Tests/Infra/TestOutputPresenter.cs b/tests/BenchmarkDotNet.Tests/Infra/TestOutputPresenter.cs new file mode 100644 index 0000000000..7ed360c266 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Infra/TestOutputPresenter.cs @@ -0,0 +1,9 @@ +using Perfolizer.Presenting; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Infra; + +public class TestOutputPresenter(ITestOutputHelper output) : BufferedPresenter +{ + protected override void Flush(string text) => output.WriteLine(text.TrimEnd('\n', '\r')); +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Infra/VerifyHelper.cs b/tests/BenchmarkDotNet.Tests/Infra/VerifyHelper.cs new file mode 100644 index 0000000000..9d563d8ca6 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Infra/VerifyHelper.cs @@ -0,0 +1,16 @@ +using VerifyTests; + +namespace BenchmarkDotNet.Tests.Infra; + +public static class VerifyHelper +{ + public static VerifySettings Create(string? typeName = null) + { + var result = new VerifySettings(); + result.UseDirectory("VerifiedFiles"); + result.DisableDiff(); + if (typeName != null) + result.UseTypeName(typeName); + return result; + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/LightJsonSerializerTests.cs b/tests/BenchmarkDotNet.Tests/LightJsonSerializerTests.cs new file mode 100644 index 0000000000..16ef22efed --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/LightJsonSerializerTests.cs @@ -0,0 +1,56 @@ +using System.Collections.Generic; +using System.Linq; +using SimpleJson; +using Xunit; + +namespace BenchmarkDotNet.Tests +{ + public class LightJsonSerializerTests + { + [Theory] + [InlineData(10.0, "test", double.NaN)] + public void SimpleJson_ReplaceUnsupportedNumericValues_Smoke(object val1, object val2, object val3) + { + //Arrange + var data = new List() + { + val1, val2, val3 + }; + + //Act + for (int i = 0; i < data.Count; i++) + { + data[i] = SimpleJsonSerializer.ReplaceUnsupportedNumericValues(data[i]); + } + + //Assert + Assert.Equal(val1, data[0]); + Assert.Equal(val2, data[1]); + Assert.Equal(SimpleJsonSerializer.JSON_EMPTY_STRING, data[2]); + } + + + [Fact] + public void SimpleJson_SerializeObjectWithUnsupportedNumericValues_ReturnsValidJson() + { + //Arrange + var data = new Dictionary + { + { "Statistic1", float.NaN}, + { "Statistic2", float.NegativeInfinity }, + { "Statistic3", float.PositiveInfinity }, + { "Statistic4", double.NaN}, + { "Statistic5", double.NegativeInfinity }, + { "Statistic6", double.PositiveInfinity } + }; + + //Act + string json = SimpleJsonSerializer.SerializeObject(data); + + //Assert + Assert.True(SimpleJsonSerializer.TryDeserializeObject(json, out var obj)); + var values = (obj as SimpleJson.JsonObject).Select(x => x.Value); + Assert.True(values.All(x => x.Equals(string.Empty))); + } + } +} diff --git a/tests/BenchmarkDotNet.Tests/Mathematics/RankTests.cs b/tests/BenchmarkDotNet.Tests/Mathematics/RankTests.cs index cb3fe7712a..2118b43be1 100644 --- a/tests/BenchmarkDotNet.Tests/Mathematics/RankTests.cs +++ b/tests/BenchmarkDotNet.Tests/Mathematics/RankTests.cs @@ -8,13 +8,13 @@ public class RankTests [Fact] public void RankTest() { - var s1 = new Statistics(100, 101, 100, 101); - var s2 = new Statistics(300, 301, 300, 301); - var s3 = new Statistics(200.3279, 200.3178, 200.4046); - var s4 = new Statistics(200.2298, 200.5738, 200.3582); - var s5 = new Statistics(195, 196, 195, 196); - var actualRanks = RankHelper.GetRanks(s1, s2, s3, s4, s5); - var expectedRanks = new[] { 1, 4, 3, 3, 2 }; + var s1 = new Statistics(100, 101, 100, 101, 100, 101, 100, 101, 100, 101, 100, 101, 100, 101, 100, 101, 100, 101, 100, 101); + var s2 = new Statistics(300, 301, 300, 301, 300, 301, 300, 301, 300, 301, 300, 301, 300, 301, 300, 301, 300, 301, 300, 301); + var s3 = new Statistics(200.3279, 200.3178, 200.4046, 200.3279, 200.3178, 200.4046, 200.3279, 200.3178, 200.4046, 200.3279, 200.3178, 200.4046); + var s4 = new Statistics(200.2298, 200.5738, 200.3582, 200.2298, 200.5738, 200.3582, 200.2298, 200.5738, 200.3582, 200.2298, 200.5738, 200.3582); + var s5 = new Statistics(195, 196, 195, 196, 195, 196, 195, 196, 195, 196, 195, 196, 195, 196, 195, 196, 195, 196, 195, 196); + int[] actualRanks = RankHelper.GetRanks(s1, s2, s3, s4, s5); + int[] expectedRanks = [1, 4, 3, 3, 2]; Assert.Equal(expectedRanks, actualRanks); } } diff --git a/tests/BenchmarkDotNet.Tests/Mathematics/StatisticsTests.cs b/tests/BenchmarkDotNet.Tests/Mathematics/StatisticsTests.cs index f9d21bf452..73843f0bc2 100644 --- a/tests/BenchmarkDotNet.Tests/Mathematics/StatisticsTests.cs +++ b/tests/BenchmarkDotNet.Tests/Mathematics/StatisticsTests.cs @@ -1,196 +1,201 @@ using System; +using System.Collections.Generic; using System.Linq; using BenchmarkDotNet.Mathematics; -using Perfolizer.Mathematics.Common; +using BenchmarkDotNet.Tests.Common; +using JetBrains.Annotations; using Xunit; using Xunit.Abstractions; -namespace BenchmarkDotNet.Tests.Mathematics +namespace BenchmarkDotNet.Tests.Mathematics; + +public class StatisticsTests(ITestOutputHelper output) { - public class StatisticsTests + private void Print(Statistics summary) { - private readonly ITestOutputHelper output; + output.WriteLine("N = " + summary.N); + output.WriteLine("Min = " + summary.Min); + output.WriteLine("LowerFence = " + summary.LowerFence); + output.WriteLine("Q1 = " + summary.Q1); + output.WriteLine("Median = " + summary.Median); + output.WriteLine("Mean = " + summary.Mean); + output.WriteLine("Q3 = " + summary.Q3); + output.WriteLine("UpperFence = " + summary.UpperFence); + output.WriteLine("Max = " + summary.Max); + output.WriteLine("InterquartileRange = " + summary.InterquartileRange); + output.WriteLine("StandardDeviation = " + summary.StandardDeviation); + output.WriteLine("Outlier = [" + string.Join("; ", summary.AllOutliers) + "]"); + output.WriteLine("CI = " + summary.PerfolizerConfidenceInterval.ToString(TestCultureInfo.Instance)); + output.WriteLine("Percentiles = " + summary.Percentiles.ToString(TestCultureInfo.Instance)); + } - public StatisticsTests(ITestOutputHelper output) - { - this.output = output; - } + [Fact] + public void StatisticsWithN0Test() + { + Assert.Throws(() => new Statistics()); + } - private void Print(Statistics summary) - { - output.WriteLine("N = " + summary.N); - output.WriteLine("Min = " + summary.Min); - output.WriteLine("LowerFence = " + summary.LowerFence); - output.WriteLine("Q1 = " + summary.Q1); - output.WriteLine("Median = " + summary.Median); - output.WriteLine("Mean = " + summary.Mean); - output.WriteLine("Q3 = " + summary.Q3); - output.WriteLine("UpperFence = " + summary.UpperFence); - output.WriteLine("Max = " + summary.Max); - output.WriteLine("InterquartileRange = " + summary.InterquartileRange); - output.WriteLine("StandardDeviation = " + summary.StandardDeviation); - output.WriteLine("Outlier = [" + string.Join("; ", summary.AllOutliers) + "]"); - output.WriteLine("CI = " + summary.ConfidenceInterval.ToString(TestCultureInfo.Instance)); - output.WriteLine("Percentiles = " + summary.Percentiles.ToString(TestCultureInfo.Instance)); - } + [Fact] + public void StatisticsWithN1Test() + { + var summary = new Statistics(1); + Print(summary); + AssertEqual(1, summary.Min); + AssertEqual(1, summary.LowerFence); + AssertEqual(1, summary.Q1); + AssertEqual(1, summary.Median); + AssertEqual(1, summary.Mean); + AssertEqual(1, summary.Q3); + AssertEqual(1, summary.UpperFence); + AssertEqual(1, summary.Max); + AssertEqual(0, summary.InterquartileRange); + AssertEqual(0, summary.StandardDeviation); + AssertEqual(Array.Empty(), summary.AllOutliers); + AssertEqual(1, summary.Percentiles.P0); + AssertEqual(1, summary.Percentiles.P25); + AssertEqual(1, summary.Percentiles.P50); + AssertEqual(1, summary.Percentiles.P85); + AssertEqual(1, summary.Percentiles.P95); + AssertEqual(1, summary.Percentiles.P100); + } - [Fact] - public void StatisticsWithN0Test() - { - Assert.Throws(() => new Statistics()); - } + [Fact] + public void StatisticsWithN2Test() + { + var summary = new Statistics(1, 2); + Print(summary); + AssertEqual(1, summary.Min); + AssertEqual(0.5, summary.LowerFence); + AssertEqual(1.25, summary.Q1); + AssertEqual(1.5, summary.Median); + AssertEqual(1.5, summary.Mean); + AssertEqual(1.75, summary.Q3); + AssertEqual(2.5, summary.UpperFence); + AssertEqual(2, summary.Max); + AssertEqual(0.5, summary.InterquartileRange); + AssertEqual(0.70711, summary.StandardDeviation, AbsoluteEqualityComparer.E4); + AssertEqual(Array.Empty(), summary.AllOutliers); + AssertEqual(1, summary.Percentiles.P0); + AssertEqual(1.25, summary.Percentiles.P25); + AssertEqual(1.5, summary.Percentiles.P50); + AssertEqual(1.85, summary.Percentiles.P85); + AssertEqual(1.95, summary.Percentiles.P95); + AssertEqual(2, summary.Percentiles.P100); + } - [Fact] - public void StatisticsWithN1Test() - { - var summary = new Statistics(1); - Print(summary); - Assert.Equal(1, summary.Min); - Assert.Equal(1, summary.LowerFence); - Assert.Equal(1, summary.Q1); - Assert.Equal(1, summary.Median); - Assert.Equal(1, summary.Mean); - Assert.Equal(1, summary.Q3); - Assert.Equal(1, summary.UpperFence); - Assert.Equal(1, summary.Max); - Assert.Equal(0, summary.InterquartileRange); - Assert.Equal(0, summary.StandardDeviation); - Assert.Equal(Array.Empty(), summary.AllOutliers); - Assert.Equal(1, summary.Percentiles.P0); - Assert.Equal(1, summary.Percentiles.P25); - Assert.Equal(1, summary.Percentiles.P50); - Assert.Equal(1, summary.Percentiles.P85); - Assert.Equal(1, summary.Percentiles.P95); - Assert.Equal(1, summary.Percentiles.P100); - } + [Fact] + public void StatisticsWithN3Test() + { + var summary = new Statistics(1, 2, 4); + Print(summary); + AssertEqual(1, summary.Min); + AssertEqual(-0.75, summary.LowerFence); + AssertEqual(1.5, summary.Q1); + AssertEqual(2, summary.Median); + AssertEqual(2.333333, summary.Mean, AbsoluteEqualityComparer.E4); + AssertEqual(3, summary.Q3); + AssertEqual(5.25, summary.UpperFence); + AssertEqual(4, summary.Max); + AssertEqual(1.5, summary.InterquartileRange); + AssertEqual(1.52753, summary.StandardDeviation, AbsoluteEqualityComparer.E4); + AssertEqual(Array.Empty(), summary.AllOutliers); + AssertEqual(1, summary.Percentiles.P0); + AssertEqual(1.5, summary.Percentiles.P25); + AssertEqual(2, summary.Percentiles.P50); + AssertEqual(3.4, summary.Percentiles.P85); + AssertEqual(3.8, summary.Percentiles.P95); + AssertEqual(4, summary.Percentiles.P100); + } - [Fact] - public void StatisticsWithN2Test() - { - var summary = new Statistics(1, 2); - Print(summary); - Assert.Equal(1, summary.Min); - Assert.Equal(0.5, summary.LowerFence); - Assert.Equal(1.25, summary.Q1); - Assert.Equal(1.5, summary.Median); - Assert.Equal(1.5, summary.Mean); - Assert.Equal(1.75, summary.Q3); - Assert.Equal(2.5, summary.UpperFence); - Assert.Equal(2, summary.Max); - Assert.Equal(0.5, summary.InterquartileRange); - Assert.Equal(0.70711, summary.StandardDeviation, 4); - Assert.Equal(Array.Empty(), summary.AllOutliers); - Assert.Equal(1, summary.Percentiles.P0); - Assert.Equal(1.25, summary.Percentiles.P25); - Assert.Equal(1.5, summary.Percentiles.P50); - Assert.Equal(1.85, summary.Percentiles.P85); - Assert.Equal(1.95, summary.Percentiles.P95); - Assert.Equal(2, summary.Percentiles.P100); - } + [Fact] + public void StatisticsWithN7Test() + { + var summary = new Statistics(1, 2, 4, 8, 16, 32, 64); + Print(summary); + AssertEqual(1, summary.Min); + AssertEqual(-28.5, summary.LowerFence); + AssertEqual(3, summary.Q1); + AssertEqual(8, summary.Median); + AssertEqual(18.1428571429, summary.Mean, AbsoluteEqualityComparer.E5); + AssertEqual(24, summary.Q3); + AssertEqual(55.5, summary.UpperFence); + AssertEqual(64, summary.Max); + AssertEqual(21, summary.InterquartileRange); + AssertEqual(22.9378, summary.StandardDeviation, AbsoluteEqualityComparer.E4); + AssertEqual(new[] { 64.0 }, summary.AllOutliers); + AssertEqual(1, summary.Percentiles.P0); + AssertEqual(3, summary.Percentiles.P25); + AssertEqual(8, summary.Percentiles.P50); + AssertEqual(35.2, summary.Percentiles.P85, AbsoluteEqualityComparer.E4); + AssertEqual(54.4, summary.Percentiles.P95, AbsoluteEqualityComparer.E4); + AssertEqual(64, summary.Percentiles.P100); + } - [Fact] - public void StatisticsWithN3Test() - { - var summary = new Statistics(1, 2, 4); - Print(summary); - Assert.Equal(1, summary.Min); - Assert.Equal(-0.75, summary.LowerFence); - Assert.Equal(1.5, summary.Q1); - Assert.Equal(2, summary.Median); - Assert.Equal(2.333333, summary.Mean, 5); - Assert.Equal(3, summary.Q3); - Assert.Equal(5.25, summary.UpperFence); - Assert.Equal(4, summary.Max); - Assert.Equal(1.5, summary.InterquartileRange); - Assert.Equal(1.52753, summary.StandardDeviation, 4); - Assert.Equal(Array.Empty(), summary.AllOutliers); - Assert.Equal(1, summary.Percentiles.P0); - Assert.Equal(1.5, summary.Percentiles.P25); - Assert.Equal(2, summary.Percentiles.P50); - Assert.Equal(3.4, summary.Percentiles.P85); - Assert.Equal(3.8, summary.Percentiles.P95); - Assert.Equal(4, summary.Percentiles.P100); - } + [Fact] + public void OutlierTest() + { + var summary = new Statistics(1, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 10, 10.1); + Print(summary); + AssertEqual(new[] { 10, 10.1 }, summary.AllOutliers); + AssertEqual(new[] { 10, 10.1 }, summary.UpperOutliers); + AssertEqual(Array.Empty(), summary.LowerOutliers); + } - [Fact] - public void StatisticsWithN7Test() - { - var summary = new Statistics(1, 2, 4, 8, 16, 32, 64); - Print(summary); - Assert.Equal(1, summary.Min); - Assert.Equal(-28.5, summary.LowerFence); - Assert.Equal(3, summary.Q1); - Assert.Equal(8, summary.Median); - Assert.Equal(18.1428571429, summary.Mean, 5); - Assert.Equal(24, summary.Q3); - Assert.Equal(55.5, summary.UpperFence); - Assert.Equal(64, summary.Max); - Assert.Equal(21, summary.InterquartileRange); - Assert.Equal(22.9378, summary.StandardDeviation, 4); - Assert.Equal(new[] { 64.0 }, summary.AllOutliers); - Assert.Equal(1, summary.Percentiles.P0); - Assert.Equal(3, summary.Percentiles.P25); - Assert.Equal(8, summary.Percentiles.P50); - Assert.Equal(35.2, summary.Percentiles.P85, 4); - Assert.Equal(54.4, summary.Percentiles.P95, 4); - Assert.Equal(64, summary.Percentiles.P100); - } + [Fact] + public void ConfidenceIntervalTest() + { + var summary = new Statistics(Enumerable.Range(1, 30)); + Print(summary); + Assert.Equal("99.9%", summary.PerfolizerConfidenceInterval.ConfidenceLevel.ToString()); + AssertEqual(15.5, summary.PerfolizerConfidenceInterval.Estimation); + AssertEqual(9.618329, summary.PerfolizerConfidenceInterval.Lower, AbsoluteEqualityComparer.E4); + AssertEqual(21.38167, summary.PerfolizerConfidenceInterval.Upper, AbsoluteEqualityComparer.E4); + } - [Fact] - public void OutlierTest() - { - var summary = new Statistics(1, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 10, 10.1); - Print(summary); - Assert.Equal(new[] { 10, 10.1 }, summary.AllOutliers); - Assert.Equal(new[] { 10, 10.1 }, summary.UpperOutliers); - Assert.Equal(Array.Empty(), summary.LowerOutliers); - } + [Fact] + public void PercentileValuesWithN30Test() + { + var summary = new Statistics(Enumerable.Range(1, 30)); + Print(summary); + AssertEqual(1, summary.Percentiles.P0); + AssertEqual(8.25, summary.Percentiles.P25); + AssertEqual(15.5, summary.Percentiles.P50); + AssertEqual(20.43, summary.Percentiles.P67, AbsoluteEqualityComparer.E4); + AssertEqual(24.2, summary.Percentiles.P80, AbsoluteEqualityComparer.E4); + AssertEqual(25.65, summary.Percentiles.P85); + AssertEqual(27.1, summary.Percentiles.P90); + AssertEqual(28.55, summary.Percentiles.P95, AbsoluteEqualityComparer.E4); + AssertEqual(30, summary.Percentiles.P100); + } - [Fact] - public void ConfidenceIntervalTest() - { - var summary = new Statistics(Enumerable.Range(1, 30)); - Print(summary); - Assert.Equal("99.9%", summary.ConfidenceInterval.Level.ToPercentStr()); - Assert.Equal(15.5, summary.ConfidenceInterval.Mean); - Assert.Equal(summary.StandardError, summary.ConfidenceInterval.StandardError); - Assert.Equal(9.618329, summary.ConfidenceInterval.Lower, 4); - Assert.Equal(21.38167, summary.ConfidenceInterval.Upper, 4); - } + [Fact] + public void PercentileValuesWithN90Test() + { + var a = Enumerable.Range(1, 30); + var b = Enumerable.Repeat(0, 30).Concat(a); + var c = b.Concat(Enumerable.Repeat(31, 30)); + var summary = new Statistics(c); + Print(summary); + AssertEqual(0, summary.Percentiles.P0); + AssertEqual(0, summary.Percentiles.P25); + AssertEqual(15.5, summary.Percentiles.P50); + AssertEqual(30.63, summary.Percentiles.P67, AbsoluteEqualityComparer.E4); + AssertEqual(31, summary.Percentiles.P80); + AssertEqual(31, summary.Percentiles.P85); + AssertEqual(31, summary.Percentiles.P90); + AssertEqual(31, summary.Percentiles.P95); + AssertEqual(31, summary.Percentiles.P100); + } - [Fact] - public void PercentileValuesWithN30Test() - { - var summary = new Statistics(Enumerable.Range(1, 30)); - Print(summary); - Assert.Equal(1, summary.Percentiles.P0); - Assert.Equal(8.25, summary.Percentiles.P25); - Assert.Equal(15.5, summary.Percentiles.P50); - Assert.Equal(20.43, summary.Percentiles.P67, 4); - Assert.Equal(24.2, summary.Percentiles.P80, 4); - Assert.Equal(25.65, summary.Percentiles.P85); - Assert.Equal(27.1, summary.Percentiles.P90); - Assert.Equal(28.55, summary.Percentiles.P95, 4); - Assert.Equal(30, summary.Percentiles.P100); - } + [AssertionMethod] + private static void AssertEqual(double a, double b, IEqualityComparer? comparer = null) + { + Assert.Equal(a, b, comparer ?? AbsoluteEqualityComparer.E9); + } - [Fact] - public void PercentileValuesWithN90Test() - { - var a = Enumerable.Range(1, 30); - var b = Enumerable.Repeat(0, 30).Concat(a); - var c = b.Concat(Enumerable.Repeat(31, 30)); - var summary = new Statistics(c); - Print(summary); - Assert.Equal(0, summary.Percentiles.P0); - Assert.Equal(0, summary.Percentiles.P25); - Assert.Equal(15.5, summary.Percentiles.P50); - Assert.Equal(30.63, summary.Percentiles.P67, 4); - Assert.Equal(31, summary.Percentiles.P80); - Assert.Equal(31, summary.Percentiles.P85); - Assert.Equal(31, summary.Percentiles.P90); - Assert.Equal(31, summary.Percentiles.P95); - Assert.Equal(31, summary.Percentiles.P100); - } + [AssertionMethod] + private static void AssertEqual(double[] a, double[] b, IEqualityComparer? comparer = null) + { + Assert.Equal(a, b, comparer ?? AbsoluteEqualityComparer.E9); } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs index ce54b83bbb..bbe715d660 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs @@ -2,9 +2,11 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Reflection; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Disassemblers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Reports; @@ -14,18 +16,42 @@ using BenchmarkDotNet.Toolchains.Results; using BenchmarkDotNet.Validators; using Perfolizer.Horology; +using Perfolizer.Metrology; using static BenchmarkDotNet.Reports.SummaryTable.SummaryTableColumn; +using Measurement = BenchmarkDotNet.Reports.Measurement; +using MethodInfo = System.Reflection.MethodInfo; namespace BenchmarkDotNet.Tests.Mocks { public static class MockFactory { - public static Summary CreateSummary(Type benchmarkType) + public static Summary CreateSummary(Type benchmarkType, params IColumnHidingRule[] columHidingRules) { var runInfo = BenchmarkConverter.TypeToBenchmarks(benchmarkType); return new Summary( "MockSummary", - runInfo.BenchmarksCases.Select((benchmark, index) => CreateReport(benchmark, 5, (index + 1) * 100)).ToImmutableArray(), + runInfo.BenchmarksCases.Select((benchmark, index) => CreateReport(benchmark, 30, (index + 1) * 100)).ToImmutableArray(), + new HostEnvironmentInfoBuilder().WithoutDotNetSdkVersion().Build(), + string.Empty, + string.Empty, + TimeSpan.FromMinutes(1), + TestCultureInfo.Instance, + ImmutableArray.Empty, + ImmutableArray.Create(columHidingRules)); + } + + public static Summary CreateSummaryWithBiasedDistribution(Type benchmarkType, int min, int median, int max, int n) + { + var runInfo = BenchmarkConverter.TypeToBenchmarks(benchmarkType); + return new Summary( + $"MockSummary-N{n}", + runInfo.BenchmarksCases.Select((benchmark, index) => CreateReportWithBiasedDistribution( + benchmark, + (index + 1) * min, + (index + 1) * median, + (index + 1) * max, + n, + Array.Empty())).ToImmutableArray(), new HostEnvironmentInfoBuilder().WithoutDotNetSdkVersion().Build(), string.Empty, string.Empty, @@ -74,7 +100,7 @@ public static Summary CreateSummary(IConfig config, bool hugeSd, Metric[] metric ImmutableArray.Empty); public static SummaryStyle CreateSummaryStyle(bool printUnitsInHeader = false, bool printUnitsInContent = true, bool printZeroValuesInContent = false, - SizeUnit sizeUnit = null, TimeUnit timeUnit = null, TextJustification textColumnJustification = TextJustification.Left, + SizeUnit? sizeUnit = null, TimeUnit? timeUnit = null, TextJustification textColumnJustification = TextJustification.Left, TextJustification numericColumnJustification = TextJustification.Left) => new SummaryStyle(DefaultCultureInfo.Instance, printUnitsInHeader, sizeUnit, timeUnit, printUnitsInContent: printUnitsInContent, printZeroValuesInContent: printZeroValuesInContent, textColumnJustification: textColumnJustification, @@ -116,6 +142,40 @@ private static BenchmarkReport CreateReport(BenchmarkCase benchmarkCase, bool hu return new BenchmarkReport(true, benchmarkCase, buildResult, buildResult, new List { executeResult }, metrics); } + private static BenchmarkReport CreateReportWithBiasedDistribution(BenchmarkCase benchmarkCase, int min, int median, int max, int n, Metric[] metrics) + { + var buildResult = BuildResult.Success(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty())); + bool isFoo = benchmarkCase.Descriptor.WorkloadMethodDisplayInfo == "Foo"; + bool isBar = benchmarkCase.Descriptor.WorkloadMethodDisplayInfo == "Bar"; + var measurements = from i in Enumerable.Range(0, Math.Max(1, n / 9)) + from m in isFoo ? new[] + { + new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, min), // 1 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 2, 1, min + ((median - min) / 2) + 1), // 3 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 4, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 7, 1, median + ((max - median) / 2)), // 7 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 8, 1, median + ((max - median) / 2)), // 7 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 9, 1, max), // 10 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 9, 1, max), // 10 + } : new[] + { + new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, min), // 1 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, min), // 1 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 2, 1, min + ((median - min) / 2) + 1), // 3 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 2, 1, min + ((median - min) / 2) + 1), // 3 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 4, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 7, 1, median + ((max - median) / 2)), // 7 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 9, 1, max), // 10 + } + select m; + var executeResult = new ExecuteResult(measurements.Take(n).ToList(), default, default, 0); + return new BenchmarkReport(true, benchmarkCase, buildResult, buildResult, new List { executeResult }, metrics); + } + [LongRunJob] public class MockBenchmarkClass { @@ -123,5 +183,10 @@ [Benchmark] public void Foo() { } [Benchmark] public void Bar() { } } + + public static readonly Type MockType = typeof(MockBenchmarkClass); + + public static readonly MethodInfo MockMethodInfo = MockType.GetTypeInfo().GetMethods() + .Single(method => method.Name == nameof(MockBenchmarkClass.Foo)); } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs b/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs index 04cff38e30..d9d2e7490e 100644 --- a/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs +++ b/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs @@ -11,6 +11,7 @@ using BenchmarkDotNet.Parameters; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Mocks; using BenchmarkDotNet.Validators; using Xunit; @@ -22,7 +23,7 @@ public class DefaultOrdererTests "", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray.Empty, ImmutableArray.Empty); private static BenchmarkCase CreateBenchmarkCase(string category, int parameter, params BenchmarkLogicalGroupRule[] rules) => new ( - new Descriptor(null, null, categories: new[] { category }), + new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo, categories: new[] { category }), new Job(), new ParameterInstances(new[] { diff --git a/tests/BenchmarkDotNet.Tests/Phd/Infra/PhdMock.cs b/tests/BenchmarkDotNet.Tests/Phd/Infra/PhdMock.cs new file mode 100644 index 0000000000..a7a6d8e812 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/Infra/PhdMock.cs @@ -0,0 +1,13 @@ +using BenchmarkDotNet.Properties; +using Perfolizer.Phd.Dto; + +namespace BenchmarkDotNet.Tests.Phd.Infra; + +public static class PhdMock +{ + public static readonly PhdEngine Engine = new () + { + Name = BenchmarkDotNetInfo.BenchmarkDotNetCaption, + Version = "0.1729.0-mock" + }; +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/Infra/PhdTestExtensions.cs b/tests/BenchmarkDotNet.Tests/Phd/Infra/PhdTestExtensions.cs new file mode 100644 index 0000000000..a3db74c376 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/Infra/PhdTestExtensions.cs @@ -0,0 +1,22 @@ +using Perfolizer.Metrology; +using Perfolizer.Phd.Base; + +namespace BenchmarkDotNet.Tests.Phd.Infra; + +public static class PhdTestExtensions +{ + public static PhdEntry AddMetrics(this PhdEntry entry, params string[] metrics) + { + for (int i = 0; i < metrics.Length; i++) + { + var measurement = Measurement.Parse(metrics[i]); + entry.Add(new PhdEntry + { + IterationIndex = i, + Value = measurement.NominalValue, + Unit = measurement.Unit + }); + } + return entry; + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/PhdTests.cs b/tests/BenchmarkDotNet.Tests/Phd/PhdTests.cs new file mode 100644 index 0000000000..1772f54789 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/PhdTests.cs @@ -0,0 +1,173 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Phd; +using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Infra; +using BenchmarkDotNet.Tests.Phd.Infra; +using JetBrains.Annotations; +using Perfolizer.Json; +using Perfolizer.Phd.Base; +using Perfolizer.Phd.Dto; +using Perfolizer.Phd.Presenting; +using Perfolizer.Phd.Tables; +using Perfolizer.Presenting; +using VerifyXunit; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Phd; + +[UsesVerify] +public class PhdTests(ITestOutputHelper output) +{ + [Theory] + [MemberData(nameof(EntryDataKeys))] + public Task PhdIndexTest(string key) => VerifyString(key, new PhdIndex(EntryDataMap[key]).Dump()); + + [Theory] + [MemberData(nameof(EntryDataKeys))] + public Task PhdTableTest(string key) + { + var entry = EntryDataMap[key]; + var table = new PhdTable(entry); + var presenter = new StringPresenter(); + new PhdMarkdownTablePresenter(presenter).Present(table, new PhdTableStyle()); + string json = LightJsonSerializer.Serialize(entry, new LightJsonSettings { Indent = true }); + return VerifyString(key, presenter.Dump() + "\n" + json); + } + + private static readonly IDictionary EntryDataMap = new Dictionary + { + { + "default01", Root().Add( + Benchmark("Foo", "10ns", "11ns", "12ns"), + Benchmark("Bar", "200ns", "201ns", "202ns") + ) + }, + { + "default02", Root().Add( + Job(RuntimeMoniker.Net481).Add( + Benchmark("Foo", "10ns", "11ns", "12ns"), + Benchmark("Bar", "20ns", "21ns", "22ns")), + Job(RuntimeMoniker.Net70).Add( + Benchmark("Foo", "30ns", "31ns", "32ns"), + Benchmark("Bar", "40ns", "41ns", "42ns"))) + }, + { + "default03", Root().Add( + Job(RuntimeMoniker.Net70, Jit.RyuJit).Add( + Benchmark("Foo", "30ns", "31ns", "32ns"), + Benchmark("Bar", "40ns", "41ns", "42ns"))) + }, + { + "default04", Root().Add( + Job(RuntimeMoniker.Net481, Jit.LegacyJit).Add( + Benchmark("Foo", "10ns", "11ns", "12ns"), + Benchmark("Bar", "20ns", "21ns", "22ns")), + Job(RuntimeMoniker.Net70, Jit.RyuJit).Add( + Benchmark("Foo", "30ns", "31ns", "32ns"), + Benchmark("Bar", "40ns", "41ns", "42ns"))) + }, + { + "default05", Root().Add( + Enumerable.Range(0, 20).Select(index => + Job((RuntimeMoniker)index, Jit.RyuJit, index).Add( + Benchmark("Foo", index * 10 + 1 + "ns", index * 10 + 2 + "ns", index * 10 + 3 + "ns"), + Benchmark("Bar", index * 10 + 6 + "ns", index * 10 + 7 + "ns", index * 10 + 8 + "ns") + )).ToArray()) + }, + { + "sort01", new PhdEntry + { + Meta = new PhdMeta + { + Table = new PhdTableConfig + { + ColumnDefinitions = + { + new PhdColumnDefinition(".benchmark.method"), + new PhdColumnDefinition("=center") + }, + SortPolicies = + [ + new PhdSortPolicy("=center", PhdSortDirection.Descending) + ] + } + } + }.Add(Benchmark("Foo", "10ms"), Benchmark("Bar", "20ms")) + }, + { + "params01", new PhdEntry + { + Meta = new PhdMeta + { + Table = new PhdTableConfig + { + ColumnDefinitions = + { + new PhdColumnDefinition(".benchmark.method"), + new PhdColumnDefinition(".parameters"), + new PhdColumnDefinition("=center") + } + } + } + }.Add( + new PhdEntry + { + Parameters = new Dictionary { { "A", 1 }, { "B", 2 } } + }.Add(Benchmark("Foo", "10ms"))).Add( + new PhdEntry + { + Parameters = new Dictionary { { "A", 10 }, { "B", 20 } } + }.Add(Benchmark("Bar", "20ms"))) + } + }; + + [UsedImplicitly] public static TheoryData EntryDataKeys = TheoryDataHelper.Create(EntryDataMap.Keys); + + private static PhdEntry Root() => new PhdEntry + { + Meta = new PhdMeta + { + Table = new PhdTableConfig + { + ColumnDefinitions = + [ + new PhdColumnDefinition(".engine") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PhdColumnDefinition(".host.os") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PhdColumnDefinition(".host.cpu") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PhdColumnDefinition(".benchmark") { Cloud = "secondary" }, + new PhdColumnDefinition(".job") { Cloud = "secondary", Compressed = true }, + new PhdColumnDefinition("=center"), + new PhdColumnDefinition("=spread") + ] + } + }, + Engine = PhdMock.Engine, + Host = new HostEnvironmentInfoBuilder().Build().ToPhd() + }; + + private static PhdEntry Job(RuntimeMoniker? runtime = null, Jit? jit = null, int? affinity = null) => new PhdEntry + { + Job = new PhdJob + { + Environment = new BdnEnvironment { Runtime = runtime, Jit = jit, Affinity = affinity } + } + }; + + private static PhdEntry Benchmark(string name, params string[] metrics) => new PhdEntry + { + Benchmark = new BdnBenchmark { Type = "Bench", Method = name } + }.AddMetrics(metrics); + + private Task VerifyString(string key, string content) + { + output.WriteLine(content); + var settings = VerifyHelper.Create("Phd"); + settings.UseParameters(key); + return Verifier.Verify(content, settings); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default01.verified.txt b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default01.verified.txt new file mode 100644 index 0000000000..a412c88898 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default01.verified.txt @@ -0,0 +1,190 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .unit = ns + .value = 10 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .unit = ns + .value = 11 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .unit = ns + .value = 12 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .unit = ns + .value = 200 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .unit = ns + .value = 201 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .unit = ns + .value = 202 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default02.verified.txt b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default02.verified.txt new file mode 100644 index 0000000000..c8759facf6 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default02.verified.txt @@ -0,0 +1,391 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .job + .job.environment + .job.environment.runtime + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 10 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 11 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 12 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 20 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 21 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 22 +[Entry6] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 30 +[Entry7] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 31 +[Entry8] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 32 +[Entry9] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 40 +[Entry10] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 41 +[Entry11] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 42 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default03.verified.txt b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default03.verified.txt new file mode 100644 index 0000000000..af8395637c --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default03.verified.txt @@ -0,0 +1,218 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .job + .job.environment + .job.environment.jit + .job.environment.runtime + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 30 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 31 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 32 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 40 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 41 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 42 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default04.verified.txt b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default04.verified.txt new file mode 100644 index 0000000000..1179e76cd5 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default04.verified.txt @@ -0,0 +1,404 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .job + .job.environment + .job.environment.jit + .job.environment.runtime + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 10 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 11 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 12 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 20 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 21 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 22 +[Entry6] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 30 +[Entry7] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 31 +[Entry8] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 32 +[Entry9] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 40 +[Entry10] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 41 +[Entry11] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 42 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default05.verified.txt b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default05.verified.txt new file mode 100644 index 0000000000..ef0556ab95 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=default05.verified.txt @@ -0,0 +1,3873 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .job + .job.environment + .job.environment.affinity + .job.environment.jit + .job.environment.runtime + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 1 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 2 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 3 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 6 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 7 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 8 +[Entry6] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 11 +[Entry7] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 12 +[Entry8] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 13 +[Entry9] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 16 +[Entry10] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 17 +[Entry11] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 18 +[Entry12] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 21 +[Entry13] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 22 +[Entry14] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 23 +[Entry15] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 26 +[Entry16] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 27 +[Entry17] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 28 +[Entry18] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 31 +[Entry19] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 32 +[Entry20] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 33 +[Entry21] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 36 +[Entry22] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 37 +[Entry23] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 38 +[Entry24] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 41 +[Entry25] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 42 +[Entry26] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 43 +[Entry27] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 46 +[Entry28] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 47 +[Entry29] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 48 +[Entry30] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 51 +[Entry31] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 52 +[Entry32] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 53 +[Entry33] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 56 +[Entry34] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 57 +[Entry35] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 58 +[Entry36] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 61 +[Entry37] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 62 +[Entry38] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 63 +[Entry39] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 66 +[Entry40] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 67 +[Entry41] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 68 +[Entry42] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 71 +[Entry43] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 72 +[Entry44] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 73 +[Entry45] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 76 +[Entry46] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 77 +[Entry47] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 78 +[Entry48] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 81 +[Entry49] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 82 +[Entry50] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 83 +[Entry51] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 86 +[Entry52] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 87 +[Entry53] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 88 +[Entry54] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 91 +[Entry55] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 92 +[Entry56] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 93 +[Entry57] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 96 +[Entry58] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 97 +[Entry59] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 98 +[Entry60] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 101 +[Entry61] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 102 +[Entry62] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 103 +[Entry63] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 106 +[Entry64] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 107 +[Entry65] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 108 +[Entry66] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 111 +[Entry67] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 112 +[Entry68] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 113 +[Entry69] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 116 +[Entry70] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 117 +[Entry71] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 118 +[Entry72] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 121 +[Entry73] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 122 +[Entry74] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 123 +[Entry75] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 126 +[Entry76] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 127 +[Entry77] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 128 +[Entry78] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 131 +[Entry79] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 132 +[Entry80] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 133 +[Entry81] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 136 +[Entry82] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 137 +[Entry83] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 138 +[Entry84] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 141 +[Entry85] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 142 +[Entry86] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 143 +[Entry87] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 146 +[Entry88] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 147 +[Entry89] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 148 +[Entry90] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 151 +[Entry91] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 152 +[Entry92] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 153 +[Entry93] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 156 +[Entry94] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 157 +[Entry95] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 158 +[Entry96] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 161 +[Entry97] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 162 +[Entry98] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 163 +[Entry99] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 166 +[Entry100] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 167 +[Entry101] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 168 +[Entry102] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 171 +[Entry103] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 172 +[Entry104] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 173 +[Entry105] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 176 +[Entry106] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 177 +[Entry107] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 178 +[Entry108] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 181 +[Entry109] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 182 +[Entry110] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 183 +[Entry111] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 186 +[Entry112] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 187 +[Entry113] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 188 +[Entry114] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 191 +[Entry115] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 192 +[Entry116] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 193 +[Entry117] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 196 +[Entry118] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 197 +[Entry119] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 198 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=params01.verified.txt b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=params01.verified.txt new file mode 100644 index 0000000000..ccc03f6896 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=params01.verified.txt @@ -0,0 +1,31 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .iterationIndex + .parameters + .parameters.a + .parameters.b + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .iterationIndex = 0 + .parameters = + .parameters.a = 1 + .parameters.b = 2 + .unit = ms + .value = 10 +[Entry1] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .iterationIndex = 0 + .parameters = + .parameters.a = 10 + .parameters.b = 20 + .unit = ms + .value = 20 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=sort01.verified.txt b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=sort01.verified.txt new file mode 100644 index 0000000000..f1c1835a59 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdIndexTest_key=sort01.verified.txt @@ -0,0 +1,22 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .iterationIndex + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .iterationIndex = 0 + .unit = ms + .value = 10 +[Entry1] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .iterationIndex = 0 + .unit = ms + .value = 20 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default01.verified.txt b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default01.verified.txt new file mode 100644 index 0000000000..49176e7c53 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default01.verified.txt @@ -0,0 +1,123 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench + +| Method | Center | Spread | +|:-------|---------:|--------:| +| Foo | 11.0 ns | 0.81 ns | +| Bar | 201.0 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 11, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 12, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 200, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 201, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 202, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ], + "meta": { + "table": { + "columnDefinitions": [ + { + "selector": ".engine", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".host.os", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".host.cpu", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".benchmark", + "cloud": "secondary" + }, + { + "selector": ".job", + "cloud": "secondary", + "compressed": true + }, + { + "selector": "=center" + }, + { + "selector": "=spread" + } + ] + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default02.verified.txt b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default02.verified.txt new file mode 100644 index 0000000000..f57b844e3f --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default02.verified.txt @@ -0,0 +1,189 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench + +| Method | Runtime | Center | Spread | +|:-------|:--------|--------:|--------:| +| Foo | Net481 | 11.0 ns | 0.81 ns | +| Bar | Net481 | 21.0 ns | 0.81 ns | +| Foo | Net70 | 31.0 ns | 0.81 ns | +| Bar | Net70 | 41.0 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "job": { + "environment": { + "runtime": "net481" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 11, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 12, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 20, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 21, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 22, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net70" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 30, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 31, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 32, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 40, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 41, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 42, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + } + ], + "meta": { + "table": { + "columnDefinitions": [ + { + "selector": ".engine", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".host.os", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".host.cpu", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".benchmark", + "cloud": "secondary" + }, + { + "selector": ".job", + "cloud": "secondary", + "compressed": true + }, + { + "selector": "=center" + }, + { + "selector": "=spread" + } + ] + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default03.verified.txt b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default03.verified.txt new file mode 100644 index 0000000000..1216d13bfd --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default03.verified.txt @@ -0,0 +1,133 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench, Runtime = Net70, Jit = RyuJit + +| Method | Center | Spread | +|:-------|--------:|--------:| +| Foo | 31.0 ns | 0.81 ns | +| Bar | 41.0 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "job": { + "environment": { + "runtime": "net70", + "jit": "ryuJit" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 30, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 31, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 32, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 40, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 41, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 42, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + } + ], + "meta": { + "table": { + "columnDefinitions": [ + { + "selector": ".engine", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".host.os", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".host.cpu", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".benchmark", + "cloud": "secondary" + }, + { + "selector": ".job", + "cloud": "secondary", + "compressed": true + }, + { + "selector": "=center" + }, + { + "selector": "=spread" + } + ] + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default04.verified.txt b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default04.verified.txt new file mode 100644 index 0000000000..d6e11e30c8 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default04.verified.txt @@ -0,0 +1,194 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench + +@Net481LegacyJit: Runtime = Net481, Jit = LegacyJit +@Net70RyuJit: Runtime = Net70, Jit = RyuJit + +| Method | Job | Center | Spread | +|:-------|:----------------|--------:|--------:| +| Foo | Net481LegacyJit | 11.0 ns | 0.81 ns | +| Bar | Net481LegacyJit | 21.0 ns | 0.81 ns | +| Foo | Net70RyuJit | 31.0 ns | 0.81 ns | +| Bar | Net70RyuJit | 41.0 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "job": { + "environment": { + "runtime": "net481", + "jit": "legacyJit" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 11, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 12, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 20, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 21, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 22, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net70", + "jit": "ryuJit" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 30, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 31, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 32, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 40, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 41, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 42, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + } + ], + "meta": { + "table": { + "columnDefinitions": [ + { + "selector": ".engine", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".host.os", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".host.cpu", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".benchmark", + "cloud": "secondary" + }, + { + "selector": ".job", + "cloud": "secondary", + "compressed": true + }, + { + "selector": "=center" + }, + { + "selector": "=spread" + } + ] + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default05.verified.txt b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default05.verified.txt new file mode 100644 index 0000000000..4c02ab89c5 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=default05.verified.txt @@ -0,0 +1,1276 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench, Jit = RyuJit + +@HostProcessAffinity0: Runtime = HostProcess, Affinity = 0 +@Squid: Runtime = NotRecognized, Affinity = 1 +@MonoAffinity2: Runtime = Mono, Affinity = 2 +@Net461Affinity3: Runtime = Net461, Affinity = 3 +@Net462Affinity4: Runtime = Net462, Affinity = 4 +@Net47Affinity5: Runtime = Net47, Affinity = 5 +@Net471Affinity6: Runtime = Net471, Affinity = 6 +@Net472Affinity7: Runtime = Net472, Affinity = 7 +@Net48Affinity8: Runtime = Net48, Affinity = 8 +@Net481Affinity9: Runtime = Net481, Affinity = 9 +@Liger: Runtime = NetCoreApp20, Affinity = 10 +@Rat: Runtime = NetCoreApp21, Affinity = 11 +@Skate: Runtime = NetCoreApp22, Affinity = 12 +@Perch: Runtime = NetCoreApp30, Affinity = 13 +@Roach: Runtime = NetCoreApp31, Affinity = 14 +@Roe: Runtime = NetCoreApp50, Affinity = 15 +@Net50Affinity16: Runtime = Net50, Affinity = 16 +@Net60Affinity17: Runtime = Net60, Affinity = 17 +@Net70Affinity18: Runtime = Net70, Affinity = 18 +@Net80Affinity19: Runtime = Net80, Affinity = 19 + +| Method | Job | Center | Spread | +|:-------|:---------------------|----------:|--------:| +| Foo | HostProcessAffinity0 | 2.00 ns | 0.81 ns | +| Bar | HostProcessAffinity0 | 7.00 ns | 0.81 ns | +| Foo | Squid | 12.00 ns | 0.81 ns | +| Bar | Squid | 17.00 ns | 0.81 ns | +| Foo | MonoAffinity2 | 22.00 ns | 0.81 ns | +| Bar | MonoAffinity2 | 27.00 ns | 0.81 ns | +| Foo | Net461Affinity3 | 32.00 ns | 0.81 ns | +| Bar | Net461Affinity3 | 37.00 ns | 0.81 ns | +| Foo | Net462Affinity4 | 42.00 ns | 0.81 ns | +| Bar | Net462Affinity4 | 47.00 ns | 0.81 ns | +| Foo | Net47Affinity5 | 52.00 ns | 0.81 ns | +| Bar | Net47Affinity5 | 57.00 ns | 0.81 ns | +| Foo | Net471Affinity6 | 62.00 ns | 0.81 ns | +| Bar | Net471Affinity6 | 67.00 ns | 0.81 ns | +| Foo | Net472Affinity7 | 72.00 ns | 0.81 ns | +| Bar | Net472Affinity7 | 77.00 ns | 0.81 ns | +| Foo | Net48Affinity8 | 82.00 ns | 0.81 ns | +| Bar | Net48Affinity8 | 87.00 ns | 0.81 ns | +| Foo | Net481Affinity9 | 92.00 ns | 0.81 ns | +| Bar | Net481Affinity9 | 97.00 ns | 0.81 ns | +| Foo | Liger | 102.00 ns | 0.81 ns | +| Bar | Liger | 107.00 ns | 0.81 ns | +| Foo | Rat | 112.00 ns | 0.81 ns | +| Bar | Rat | 117.00 ns | 0.81 ns | +| Foo | Skate | 122.00 ns | 0.81 ns | +| Bar | Skate | 127.00 ns | 0.81 ns | +| Foo | Perch | 132.00 ns | 0.81 ns | +| Bar | Perch | 137.00 ns | 0.81 ns | +| Foo | Roach | 142.00 ns | 0.81 ns | +| Bar | Roach | 147.00 ns | 0.81 ns | +| Foo | Roe | 152.00 ns | 0.81 ns | +| Bar | Roe | 157.00 ns | 0.81 ns | +| Foo | Net50Affinity16 | 162.00 ns | 0.81 ns | +| Bar | Net50Affinity16 | 167.00 ns | 0.81 ns | +| Foo | Net60Affinity17 | 172.00 ns | 0.81 ns | +| Bar | Net60Affinity17 | 177.00 ns | 0.81 ns | +| Foo | Net70Affinity18 | 182.00 ns | 0.81 ns | +| Bar | Net70Affinity18 | 187.00 ns | 0.81 ns | +| Foo | Net80Affinity19 | 192.00 ns | 0.81 ns | +| Bar | Net80Affinity19 | 197.00 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "job": { + "environment": { + "runtime": "hostProcess", + "jit": "ryuJit", + "affinity": 0 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 1, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 2, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 3, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 6, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 7, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 8, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "notRecognized", + "jit": "ryuJit", + "affinity": 1 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 11, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 12, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 13, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 16, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 17, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 18, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "mono", + "jit": "ryuJit", + "affinity": 2 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 21, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 22, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 23, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 26, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 27, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 28, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net461", + "jit": "ryuJit", + "affinity": 3 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 31, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 32, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 33, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 36, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 37, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 38, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net462", + "jit": "ryuJit", + "affinity": 4 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 41, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 42, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 43, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 46, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 47, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 48, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net47", + "jit": "ryuJit", + "affinity": 5 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 51, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 52, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 53, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 56, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 57, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 58, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net471", + "jit": "ryuJit", + "affinity": 6 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 61, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 62, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 63, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 66, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 67, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 68, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net472", + "jit": "ryuJit", + "affinity": 7 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 71, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 72, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 73, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 76, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 77, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 78, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net48", + "jit": "ryuJit", + "affinity": 8 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 81, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 82, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 83, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 86, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 87, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 88, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net481", + "jit": "ryuJit", + "affinity": 9 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 91, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 92, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 93, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 96, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 97, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 98, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp20", + "jit": "ryuJit", + "affinity": 10 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 101, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 102, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 103, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 106, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 107, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 108, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp21", + "jit": "ryuJit", + "affinity": 11 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 111, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 112, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 113, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 116, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 117, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 118, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp22", + "jit": "ryuJit", + "affinity": 12 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 121, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 122, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 123, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 126, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 127, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 128, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp30", + "jit": "ryuJit", + "affinity": 13 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 131, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 132, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 133, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 136, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 137, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 138, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp31", + "jit": "ryuJit", + "affinity": 14 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 141, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 142, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 143, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 146, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 147, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 148, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp50", + "jit": "ryuJit", + "affinity": 15 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 151, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 152, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 153, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 156, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 157, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 158, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net50", + "jit": "ryuJit", + "affinity": 16 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 161, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 162, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 163, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 166, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 167, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 168, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net60", + "jit": "ryuJit", + "affinity": 17 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 171, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 172, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 173, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 176, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 177, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 178, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net70", + "jit": "ryuJit", + "affinity": 18 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 181, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 182, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 183, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 186, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 187, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 188, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net80", + "jit": "ryuJit", + "affinity": 19 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 191, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 192, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 193, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 196, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 197, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 198, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + } + ], + "meta": { + "table": { + "columnDefinitions": [ + { + "selector": ".engine", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".host.os", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".host.cpu", + "cloud": "primary", + "isSelfExplanatory": true, + "isAtomic": true + }, + { + "selector": ".benchmark", + "cloud": "secondary" + }, + { + "selector": ".job", + "cloud": "secondary", + "compressed": true + }, + { + "selector": "=center" + }, + { + "selector": "=spread" + } + ] + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=params01.verified.txt b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=params01.verified.txt new file mode 100644 index 0000000000..afa4ccffd7 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=params01.verified.txt @@ -0,0 +1,66 @@ +| Method | A | B | Center | +|:-------|---:|---:|--------:| +| Foo | 1 | 2 | 10.0 ms | +| Bar | 10 | 20 | 20.0 ms | + +{ + "nested": [ + { + "parameters": { + "a": 1, + "b": 2 + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ms", + "iterationIndex": 0 + } + ] + } + ] + }, + { + "parameters": { + "a": 10, + "b": 20 + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 20, + "unit": "ms", + "iterationIndex": 0 + } + ] + } + ] + } + ], + "meta": { + "table": { + "columnDefinitions": [ + { + "selector": ".benchmark.method" + }, + { + "selector": ".parameters" + }, + { + "selector": "=center" + } + ] + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=sort01.verified.txt b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=sort01.verified.txt new file mode 100644 index 0000000000..2466122dde --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Phd/VerifiedFiles/Phd.PhdTableTest_key=sort01.verified.txt @@ -0,0 +1,53 @@ +| Method | Center | +|:-------|--------:| +| Bar | 20.0 ms | +| Foo | 10.0 ms | + +{ + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ms", + "iterationIndex": 0 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 20, + "unit": "ms", + "iterationIndex": 0 + } + ] + } + ], + "meta": { + "table": { + "columnDefinitions": [ + { + "selector": ".benchmark.method" + }, + { + "selector": "=center" + } + ], + "sortPolicies": [ + { + "selector": "=center", + "direction": "descending" + } + ] + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/CpuInfoFormatterTests.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/CpuInfoFormatterTests.cs deleted file mode 100644 index 83d14685a4..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/CpuInfoFormatterTests.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System.Text; -using System.Threading.Tasks; -using BenchmarkDotNet.Portability.Cpu; -using BenchmarkDotNet.Tests.Builders; -using VerifyXunit; -using Xunit; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - [Collection("VerifyTests")] - [UsesVerify] - public class CpuInfoFormatterTests - { - [Fact] - public Task FormatTest() - { - var captions = new StringBuilder(); - foreach (var processorName in new[] { null, "", "Intel" }) - foreach (var physicalProcessorCount in new int?[] { null, 0, 1, 2 }) - foreach (var physicalCoreCount in new int?[] { null, 0, 1, 2 }) - foreach (var logicalCoreCount in new int?[] { null, 0, 1, 2 }) - { - var mockCpuInfo = new CpuInfo(processorName, physicalProcessorCount, physicalCoreCount, logicalCoreCount, null, null, null); - captions.AppendLine(CpuInfoFormatter.Format(mockCpuInfo)); - } - - var settings = VerifySettingsFactory.Create(); - return Verifier.Verify(captions.ToString(), settings); - } - } -} diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/ProcCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/ProcCpuInfoParserTests.cs deleted file mode 100644 index 6d2faa423e..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/ProcCpuInfoParserTests.cs +++ /dev/null @@ -1,90 +0,0 @@ -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; -using Xunit; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - public class ProcCpuInfoParserTests - { - [Fact] - public void EmptyTest() - { - var parser = ProcCpuInfoParser.ParseOutput(string.Empty); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - - Assert.Null(parser.MaxFrequency); - Assert.Null(parser.MinFrequency); - } - - [Fact] - public void MalformedTest() - { - var parser = ProcCpuInfoParser.ParseOutput("malformedkey: malformedvalue\n\nmalformedkey2: malformedvalue2"); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - Assert.Null(parser.MaxFrequency); - Assert.Null(parser.MinFrequency); - } - - [Fact] - public void TwoProcessorWithDifferentCoresCountTest() - { - string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoProcessorWithDifferentCoresCount.txt"); - var parser = ProcCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Unknown processor with 2 cores and hyper threading, Unknown processor with 4 cores", parser.ProcessorName); - Assert.Equal(2, parser.PhysicalProcessorCount); - Assert.Equal(6, parser.PhysicalCoreCount); - Assert.Equal(8, parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - Assert.Equal(0.8 * Frequency.GHz, parser.MinFrequency); - Assert.Equal(2.5 * Frequency.GHz, parser.MaxFrequency); - } - - - [Fact] - public void RealOneProcessorTwoCoresTest() - { - string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoRealOneProcessorTwoCores.txt"); - var parser = ProcCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Core(TM) i5-6200U CPU @ 2.30GHz", parser.ProcessorName); - Assert.Equal(1, parser.PhysicalProcessorCount); - Assert.Equal(2, parser.PhysicalCoreCount); - Assert.Equal(4, parser.LogicalCoreCount); - Assert.Equal(2.3 * Frequency.GHz, parser.NominalFrequency); - Assert.Equal(0.8 * Frequency.GHz, parser.MinFrequency); - Assert.Equal(2.3 * Frequency.GHz, parser.MaxFrequency); - } - - [Fact] - public void RealOneProcessorFourCoresTest() - { - string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoRealOneProcessorFourCores.txt"); - var parser = ProcCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", parser.ProcessorName); - Assert.Equal(1, parser.PhysicalProcessorCount); - Assert.Equal(4, parser.PhysicalCoreCount); - Assert.Equal(8, parser.LogicalCoreCount); - Assert.Equal(2.5 * Frequency.GHz, parser.NominalFrequency); - Assert.Equal(0.8 * Frequency.GHz, parser.MinFrequency); - Assert.Equal(2.5 * Frequency.GHz, parser.MaxFrequency); - } - - [Theory] - [InlineData("Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", 2.50)] - [InlineData("Intel(R) Core(TM) i5-6200U CPU @ 2.30GHz", 2.30)] - [InlineData("Unknown processor with 2 cores and hyper threading, Unknown processor with 4 cores", 0)] - [InlineData("Intel(R) Core(TM) i5-2500 CPU @ 3.30GHz", 3.30)] - public void ParseFrequencyFromBrandStringTests(string brandString, double expectedGHz) - { - var frequency = ProcCpuInfoParser.ParseFrequencyFromBrandString(brandString); - Assert.Equal(Frequency.FromGHz(expectedGHz), frequency); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/SysctlCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/SysctlCpuInfoParserTests.cs deleted file mode 100644 index 5d2ba82520..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/SysctlCpuInfoParserTests.cs +++ /dev/null @@ -1,43 +0,0 @@ -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; -using Xunit; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - public class SysctlCpuInfoParserTests - { - [Fact] - public void EmptyTest() - { - var parser = SysctlCpuInfoParser.ParseOutput(string.Empty); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - } - - [Fact] - public void MalformedTest() - { - var parser = SysctlCpuInfoParser.ParseOutput("malformedkey=malformedvalue\n\nmalformedkey2=malformedvalue2"); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - } - - [Fact] - public void RealOneProcessorFourCoresTest() - { - string cpuInfo = TestHelper.ReadTestFile("SysctlRealOneProcessorFourCores.txt"); - var parser = SysctlCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz", parser.ProcessorName); - Assert.Equal(1, parser.PhysicalProcessorCount); - Assert.Equal(4, parser.PhysicalCoreCount); - Assert.Equal(8, parser.LogicalCoreCount); - Assert.Equal(2200 * Frequency.MHz, parser.NominalFrequency); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestHelper.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestHelper.cs deleted file mode 100644 index e4af5025c0..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestHelper.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System.IO; -using System.Reflection; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - public static class TestHelper - { - public static string ReadTestFile(string name) - { - var assembly = typeof(TestHelper).GetTypeInfo().Assembly; - string resourceName = $"{typeof(TestHelper).Namespace}.TestFiles.{name}"; - - using (var stream = assembly.GetManifestResourceStream(resourceName)) - using (var reader = new StreamReader(stream)) - { - return reader.ReadToEnd(); - } - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/WmicCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/WmicCpuInfoParserTests.cs deleted file mode 100644 index b1faa4b521..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/WmicCpuInfoParserTests.cs +++ /dev/null @@ -1,103 +0,0 @@ -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; -using Xunit; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - public class WmicCpuInfoParserTests - { - [Fact] - public void EmptyTest() - { - var parser = WmicCpuInfoParser.ParseOutput(string.Empty); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - } - - [Fact] - public void MalformedTest() - { - var parser = WmicCpuInfoParser.ParseOutput("malformedkey=malformedvalue\n\nmalformedkey2=malformedvalue2"); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - } - - [Fact] - public void RealTwoProcessorEightCoresTest() - { - const string cpuInfo = @" - -MaxClockSpeed=2400 -Name=Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz -NumberOfCores=8 -NumberOfLogicalProcessors=16 - - -MaxClockSpeed=2400 -Name=Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz -NumberOfCores=8 -NumberOfLogicalProcessors=16 - -"; - var parser = WmicCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz", parser.ProcessorName); - Assert.Equal(2, parser.PhysicalProcessorCount); - Assert.Equal(16, parser.PhysicalCoreCount); - Assert.Equal(32, parser.LogicalCoreCount); - Assert.Equal(2400 * Frequency.MHz, parser.MaxFrequency); - } - - [Fact] - public void RealTwoProcessorEightCoresWithWmicBugTest() - { - const string cpuInfo = - "\r\r\n" + - "\r\r\n" + - "MaxClockSpeed=3111\r\r\n" + - "Name=Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz\r\r\n" + - "NumberOfCores=8\r\r\n" + - "NumberOfLogicalProcessors=16\r\r\n" + - "\r\r\n" + - "\r\r\n" + - "MaxClockSpeed=3111\r\r\n" + - "Name=Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz\r\r\n" + - "NumberOfCores=8\r\r\n" + - "NumberOfLogicalProcessors=16\r\r\n" + - "\r\r\n" + - "\r\r\n" + - "\r\r\n"; - var parser = WmicCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz", parser.ProcessorName); - Assert.Equal(2, parser.PhysicalProcessorCount); - Assert.Equal(16, parser.PhysicalCoreCount); - Assert.Equal(32, parser.LogicalCoreCount); - Assert.Equal(3111 * Frequency.MHz, parser.MaxFrequency); - } - - [Fact] - public void RealOneProcessorFourCoresTest() - { - const string cpuInfo = @" - -MaxClockSpeed=2500 -Name=Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz -NumberOfCores=4 -NumberOfLogicalProcessors=8 - -"; - - var parser = WmicCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", parser.ProcessorName); - Assert.Equal(1, parser.PhysicalProcessorCount); - Assert.Equal(4, parser.PhysicalCoreCount); - Assert.Equal(8, parser.LogicalCoreCount); - Assert.Equal(2500 * Frequency.MHz, parser.MaxFrequency); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/HyperVTests.cs b/tests/BenchmarkDotNet.Tests/Portability/HyperVTests.cs index 8abe71fc64..8e2588bbfb 100644 --- a/tests/BenchmarkDotNet.Tests/Portability/HyperVTests.cs +++ b/tests/BenchmarkDotNet.Tests/Portability/HyperVTests.cs @@ -21,7 +21,7 @@ public void ContainsCorrectName() [InlineData("Dell", "ubuntu", false)] [InlineData("microsoft corporation", null, false)] [InlineData(null, "virtual machine", false)] - public void DetectsVirtualMachine(string manufacturer, string model, bool expectedResult) + public void DetectsVirtualMachine(string? manufacturer, string? model, bool expectedResult) { bool result = hypervisor.IsVirtualMachine(manufacturer, model); Assert.Equal(expectedResult, result); diff --git a/tests/BenchmarkDotNet.Tests/Portability/VMWareTests.cs b/tests/BenchmarkDotNet.Tests/Portability/VMWareTests.cs index 96758a97c2..f2e383ba85 100644 --- a/tests/BenchmarkDotNet.Tests/Portability/VMWareTests.cs +++ b/tests/BenchmarkDotNet.Tests/Portability/VMWareTests.cs @@ -20,7 +20,7 @@ public void ContainsCorrectName() [InlineData(null, "vmWare", true)] [InlineData("VMWare Inc", "redundant", false)] [InlineData("VMWare Inc", null, false)] - public void DetectsVirtualMachine(string manufacturer, string model, bool expectedResult) + public void DetectsVirtualMachine(string? manufacturer, string? model, bool expectedResult) { bool result = hypervisor.IsVirtualMachine(manufacturer, model); Assert.Equal(expectedResult, result); diff --git a/tests/BenchmarkDotNet.Tests/Portability/VirtualBoxTests.cs b/tests/BenchmarkDotNet.Tests/Portability/VirtualBoxTests.cs index 83e13ef500..56cf697fb0 100644 --- a/tests/BenchmarkDotNet.Tests/Portability/VirtualBoxTests.cs +++ b/tests/BenchmarkDotNet.Tests/Portability/VirtualBoxTests.cs @@ -18,7 +18,7 @@ public void ContainsCorrectName() [InlineData("redundant", "VirtualBox", true)] [InlineData("redundant", "vmware", false)] [InlineData("redundant", null, false)] - public void DetectsVirtualMachine(string manufacturer, string model, bool expectedResult) + public void DetectsVirtualMachine(string manufacturer, string? model, bool expectedResult) { bool result = hypervisor.IsVirtualMachine(manufacturer, model); Assert.Equal(expectedResult, result); diff --git a/tests/BenchmarkDotNet.Tests/Properties/BenchmarkDotNetInfoTests.cs b/tests/BenchmarkDotNet.Tests/Properties/BenchmarkDotNetInfoTests.cs new file mode 100644 index 0000000000..66581494e7 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Properties/BenchmarkDotNetInfoTests.cs @@ -0,0 +1,19 @@ +using BenchmarkDotNet.Properties; +using Xunit; + +namespace BenchmarkDotNet.Tests.Properties; + +public class BenchmarkDotNetInfoTests +{ + [Theory] + [InlineData("", "")] + [InlineData("1.2.3.4", "1.2.3.4")] + [InlineData("1729-foo", "1729-foo")] + [InlineData("0.13.9+228a464e8be6c580ad9408e98f18813f6407fb5a", "0.13.9")] + [InlineData("1-2+3", "1-2")] + public void RemoveVersionMetadata(string input, string expectedOutput) + { + string? actualOutput = BenchmarkDotNetInfo.RemoveVersionMetadata(input); + Assert.Equal(expectedOutput, actualOutput); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Reports/FakeMetricDescriptor.cs b/tests/BenchmarkDotNet.Tests/Reports/FakeMetricDescriptor.cs index 8c7316e638..22d06c2b3e 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/FakeMetricDescriptor.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/FakeMetricDescriptor.cs @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Tests.Reports { internal sealed class FakeMetricDescriptor : IMetricDescriptor { - public FakeMetricDescriptor(string id, string legend, string numberFormat = null) + public FakeMetricDescriptor(string id, string legend, string? numberFormat = null) { Id = id; Legend = legend; diff --git a/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs b/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs index 96c9519a14..f36066fa8a 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs @@ -48,7 +48,7 @@ public TestData(RatioStyle ratioStyle, int[] meanValues, int noise, string[] exp private static readonly IDictionary Data = new Dictionary { { "Percentage", new TestData(RatioStyle.Percentage, new[] { 100, 15, 115 }, 1, new[] { "baseline", "-85%", "+15%" }) }, - { "Trend", new TestData(RatioStyle.Trend, new[] { 100, 15, 115 }, 1, new[] { "baseline", "6.83x faster", "1.15x slower" }) } + { "Trend", new TestData(RatioStyle.Trend, new[] { 100, 15, 115 }, 1, new[] { "baseline", "6.84x faster", "1.15x slower" }) } }; [UsedImplicitly] diff --git a/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs b/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs index ebea88b5ad..fb15149aa3 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs @@ -1,6 +1,8 @@ using System.Linq; +using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Order; @@ -171,5 +173,311 @@ public void MissingValueInMetricColumnIsNA() // assert Assert.Equal(new[] { "-", "NA" }, actual); } + + #region Issue #2673 + [Fact] + public void DefaultExceptionDiagnoserConfig_WhenExceptionsIsNotZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var exceptionConfig = new ExceptionDiagnoserConfig(); + + var exceptionsFrequencyMetricDescriptor = new ExceptionDiagnoser.ExceptionsFrequencyMetricDescriptor(exceptionConfig); + + var metric = new Metric(exceptionsFrequencyMetricDescriptor, 5); + var metrics = new[] { metric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + var actual = table.Columns.First(c => c.Header == "Exceptions").Content; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, actual); + } + + [Fact] + public void DefaultExceptionDiagnoserConfig_WhenExceptionsIsZero() + { + + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var exceptionConfig = new ExceptionDiagnoserConfig(); + + var exceptionsFrequencyMetricDescriptor = new ExceptionDiagnoser.ExceptionsFrequencyMetricDescriptor(exceptionConfig); + + var metric = new Metric(exceptionsFrequencyMetricDescriptor, 0); + var metrics = new[] { metric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + var actual = table.Columns.First(c => c.Header == "Exceptions").Content; + + // assert + Assert.Equal(new[] { "-", "-" }, actual); + } + + [Fact] + public void HideExceptionDiagnoserConfig_WhenExceptionsIsNotZero() + { + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var exceptionConfig = new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: false); + + var exceptionsFrequencyMetricDescriptor = new ExceptionDiagnoser.ExceptionsFrequencyMetricDescriptor(exceptionConfig); + + var metric = new Metric(exceptionsFrequencyMetricDescriptor, 5); + var metrics = new[] { metric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + var actual = table.Columns.First(c => c.Header == "Exceptions").Content; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, actual); + } + + [Fact] + public void HideExceptionDiagnoserConfig_WhenExceptionsIsZero() + { + + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var exceptionConfig = new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: false); + + var exceptionsFrequencyMetricDescriptor = new ExceptionDiagnoser.ExceptionsFrequencyMetricDescriptor(exceptionConfig); + + var metric = new Metric(exceptionsFrequencyMetricDescriptor, 0); + var metrics = new[] { metric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + var isExist = table.Columns.Any(c => c.Header == "Exceptions"); + + // assert + Assert.False(isExist); + } + + [Fact] + public void DefaultThreadingDiagnoserConfig_WhenDescriptorValuesAreNotZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 5); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 5); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + var lockContentionCount = table.Columns.FirstOrDefault(c => c.Header == "Lock Contentions").Content; + var completedWorkItemCount = table.Columns.FirstOrDefault(c => c.Header == "Completed Work Items").Content; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, lockContentionCount); + Assert.Equal(new[] { "5.0000", "5.0000" }, completedWorkItemCount); + } + + [Fact] + public void DefaultThreadingDiagnoserConfig_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + var lockContentionCount = table.Columns.FirstOrDefault(c => c.Header == "Lock Contentions").Content; + var completedWorkItemCount = table.Columns.FirstOrDefault(c => c.Header == "Completed Work Items").Content; + + // assert + Assert.Equal(new[] { "-", "-" }, lockContentionCount); + Assert.Equal(new[] { "-", "-" }, completedWorkItemCount); + } + + [Fact] + public void HideLockContentionCountThreadingDiagnoserConfig_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayLockContentionWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Null(lockContentionCount); + Assert.Equal(new[] { "-", "-" }, completedWorkItemCount); + } + + [Fact] + public void HideLockContentionCountThreadingDiagnoserConfig_WhenDescriptorValuesAreNotZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayLockContentionWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 5); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 5); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, lockContentionCount); + Assert.Equal(new[] { "5.0000", "5.0000" }, completedWorkItemCount); + } + + [Fact] + public void HideCompletedWorkItemCountThreadingDiagnoserConfig_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Null(completedWorkItemCount); + Assert.Equal(new[] { "-", "-" }, lockContentionCount); + } + + [Fact] + public void HideCompletedWorkItemCountThreadingDiagnoserConfig_WhenDescriptorValuesAreNotZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 5); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 5); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, lockContentionCount); + Assert.Equal(new[] { "5.0000", "5.0000" }, completedWorkItemCount); + } + + [Fact] + public void HideThreadingDiagnoserConfigs_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false, displayLockContentionWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Null(lockContentionCount); + Assert.Null(completedWorkItemCount); + } + + [Fact] + public void DisplayThreadingDiagnoserConfigs_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: true, displayLockContentionWhenZero: true); + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Equal(new[] { "-", "-" }, lockContentionCount); + Assert.Equal(new[] { "-", "-" }, completedWorkItemCount); + } + #endregion } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs b/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs index 8952accb40..92811e5933 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs @@ -35,7 +35,7 @@ public void SummaryWithFailureReportDoesNotThrowNre() private static IConfig CreateConfig() { // We use runtime as selector later. It is chosen as selector just to be close to initial issue. Nothing particularly special about it. - Job coreJob = new Job(Job.Default).WithRuntime(CoreRuntime.Core20).ApplyAndFreeze(RunMode.Dry); + Job coreJob = new Job(Job.Default).WithRuntime(CoreRuntime.Core80).ApplyAndFreeze(RunMode.Dry); Job clrJob = new Job(Job.Default).WithRuntime(ClrRuntime.Net462).ApplyAndFreeze(RunMode.Dry); return ManualConfig.Create(DefaultConfig.Instance).AddJob(coreJob).AddJob(clrJob); } diff --git a/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs b/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs index 4eb72b5d59..4b3e92d95d 100644 --- a/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs @@ -135,7 +135,7 @@ public void JobMutatorsApplySettingsToAllNonMutatorJobs() typeof(WithMutator), DefaultConfig.Instance .AddJob(Job.Default.WithRuntime(ClrRuntime.Net462)) - .AddJob(Job.Default.WithRuntime(CoreRuntime.Core21))); + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80))); Assert.Equal(2, info.BenchmarksCases.Length); Assert.All(info.BenchmarksCases, benchmark => Assert.Equal(int.MaxValue, benchmark.Job.Run.MaxIterationCount)); diff --git a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs new file mode 100644 index 0000000000..05ce92b2cf --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs @@ -0,0 +1,359 @@ +using System; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Running; +using Xunit; +using System.Reflection; +using System.Reflection.Emit; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Tests.Running +{ + public class RunningEmptyBenchmarkTests + { + #region BenchmarkRunner Methods Overview + /* + * Available BenchmarkRunner.Run methods: + * 1. Generic Type: + * - BenchmarkRunner.Run(IConfig? config = null, string[]? args = null) + * 2. Type-based: + * - BenchmarkRunner.Run(Type type, IConfig? config = null, string[]? args = null) + * - BenchmarkRunner.Run(Type[] types, IConfig? config = null, string[]? args = null) + * - BenchmarkRunner.Run(Type type, MethodInfo[] methods, IConfig? config = null) + * 3. Assembly-based: + * - BenchmarkRunner.Run(Assembly assembly, IConfig? config = null, string[]? args = null) + * 4. BenchmarkRunInfo-based: + * - BenchmarkRunner.Run(BenchmarkRunInfo benchmarkRunInfo) + * - BenchmarkRunner.Run(BenchmarkRunInfo[] benchmarkRunInfos) + * 5. Deprecated methods: + * - BenchmarkRunner.RunUrl(string url, IConfig? config = null) + * - BenchmarkRunner.RunSource(string source, IConfig? config = null) + */ + #endregion + #region Generic Type Tests + /// + /// Tests for BenchmarkRunner.Run method + /// + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void GenericTypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(config, args); + if (args == null) + { + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + } + + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void GenericTypeWithBenchmarkAttribute_RunsSuccessfully(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(config, args); + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region Type-based Tests + /// + /// Tests for BenchmarkRunner.Run(Type) method + /// + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void TypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + + var summary = BenchmarkRunner.Run(typeof(EmptyBenchmark), config, args); + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypeWithBenchmarkAttribute_RunsSuccessfully(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(typeof(NotEmptyBenchmark), config, args); + Assert.False(summaries.HasCriticalValidationErrors); + Assert.DoesNotContain(summaries.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + + /// + /// Tests for BenchmarkRunner.Run(Type[]) method + /// + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypesWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(EmptyBenchmark), typeof(EmptyBenchmark2) }, config, args); + if (args != null) + { + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); + } + else + { + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark2))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); + } + + + } + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypesWithBenchmarkAttribute_RunsSuccessfully(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(NotEmptyBenchmark) }, config, args); + var summary = summaries[0]; + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region BenchmarkRunInfo Tests + /// + /// Tests for BenchmarkRunner.Run(BenchmarkRunInfo) method + /// + [Fact] + public void BenchmarkRunInfoWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute() + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), config)); + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + + [Fact] + public void BenchmarkRunInfoWithBenchmarkAttribute_RunsSuccessfully() + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), config)); + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + + /// + /// Tests for BenchmarkRunner.Run(BenchmarkRunInfo[]) method + /// + [Fact] + public void BenchmarkRunInfosWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute() + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { + BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), config), + BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark2), config) + }); + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark2))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); + } + + [Fact] + public void BenchmarkRunInfosWithBenchmarkAttribute_RunsSuccessfully() + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), config) }); + var summary = summaries[0]; + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region Mixed Types Tests + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void MixedTypes_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(EmptyBenchmark), typeof(NotEmptyBenchmark) }, config, args); + if (args != null) + { + Assert.Contains(GetExpandedValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + else + { + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + } + #endregion + #region Assembly Tests + // In this tests there is no config and logger because the logger is initiated at CreateCompositeLogger when the BenchmarkRunInfo[] is empty + // those cannot be inserted using config + [Theory] + + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void AssemblyWithoutBenchmarks_ThrowsValidationError_WhenNoBenchmarksFound(string[]? args) + { + + // Create a mock assembly with no benchmark types + var assemblyName = new AssemblyName("MockAssembly"); + var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run); + var moduleBuilder = assemblyBuilder.DefineDynamicModule("MockModule"); + // Create a simple type in the assembly (no benchmarks) + var typeBuilder = moduleBuilder.DefineType("MockType", TypeAttributes.Public); + typeBuilder.CreateType(); + + Summary[] summaries = null; + if (args != null) + { + GetConfigWithLogger(out var logger, out var config); + summaries = BenchmarkRunner.Run(assemblyBuilder, config, args); + Assert.Contains(GetAssemblylValidationError(assemblyBuilder), logger.GetLog()); + } + else + { + summaries = BenchmarkRunner.Run(assemblyBuilder, null, args); + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetGeneralValidationError()); + } + } + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void AssemblyWithBenchmarks_RunsSuccessfully_WhenBenchmarkAttributePresent(string[]? args) + { + // Skip test on .NET Framework 4.6.2 + if (RuntimeInformation.FrameworkDescription.Contains(".NET Framework 4")) + return; + + // Create a mock assembly with benchmark types + var assemblyName = new AssemblyName("MockAssemblyWithBenchmarks"); + var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run); + var moduleBuilder = assemblyBuilder.DefineDynamicModule("MockModule"); + + // Create a benchmark type + var benchmarkTypeBuilder = moduleBuilder.DefineType("MockBenchmark", TypeAttributes.Public); + var benchmarkMethod = benchmarkTypeBuilder.DefineMethod("Benchmark", MethodAttributes.Public, typeof(void), Type.EmptyTypes); + + // Generate method body + var ilGenerator = benchmarkMethod.GetILGenerator(); + ilGenerator.Emit(OpCodes.Ret); // Just return from the method + + var benchmarkAttributeCtor = typeof(BenchmarkAttribute).GetConstructor(new[] { typeof(int), typeof(string) }); + if (benchmarkAttributeCtor == null) + throw new InvalidOperationException("Could not find BenchmarkAttribute constructor"); + benchmarkMethod.SetCustomAttribute(new CustomAttributeBuilder( + benchmarkAttributeCtor, + new object[] { 0, "" })); + benchmarkTypeBuilder.CreateType(); + + Summary[] summaries = null; + if (args != null) + { + GetConfigWithLogger(out var logger, out var config); + summaries = BenchmarkRunner.Run(assemblyBuilder, config, args); + Assert.DoesNotContain(GetAssemblylValidationError(assemblyBuilder), logger.GetLog()); + } + else + { + summaries = BenchmarkRunner.Run(assemblyBuilder); + var summary = summaries[0]; + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetGeneralValidationError()); + } + } + #endregion + #region Helper Methods + private string GetValidationErrorForType(Type type) + { + return $"No [Benchmark] attribute found on '{type.Name}' benchmark case."; + } + + private string GetAssemblylValidationError(Assembly assembly) + { + return $"No [Benchmark] attribute found on '{assembly.GetName().Name}' assembly."; + } + + private string GetExpandedValidationErrorForType(Type type) + { + return $"Type {type} is invalid. Only public, non-generic (closed generic types with public parameterless ctors are supported), non-abstract, non-sealed, non-static types with public instance [Benchmark] method(s) are supported."; + } + + private string GetGeneralValidationError() + { + return $"No benchmarks were found."; + } + + private void GetConfigWithLogger(out AccumulationLogger logger, out ManualConfig manualConfig) + { + logger = new AccumulationLogger(); + manualConfig = ManualConfig.CreateEmpty() + .AddLogger(logger) + .AddColumnProvider(DefaultColumnProviders.Instance); + } + + #endregion + #region Test Classes + public class EmptyBenchmark + { + } + + public class EmptyBenchmark2 + { + } + + [SimpleJob(launchCount: 1, warmupCount: 1, iterationCount: 1, invocationCount: 1, id: "QuickJob")] + public class NotEmptyBenchmark + { + [Benchmark] + public void Benchmark() + { + var sum = 0; + for (int i = 0; i < 1; i++) + { + sum += i; + } + } + } + #endregion + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs b/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs index e49bd344e1..f6286720fd 100644 --- a/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs +++ b/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; +using BenchmarkDotNet.Detectors; using Xunit; namespace BenchmarkDotNet.Tests @@ -17,6 +18,10 @@ public class RuntimeVersionDetectionTests [InlineData(".NETCoreApp,Version=v3.0", RuntimeMoniker.NetCoreApp30, "netcoreapp3.0")] [InlineData(".NETCoreApp,Version=v3.1", RuntimeMoniker.NetCoreApp31, "netcoreapp3.1")] [InlineData(".NETCoreApp,Version=v5.0", RuntimeMoniker.Net50, "net5.0")] + [InlineData(".NETCoreApp,Version=v6.0", RuntimeMoniker.Net60, "net6.0")] + [InlineData(".NETCoreApp,Version=v7.0", RuntimeMoniker.Net70, "net7.0")] + [InlineData(".NETCoreApp,Version=v8.0", RuntimeMoniker.Net80, "net8.0")] + [InlineData(".NETCoreApp,Version=v9.0", RuntimeMoniker.Net90, "net9.0")] [InlineData(".NETCoreApp,Version=v123.0", RuntimeMoniker.NotRecognized, "net123.0")] public void TryGetVersionFromFrameworkNameHandlesValidInput(string frameworkName, RuntimeMoniker expectedTfm, string expectedMsBuildMoniker) { @@ -33,7 +38,7 @@ public void TryGetVersionFromFrameworkNameHandlesValidInput(string frameworkName [InlineData("")] [InlineData(".NETCoreApp,Version=v")] [InlineData("just wrong")] - public void TryGetVersionFromFrameworkNameHandlesInvalidInput(string frameworkName) + public void TryGetVersionFromFrameworkNameHandlesInvalidInput(string? frameworkName) { Assert.False(CoreRuntime.TryGetVersionFromFrameworkName(frameworkName, out _)); } @@ -60,7 +65,7 @@ public void TryGetVersionFromProductInfoHandlesValidInput(RuntimeMoniker expecte [InlineData("", "")] [InlineData("not", "ok")] [InlineData("Microsoft .NET Framework", "4.6.26614.01 @BuiltBy: dlab14-DDVSOWINAGE018 @Commit: a536e7eec55c538c94639cefe295aa672996bf9b")] // this is an actual output for 2.0 but it simply does not contain enough info - public void TryGetVersionFromProductInfoHandlesInvalidInput(string productName, string productVersion) + public void TryGetVersionFromProductInfoHandlesInvalidInput(string? productName, string? productVersion) { Assert.False(CoreRuntime.TryGetVersionFromProductInfo(productVersion, productName, out _)); } @@ -110,7 +115,7 @@ public void CurrentRuntimeIsProperlyRecognized() var runtime = RuntimeInformation.GetCurrentRuntime(); #if NETFRAMEWORK - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) Assert.True(runtime is ClrRuntime); else Assert.True(runtime is MonoRuntime); @@ -124,6 +129,8 @@ public void CurrentRuntimeIsProperlyRecognized() Assert.True(runtime is CoreRuntime coreRuntime && coreRuntime.RuntimeMoniker == RuntimeMoniker.NetCoreApp31); #elif NETCOREAPP5_0 Assert.True(runtime is CoreRuntime coreRuntime && coreRuntime.RuntimeMoniker == RuntimeMoniker.NetCoreApp50); +#elif NET8_0 + Assert.True(runtime is CoreRuntime coreRuntime && coreRuntime.RuntimeMoniker == RuntimeMoniker.Net80); #endif } } diff --git a/tests/BenchmarkDotNet.Tests/SimpleJsonTests.cs b/tests/BenchmarkDotNet.Tests/SimpleJsonTests.cs index 2bb57fe436..fd599aea5b 100644 --- a/tests/BenchmarkDotNet.Tests/SimpleJsonTests.cs +++ b/tests/BenchmarkDotNet.Tests/SimpleJsonTests.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; using System.Linq; using Xunit; -using JsonSerializer = SimpleJson.SimpleJson; +using JsonSerializer = SimpleJson.SimpleJsonSerializer; namespace BenchmarkDotNet.Tests { diff --git a/tests/BenchmarkDotNet.Tests/SizeUnitTests.cs b/tests/BenchmarkDotNet.Tests/SizeUnitTests.cs deleted file mode 100644 index e8cfbd5989..0000000000 --- a/tests/BenchmarkDotNet.Tests/SizeUnitTests.cs +++ /dev/null @@ -1,63 +0,0 @@ -using BenchmarkDotNet.Columns; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests -{ - public class SizeUnitTests - { - private readonly ITestOutputHelper output; - - public SizeUnitTests(ITestOutputHelper output) - { - this.output = output; - } - - - [Fact] - public void ConvertTest() - { - CheckConvertTwoWay(1024, SizeUnit.B, 1, SizeUnit.KB); - CheckConvertTwoWay(1024, SizeUnit.KB, 1, SizeUnit.MB); - CheckConvertTwoWay(1024, SizeUnit.MB, 1, SizeUnit.GB); - CheckConvertTwoWay(1024, SizeUnit.GB, 1, SizeUnit.TB); - CheckConvertTwoWay(1024L * 1024 * 1024 * 1024, SizeUnit.B, 1, SizeUnit.TB); - } - - [Theory] - [InlineData("0 B", 0)] - [InlineData("1 B", 1)] - [InlineData("10 B", 10)] - [InlineData("100 B", 100)] - [InlineData("1000 B", 1000)] - [InlineData("1023 B", 1023)] - [InlineData("1 KB", 1024)] - [InlineData("1 KB", 1025)] - [InlineData("1.07 KB", 1100)] - [InlineData("1.5 KB", 1024 + 512)] - [InlineData("10 KB", 10 * 1024)] - [InlineData("1023 KB", 1023 * 1024)] - [InlineData("1 MB", 1024 * 1024)] - [InlineData("1 GB", 1024 * 1024 * 1024)] - [InlineData("1 TB", 1024L * 1024 * 1024 * 1024)] - public void SizeUnitFormattingTest(string expected, long bytes) - { - Assert.Equal(expected, SizeValue.FromBytes(bytes).ToString(TestCultureInfo.Instance)); - } - - private void CheckConvertTwoWay(long value1, SizeUnit unit1, long value2, SizeUnit unit2) - { - CheckConvertOneWay(value1, unit1, value2, unit2); - CheckConvertOneWay(value2, unit2, value1, unit1); - } - - private void CheckConvertOneWay(long value1, SizeUnit unit1, long value2, SizeUnit unit2) - { - double convertedValue2 = SizeUnit.Convert(value1, unit1, unit2); - output.WriteLine($"Expected: {value1} {unit1.Name} = {value2} {unit2.Name}"); - output.WriteLine($"Actual: {value1} {unit1.Name} = {convertedValue2} {unit2.Name}"); - output.WriteLine(""); - Assert.Equal(value2, convertedValue2, 4); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/SourceCodeHelperTests.cs b/tests/BenchmarkDotNet.Tests/SourceCodeHelperTests.cs index 846d59faa9..6e8e504a60 100644 --- a/tests/BenchmarkDotNet.Tests/SourceCodeHelperTests.cs +++ b/tests/BenchmarkDotNet.Tests/SourceCodeHelperTests.cs @@ -24,7 +24,7 @@ public class SourceCodeHelperTests [InlineData(0.123f, "0.123f")] [InlineData(0.123d, "0.123d")] [InlineData(BindingFlags.Public, "(System.Reflection.BindingFlags)(16)")] - public void ToSourceCodeSimpleTest(object original, string expected) + public void ToSourceCodeSimpleTest(object? original, string expected) { string actual = SourceCodeHelper.ToSourceCode(original); output.WriteLine("ORIGINAL : " + original + " (" + original?.GetType() + ")"); diff --git a/tests/BenchmarkDotNet.Tests/StringExtensionsTests.cs b/tests/BenchmarkDotNet.Tests/StringExtensionsTests.cs index 2dbaf0f6bc..a03f308ad6 100644 --- a/tests/BenchmarkDotNet.Tests/StringExtensionsTests.cs +++ b/tests/BenchmarkDotNet.Tests/StringExtensionsTests.cs @@ -43,7 +43,7 @@ public void HtmlEncodeCharacters() [InlineData(" a ", " a")] [InlineData(" a ", " a")] [InlineData(" \r\n a \r\n", " a")] - public void AppendArgumentMakesSureOneSpaceBeforeStringArgument(string input, string expectedOutput) + public void AppendArgumentMakesSureOneSpaceBeforeStringArgument(string? input, string expectedOutput) { var stringBuilder = new StringBuilder(); var result = stringBuilder.AppendArgument(input).ToString(); @@ -58,7 +58,7 @@ public void AppendArgumentMakesSureOneSpaceBeforeStringArgument(string input, st [InlineData("http://test.com/ ", " http://test.com/")] [InlineData(" http://test.com/ ", " http://test.com/")] [InlineData("\r\n http://test.com/ \r\n", " http://test.com/")] - public void AppendArgumentMakesSureOneSpaceBeforeObjectArgument(string input, string expectedOutput) + public void AppendArgumentMakesSureOneSpaceBeforeObjectArgument(string? input, string expectedOutput) { Uri uri = input != null ? new Uri(input) : null; // Use Uri for our object type since that is what is used in code var stringBuilder = new StringBuilder(); diff --git a/tests/BenchmarkDotNet.Tests/SummaryStyleTests.cs b/tests/BenchmarkDotNet.Tests/SummaryStyleTests.cs index e627bf7db9..e75729d584 100644 --- a/tests/BenchmarkDotNet.Tests/SummaryStyleTests.cs +++ b/tests/BenchmarkDotNet.Tests/SummaryStyleTests.cs @@ -3,6 +3,7 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Reports; using Perfolizer.Horology; +using Perfolizer.Metrology; using Xunit; namespace BenchmarkDotNet.Tests diff --git a/tests/BenchmarkDotNet.Tests/TimeUnitTests.cs b/tests/BenchmarkDotNet.Tests/TimeUnitTests.cs deleted file mode 100644 index 6f7a26c9b0..0000000000 --- a/tests/BenchmarkDotNet.Tests/TimeUnitTests.cs +++ /dev/null @@ -1,63 +0,0 @@ -using Perfolizer.Horology; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests -{ - public class TimeUnitTests - { - private readonly ITestOutputHelper output; - - public TimeUnitTests(ITestOutputHelper output) - { - this.output = output; - } - - [Fact] - public void ConvertTest() - { - CheckConvertTwoWay(1000, TimeUnit.Nanosecond, 1, TimeUnit.Microsecond); - CheckConvertTwoWay(1000, TimeUnit.Microsecond, 1, TimeUnit.Millisecond); - CheckConvertTwoWay(1000, TimeUnit.Millisecond, 1, TimeUnit.Second); - CheckConvertTwoWay(60, TimeUnit.Second, 1, TimeUnit.Minute); - CheckConvertTwoWay(60, TimeUnit.Minute, 1, TimeUnit.Hour); - CheckConvertTwoWay(24, TimeUnit.Hour, 1, TimeUnit.Day); - } - - [Fact] - public void GetBestTimeUnitTest() - { - CheckGetBestTimeUnit(TimeUnit.Nanosecond); - CheckGetBestTimeUnit(TimeUnit.Nanosecond, 1.0); - CheckGetBestTimeUnit(TimeUnit.Nanosecond, 100.0); - CheckGetBestTimeUnit(TimeUnit.Microsecond, 1.0 * 1000); - CheckGetBestTimeUnit(TimeUnit.Microsecond, 100.0 * 1000); - CheckGetBestTimeUnit(TimeUnit.Millisecond, 1.0 * 1000 * 1000); - CheckGetBestTimeUnit(TimeUnit.Millisecond, 100.0 * 1000 * 1000); - CheckGetBestTimeUnit(TimeUnit.Second, 1.0 * 1000 * 1000 * 1000); - CheckGetBestTimeUnit(TimeUnit.Second, 100.0 * 1000 * 1000 * 1000); - CheckGetBestTimeUnit(TimeUnit.Day, double.MaxValue); - } - - private void CheckGetBestTimeUnit(TimeUnit timeUnit, params double[] values) - { - output.WriteLine($"Best TimeUnit for ({string.Join(";", values)})ns is {timeUnit.Description}"); - Assert.Equal(timeUnit.Name, TimeUnit.GetBestTimeUnit(values).Name); - } - - private void CheckConvertTwoWay(double value1, TimeUnit unit1, double value2, TimeUnit unit2) - { - CheckConvertOneWay(value1, unit1, value2, unit2); - CheckConvertOneWay(value2, unit2, value1, unit1); - } - - private void CheckConvertOneWay(double value1, TimeUnit unit1, double value2, TimeUnit unit2) - { - double convertedValue2 = TimeUnit.Convert(value1, unit1, unit2); - output.WriteLine($"Expected: {value1} {unit1.Name} = {value2} {unit2.Name}"); - output.WriteLine($"Actual: {value1} {unit1.Name} = {convertedValue2} {unit2.Name}"); - output.WriteLine(""); - Assert.Equal(value2, convertedValue2, 4); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/TypeFilterTests.cs b/tests/BenchmarkDotNet.Tests/TypeFilterTests.cs index 9ec2c63ffc..b1bcdb1dcd 100644 --- a/tests/BenchmarkDotNet.Tests/TypeFilterTests.cs +++ b/tests/BenchmarkDotNet.Tests/TypeFilterTests.cs @@ -213,7 +213,7 @@ public void GenericTypesCanBeFilteredByDisplayName() Assert.Contains("SomeGeneric.Create", benchmarks); } - private HashSet Filter(Type[] types, string[] args, ILogger logger = null) + private HashSet Filter(Type[] types, string[] args, ILogger? logger = null) { var nonNullLogger = logger ?? new OutputLogger(Output); diff --git a/tests/BenchmarkDotNet.Tests/Validators/CompilationValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/CompilationValidatorTests.cs index 18d947acea..683e258bbe 100644 --- a/tests/BenchmarkDotNet.Tests/Validators/CompilationValidatorTests.cs +++ b/tests/BenchmarkDotNet.Tests/Validators/CompilationValidatorTests.cs @@ -7,6 +7,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Validators; using Xunit; +using System.Reflection; namespace BenchmarkDotNet.Tests.Validators { @@ -22,9 +23,7 @@ public void BenchmarkedMethodNameMustNotContainWhitespaces() new[] { BenchmarkCase.Create( - new Descriptor( - typeof(CompilationValidatorTests), - method.Method), + new Descriptor(typeof(CompilationValidatorTests), method.Method), Job.Dry, null, config @@ -48,9 +47,7 @@ public void BenchmarkedMethodNameMustNotUseCsharpKeywords() new[] { BenchmarkCase.Create( - new Descriptor( - typeof(CompilationValidatorTests), - method.Method), + new Descriptor(typeof(CompilationValidatorTests), method.Method), Job.Dry, null, config) @@ -63,6 +60,29 @@ public void BenchmarkedMethodNameMustNotUseCsharpKeywords() "Benchmarked method `typeof` contains illegal character(s) or uses C# keyword. Please use `[]` to set custom display name.")); } + [Theory] + /* BenchmarkDotNet can only benchmark public unsealed classes*/ + [InlineData(typeof(BenchMarkPublicClass), false)] + [InlineData(typeof(BenchMarkPublicClass.PublicNestedClass), false)] + [InlineData(typeof(SealedClass.PublicNestedClass), false)] + [InlineData(typeof(OuterClass.PublicNestedClass), true)] + [InlineData(typeof(SealedClass), true)] + [InlineData(typeof(MyPrivateClass), true)] + [InlineData(typeof(MyPublicProtectedClass), true)] + [InlineData(typeof(MyPrivateProtectedClass), true)] + [InlineData(typeof(MyProtectedInternalClass), true)] + [InlineData(typeof(MyInternalClass), true)] + [InlineData(typeof(OuterClass), true)] + [InlineData(typeof(OuterClass.InternalNestedClass), true)] + [InlineData(typeof(BenchMarkPublicClass.InternalNestedClass), true)] + /* Generics Remaining */ + public void Benchmark_Class_Modifers_Must_Be_Public(Type type, bool hasErrors) + { + var validationErrors = CompilationValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(type)); + + Assert.Equal(hasErrors, validationErrors.Any()); + } + [Theory] [InlineData(typeof(BenchmarkClassWithStaticMethod), true)] [InlineData(typeof(BenchmarkClass), false)] @@ -119,7 +139,17 @@ protected internal class ProtectedInternalClass { protected internal class ProtectedInternalNestedClass { } } - } + + private class MyPrivateClass{ [Benchmark] public void PublicMethod(){} } + + protected class MyPublicProtectedClass{ [Benchmark] public void PublicMethod(){} } + + private protected class MyPrivateProtectedClass{ [Benchmark] public void PublicMethod(){} } + + internal class MyInternalClass{ [Benchmark] public void PublicMethod(){} } + + protected internal class MyProtectedInternalClass{ [Benchmark] public void PublicMethod() { } } + } public class BenchmarkClassWithStaticMethod { @@ -142,4 +172,29 @@ internal class InternalClass { internal class InternalNestedClass { } } + + public sealed class SealedClass + { + [Benchmark] public void PublicMethod() { } + + public class PublicNestedClass { [Benchmark] public void PublicMethod() { } } + } + + internal class OuterClass + { + [Benchmark] public void PublicMethod(){} + + internal class InternalNestedClass { [Benchmark] public void PublicMethod() { } } + + public class PublicNestedClass { [Benchmark] public void PublicMethod() { } } + } + + public class BenchMarkPublicClass + { + [Benchmark] public void PublicMethod(){} + + public class PublicNestedClass { [Benchmark] public void PublicMethod() { } } + + internal class InternalNestedClass { [Benchmark] public void PublicMethod() { } } + } } diff --git a/tests/BenchmarkDotNet.Tests/Validators/ConfigCompatibilityValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/ConfigCompatibilityValidatorTests.cs deleted file mode 100644 index fc73793980..0000000000 --- a/tests/BenchmarkDotNet.Tests/Validators/ConfigCompatibilityValidatorTests.cs +++ /dev/null @@ -1,180 +0,0 @@ -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Columns; -using BenchmarkDotNet.Order; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Tests.Loggers; -using BenchmarkDotNet.Validators; -using System.Linq; -using BenchmarkDotNet.Portability; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Validators -{ - public class ConfigCompatibilityValidatorTests - { - private ITestOutputHelper Output { get; } - - public ConfigCompatibilityValidatorTests(ITestOutputHelper output) => Output = output; - - [Fact] - public void RunningBenchmarksWithIncompatibleConfigsMustFailWithCriticalError() - { - if (!KnownIssue.Issue2299.IsFixed && RuntimeInformation.IsMono && RuntimeInformation.IsLinux()) - { - Output.WriteLine(KnownIssue.Issue2299.IgnoreMessage); - return; - } - - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); - var summary = - BenchmarkSwitcher - .FromTypes(new[] { typeof(BenchmarkClassWithExtraOrderer1), typeof(BenchmarkClassWithExtraOrderer2) }) - .RunAllJoined(config); - Assert.True(summary.HasCriticalValidationErrors); - Assert.Contains("You use JoinSummary options, but provided configurations cannot be joined", logger.GetLog()); - Assert.Contains("Orderer", logger.GetLog()); - } - - [Fact] - public void JoinedBenchmarksMustNotHaveDifferentExtraOrderers() - { - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithExtraOrderer1)), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithExtraOrderer2)) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.NotEmpty(validationErrors); - Assert.StartsWith("You use JoinSummary options, but provided configurations cannot be joined", validationErrors.Single().Message); - Assert.Contains("Orderer", validationErrors.Single().Message); - } - - [Fact] - public void JoinedBenchmarksMayHaveOneExtraOrderer() - { - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithExtraOrderer1)), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults1)) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.Empty(validationErrors); - } - - [Fact] - public void JoinedBenchmarksMayHaveDefaultOrderers() - { - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults1)), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults2)) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.Empty(validationErrors); - } - - [Fact] - public void JoinedBenchmarksMustNotHaveDifferentExtraSummaryStyles() - { - // Note: RatioStyle.Value would be the same as SummaryStyle.Default (SummaryStyle implements IEquatable). - ManualConfig config1 = DefaultConfig.Instance.WithSummaryStyle( - SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend)); - ManualConfig config2 = DefaultConfig.Instance.WithSummaryStyle( - SummaryStyle.Default.WithRatioStyle(RatioStyle.Percentage)); - - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults1), config1), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults2), config2) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.NotEmpty(validationErrors); - Assert.StartsWith("You use JoinSummary options, but provided configurations cannot be joined", validationErrors.Single().Message); - Assert.Contains("SummaryStyle", validationErrors.Single().Message); - } - - [Fact] - public void JoinedBenchmarksMayHaveOneExtraSummaryStyle() - { - ManualConfig config = DefaultConfig.Instance.WithSummaryStyle( - SummaryStyle.Default.WithZeroMetricValuesInContent()); - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults1), null), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults2), config) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.Empty(validationErrors); - } - - [Orderer(SummaryOrderPolicy.Method)] - public class BenchmarkClassWithExtraOrderer1 - { - [Benchmark] - public void Foo() { } - } - - [Orderer(SummaryOrderPolicy.Method)] - public class BenchmarkClassWithExtraOrderer2 - { - [Benchmark] - public void Bar() { } - } - - public class BenchmarkClassWithDefaults1 - { - [Benchmark] - public void Baz() { } - } - - public class BenchmarkClassWithDefaults2 - { - [Benchmark] - public void Buzz() { } - } - } -} diff --git a/tests/BenchmarkDotNet.Tests/Validators/ExecutionValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/ExecutionValidatorTests.cs index 343bdc53dc..fece3b3939 100644 --- a/tests/BenchmarkDotNet.Tests/Validators/ExecutionValidatorTests.cs +++ b/tests/BenchmarkDotNet.Tests/Validators/ExecutionValidatorTests.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Threading.Tasks; +using System.Threading.Tasks.Sources; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; @@ -563,5 +564,82 @@ public async ValueTask GlobalCleanup() [Benchmark] public void NonThrowing() { } } + + private class ValueTaskSource : IValueTaskSource, IValueTaskSource + { + private ManualResetValueTaskSourceCore _core; + + T IValueTaskSource.GetResult(short token) => _core.GetResult(token); + void IValueTaskSource.GetResult(short token) => _core.GetResult(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _core.GetStatus(token); + ValueTaskSourceStatus IValueTaskSource.GetStatus(short token) => _core.GetStatus(token); + void IValueTaskSource.OnCompleted(Action continuation, object state, short token, ValueTaskSourceOnCompletedFlags flags) => _core.OnCompleted(continuation, state, token, flags); + void IValueTaskSource.OnCompleted(Action continuation, object state, short token, ValueTaskSourceOnCompletedFlags flags) => _core.OnCompleted(continuation, state, token, flags); + public void Reset() => _core.Reset(); + public short Token => _core.Version; + public void SetResult(T result) => _core.SetResult(result); + } + + [Fact] + public void AsyncValueTaskBackedByIValueTaskSourceIsAwaitedProperly() + { + var validationErrors = ExecutionValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(typeof(AsyncValueTaskSource))).ToList(); + + Assert.True(AsyncValueTaskSource.WasCalled); + Assert.Empty(validationErrors); + } + + public class AsyncValueTaskSource + { + private readonly ValueTaskSource valueTaskSource = new (); + + public static bool WasCalled; + + [GlobalSetup] + public ValueTask GlobalSetup() + { + valueTaskSource.Reset(); + Task.Delay(1).ContinueWith(_ => + { + WasCalled = true; + valueTaskSource.SetResult(true); + }); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + + [Benchmark] + public void NonThrowing() { } + } + + [Fact] + public void AsyncGenericValueTaskBackedByIValueTaskSourceIsAwaitedProperly() + { + var validationErrors = ExecutionValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(typeof(AsyncGenericValueTaskSource))).ToList(); + + Assert.True(AsyncGenericValueTaskSource.WasCalled); + Assert.Empty(validationErrors); + } + + public class AsyncGenericValueTaskSource + { + private readonly ValueTaskSource valueTaskSource = new (); + + public static bool WasCalled; + + [GlobalSetup] + public ValueTask GlobalSetup() + { + valueTaskSource.Reset(); + Task.Delay(1).ContinueWith(_ => + { + WasCalled = true; + valueTaskSource.SetResult(1); + }); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + + [Benchmark] + public void NonThrowing() { } + } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs index cb2b1a3878..79bfb5a1ae 100644 --- a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs +++ b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs @@ -8,5 +8,5 @@ public enum EnvRequirement FullFrameworkOnly, NonFullFramework, DotNetCoreOnly, - DotNetCore30Only + NeedsPrivilegedProcess } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs index cbe4ac930e..22f131455d 100644 --- a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs +++ b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Runtime.InteropServices; +using System.Security.Principal; using BenchmarkDotNet.Jobs; using JetBrains.Annotations; using BdnRuntimeInformation = BenchmarkDotNet.Portability.RuntimeInformation; @@ -9,11 +10,9 @@ namespace BenchmarkDotNet.Tests.XUnit; public static class EnvRequirementChecker { - [CanBeNull] - public static string GetSkip(params EnvRequirement[] requirements) => requirements.Select(GetSkip).FirstOrDefault(skip => skip != null); + public static string? GetSkip(params EnvRequirement[] requirements) => requirements.Select(GetSkip).FirstOrDefault(skip => skip != null); - [CanBeNull] - internal static string GetSkip(EnvRequirement requirement) => requirement switch + internal static string? GetSkip(EnvRequirement requirement) => requirement switch { EnvRequirement.WindowsOnly => RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? null : "Windows-only test", EnvRequirement.NonWindows => !RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? null : "Non-Windows test", @@ -21,9 +20,19 @@ public static class EnvRequirementChecker EnvRequirement.FullFrameworkOnly => BdnRuntimeInformation.IsFullFramework ? null : "Full .NET Framework-only test", EnvRequirement.NonFullFramework => !BdnRuntimeInformation.IsFullFramework ? null : "Non-Full .NET Framework test", EnvRequirement.DotNetCoreOnly => BdnRuntimeInformation.IsNetCore ? null : ".NET/.NET Core-only test", - EnvRequirement.DotNetCore30Only => IsRuntime(RuntimeMoniker.NetCoreApp30) ? null : ".NET Core 3.0-only test", + EnvRequirement.NeedsPrivilegedProcess => IsPrivilegedProcess() ? null : "Needs authorization to perform security-relevant functions", _ => throw new ArgumentOutOfRangeException(nameof(requirement), requirement, "Unknown value") }; + private static bool IsPrivilegedProcess() + { +#if NET462 + using WindowsIdentity currentUser = WindowsIdentity.GetCurrent(); + return new WindowsPrincipal(currentUser).IsInRole(WindowsBuiltInRole.Administrator); +#else + return Environment.IsPrivilegedProcess; +#endif + } + private static bool IsRuntime(RuntimeMoniker moniker) => BdnRuntimeInformation.GetCurrentRuntime().RuntimeMoniker == moniker; } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/dotMemory/DotMemoryTests.cs b/tests/BenchmarkDotNet.Tests/dotMemory/DotMemoryTests.cs new file mode 100644 index 0000000000..d3aec8dd96 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/dotMemory/DotMemoryTests.cs @@ -0,0 +1,17 @@ +using System; +using BenchmarkDotNet.Diagnostics.dotMemory; +using BenchmarkDotNet.Jobs; +using Xunit; + +namespace BenchmarkDotNet.Tests.dotMemory; + +public class DotMemoryTests +{ + [Fact] + public void AllRuntimeMonikerAreKnown() + { + var diagnoser = new DotMemoryDiagnoser(); + foreach (RuntimeMoniker moniker in Enum.GetValues(typeof(RuntimeMoniker))) + diagnoser.IsSupported(moniker); // Just check that it doesn't throw exceptions + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs b/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs index a71cae2aef..751412fd55 100644 --- a/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs +++ b/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs @@ -3,15 +3,15 @@ using BenchmarkDotNet.Jobs; using Xunit; -namespace BenchmarkDotNet.Tests.dotTrace +namespace BenchmarkDotNet.Tests.dotTrace; + +public class DotTraceTests { - public class DotTraceTests + [Fact] + public void AllRuntimeMonikerAreKnown() { - [Fact] - public void AllRuntimeMonikerAreKnown() - { - foreach (RuntimeMoniker moniker in Enum.GetValues(typeof(RuntimeMoniker))) - DotTraceDiagnoser.IsSupported(moniker); // Just check that it doesn't throw exceptions - } + var diagnoser = new DotTraceDiagnoser(); + foreach (RuntimeMoniker moniker in Enum.GetValues(typeof(RuntimeMoniker))) + diagnoser.IsSupported(moniker); // Just check that it doesn't throw exceptions } } \ No newline at end of file diff --git a/tests/runCoreTests.cmd b/tests/runCoreTests.cmd index 93c5c0f258..137ea952d9 100644 --- a/tests/runCoreTests.cmd +++ b/tests/runCoreTests.cmd @@ -12,24 +12,24 @@ if NOT %ERRORLEVEL% == 0 ( ) echo ----------------------------- -echo Running Core 2.1 Unit tests +echo Running Core 8.0 Unit tests echo ----------------------------- -call dotnet test "BenchmarkDotNet.Tests\BenchmarkDotNet.Tests.csproj" --configuration Release --framework netcoreapp2.1 +call dotnet test "BenchmarkDotNet.Tests\BenchmarkDotNet.Tests.csproj" --configuration Release --framework net8.0 if NOT %ERRORLEVEL% == 0 ( - echo CORE 2.1 Unit tests has failed + echo Core 8.0 Unit tests has failed goto end ) echo ----------------------------- -echo Running Core 2.1 Integration tests +echo Running 8.0 Integration tests echo ----------------------------- -call dotnet test "BenchmarkDotNet.IntegrationTests\BenchmarkDotNet.IntegrationTests.csproj" --configuration Release --framework netcoreapp2.1 +call dotnet test "BenchmarkDotNet.IntegrationTests\BenchmarkDotNet.IntegrationTests.csproj" --configuration Release --framework net8.0 if NOT %ERRORLEVEL% == 0 ( - echo CORE 2.1 Integration tests has failed + echo Core 8.0 Integration tests has failed goto end ) diff --git a/tests/runCoreTests.sh b/tests/runCoreTests.sh index 4e3ad59e6a..1b5bb28fa3 100755 --- a/tests/runCoreTests.sh +++ b/tests/runCoreTests.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -dotnet test BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj --configuration Release --framework netcoreapp2.1 2>&1 | tee tests.log -dotnet test BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj --configuration Release --framework netcoreapp2.1 2>&1 | tee integration-tests.log +dotnet test BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj --configuration Release --framework net8.0 2>&1 | tee tests.log +dotnet test BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj --configuration Release --framework net8.0 2>&1 | tee integration-tests.log