diff --git a/.github/workflows/generate-changelog.yaml b/.github/workflows/generate-changelog.yaml index 71592e4d46..a0538e4eac 100644 --- a/.github/workflows/generate-changelog.yaml +++ b/.github/workflows/generate-changelog.yaml @@ -19,8 +19,8 @@ jobs: with: ref: master - - name: Download changelog - run: ./build.cmd docs-update --depth 1 --preview + - name: Fetch changelog + run: ./build.cmd docs-fetch --depth 1 --preview --force-clone env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -28,8 +28,8 @@ jobs: uses: JamesIves/github-pages-deploy-action@3.7.1 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: docs-changelog-details - FOLDER: docs/_changelog/details + BRANCH: docs-changelog + FOLDER: docs/_changelog GIT_CONFIG_NAME: Andrey Akinshin GIT_CONFIG_EMAIL: andrey.akinshin@gmail.com - CLEAN: true \ No newline at end of file + CLEAN: true diff --git a/.github/workflows/generate-gh-pages.yaml b/.github/workflows/generate-gh-pages.yaml index cb01860995..8983ed711f 100644 --- a/.github/workflows/generate-gh-pages.yaml +++ b/.github/workflows/generate-gh-pages.yaml @@ -22,8 +22,8 @@ jobs: - name: Build BenchmarkDotNet run: ./build.cmd build - - name: Download changelog - run: ./build.cmd docs-update --depth 1 --preview + - name: Fetch changelog + run: ./build.cmd docs-fetch --depth 1 --preview --force-clone env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -31,7 +31,7 @@ jobs: run: ./build.cmd docs-build - name: Upload Artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: name: site path: docs/_site @@ -48,9 +48,13 @@ jobs: ref: docs-stable - name: Download Artifacts - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v4 with: name: site + path: site + + - name: Print file tree + run: tree $GITHUB_WORKSPACE - name: Deploy documentation uses: JamesIves/github-pages-deploy-action@3.7.1 diff --git a/.github/workflows/publish-nightly.yaml b/.github/workflows/publish-nightly.yaml index 3a3dc0bc9b..01f439f41e 100644 --- a/.github/workflows/publish-nightly.yaml +++ b/.github/workflows/publish-nightly.yaml @@ -5,19 +5,20 @@ on: push: branches: - master + workflow_dispatch: jobs: publish: runs-on: ubuntu-latest if: ${{ github.repository == 'dotnet/BenchmarkDotNet' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set date run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV - name: Pack run: ./build.cmd pack /p:VersionSuffix=nightly.$DATE.$GITHUB_RUN_NUMBER - name: Upload nupkg to artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: nupkgs path: "**/*.*nupkg" diff --git a/.github/workflows/report-test-results.yaml b/.github/workflows/report-test-results.yaml index 005465b329..f1c627295e 100644 --- a/.github/workflows/report-test-results.yaml +++ b/.github/workflows/report-test-results.yaml @@ -11,13 +11,26 @@ jobs: report: runs-on: ubuntu-latest permissions: write-all + if: ${{ github.event.workflow_run.conclusion != 'cancelled' }} steps: + # Cleanup Old Files + - name: Cleanup Old Files + run: rm -rf $GITHUB_WORKSPACE/*.trx + + # Download the Latest Artifacts with Unique Name - name: Download Artifacts - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v6 with: - workflow: ${{ github.event.workflow_run.workflow_id }} + run_id: ${{ github.event.workflow_run.id }} + + # Display the Structure of Downloaded Files - name: Display structure of downloaded files run: ls -R + + # Display the Contents of .trx Files + - name: Display .trx file contents + run: cat **/*.trx || echo "No .trx files found" + - name: Report tests results uses: AndreyAkinshin/test-reporter@0e2c48ebec2007001dd77dd4bcbcd450b96d5a38 with: diff --git a/.github/workflows/run-tests-selected.yaml b/.github/workflows/run-tests-selected.yaml new file mode 100644 index 0000000000..ddbcc74e09 --- /dev/null +++ b/.github/workflows/run-tests-selected.yaml @@ -0,0 +1,89 @@ +name: run-tests-selected +run-name: Run selected tests (${{ inputs.runs_on }} --framework ${{ inputs.framework}} --filter ${{ inputs.filter }}) + +on: + workflow_dispatch: + inputs: + runs_on: + type: choice + description: GitHub Actions runner image name + required: true + default: ubuntu-latest + options: + - windows-latest + - ubuntu-latest + - macos-latest + - windows-11-arm + - ubuntu-24.04-arm + - macos-13 + project: + type: string + description: Specify test project path + required: true + default: tests/BenchmarkDotNet.IntegrationTests + options: + - tests/BenchmarkDotNet.Tests + - tests/BenchmarkDotNet.IntegrationTests + - tests/BenchmarkDotNet.IntegrationTests.ManualRunning + framework: + type: choice + description: Specify target framework + required: true + options: + - net8.0 + - net462 + filter: + type: string + description: Test filter text (It's used for `dotnet test --filter`) Use default value when running all tests + required: true + default: "BenchmarkDotNet" + iteration_count: + type: number + description: Count of test loop (It's expected to be used for flaky tests) + required: true + default: 1 + +jobs: + test: + name: test (${{ inputs.runs_on }} --framework ${{ inputs.framework}} --filter ${{ inputs.filter }}) + runs-on: ${{ inputs.runs_on }} + timeout-minutes: 60 # Explicitly set timeout. When wrong input parameter is passed. It may continue to run until it times out (Default:360 minutes)) + steps: + - uses: actions/checkout@v4 + + # Setup + - name: Setup + run: | + mkdir artifacts + + # Build + - name: Run build + working-directory: ${{ github.event.inputs.project }} + run: | + dotnet build -c Release --framework ${{ inputs.framework }} -tl:off + + # Test + - name: Run tests + shell: pwsh + working-directory: ${{ github.event.inputs.project }} + run: | + $PSNativeCommandUseErrorActionPreference = $true + $iterationCount = ${{ inputs.iteration_count }} + + foreach($i in 1..$iterationCount) { + Write-Output ('##[group]Executing Iteration: {0}/${{ inputs.iteration_count }}' -f $i) + + dotnet test -c Release --framework ${{ inputs.framework }} --filter ${{ inputs.filter }} -tl:off --no-build --logger "console;verbosity=normal" + + Write-Output '##[endgroup]' + } + + # Upload artifact files that are located at `$(GITHUB_WORKSPACE)/artifacts` directory + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: results + if-no-files-found: ignore + path: | + artifacts/**/* diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 698b429514..9d3f25a9e8 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -18,17 +18,19 @@ jobs: run: Set-MpPreference -DisableRealtimeMonitoring $true shell: powershell - uses: actions/checkout@v4 + # Build and Test - name: Run task 'build' shell: cmd run: ./build.cmd build - name: Run task 'in-tests-core' shell: cmd run: ./build.cmd in-tests-core -e + # Upload Artifacts with Unique Name - name: Upload test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: - name: test-windows-core-trx + name: test-windows-core-trx-${{ github.run_id }} path: "**/*.trx" test-windows-full: @@ -38,25 +40,28 @@ jobs: run: Set-MpPreference -DisableRealtimeMonitoring $true shell: powershell - uses: actions/checkout@v4 + # Build and Test - name: Run task 'build' shell: cmd run: ./build.cmd build - name: Run task 'in-tests-full' shell: cmd run: ./build.cmd in-tests-full -e + # Upload Artifacts with Unique Name - name: Upload test results uses: actions/upload-artifact@v4 if: always() with: - name: test-windows-full-trx + name: test-windows-full-trx-${{ github.run_id }} path: "**/*.trx" test-linux: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + # Set up the environment - name: Set up Clang - uses: egor-tensin/setup-clang@v1 + uses: egor-tensin/setup-clang@ef434b41eb33a70396fb336b1bae39c76d740c3d # v1.4 with: version: latest platform: x64 @@ -65,47 +70,61 @@ jobs: - name: Set up node uses: actions/setup-node@v4 with: - node-version: "20" + node-version: "22" - name: Set up v8 run: npm install jsvu -g && jsvu --os=linux64 --engines=v8 && echo "$HOME/.jsvu/bin" >> $GITHUB_PATH - name: Install wasm-tools workload run: ./build.cmd install-wasm-tools + # Build and Test - name: Run task 'build' run: ./build.cmd build - name: Run task 'unit-tests' run: ./build.cmd unit-tests -e - name: Run task 'in-tests-core' run: ./build.cmd in-tests-core -e + # Upload Artifacts with Unique Name - name: Upload test results uses: actions/upload-artifact@v4 if: always() with: - name: test-linux-trx + name: test-linux-trx-${{ github.run_id }} path: "**/*.trx" test-macos: - runs-on: macos-13 + name: test-macos (${{ matrix.os.arch }}) + runs-on: ${{ matrix.os.runs-on }} + strategy: + matrix: + os: + - runs-on: 'macos-latest' + jsvu-os: 'mac64arm' + arch: 'arm64' + - runs-on: 'macos-13' + jsvu-os: 'mac64' + arch: 'x64' steps: - uses: actions/checkout@v4 - name: Set up node uses: actions/setup-node@v4 with: - node-version: "20" + node-version: "22" - name: Set up v8 - run: npm install jsvu -g && jsvu --os=mac64 --engines=v8 && echo "$HOME/.jsvu/bin" >> $GITHUB_PATH + run: npm install jsvu -g && jsvu --os=${{ matrix.os.jsvu-os }} --engines=v8 && echo "$HOME/.jsvu/bin" >> $GITHUB_PATH - name: Install wasm-tools workload run: ./build.cmd install-wasm-tools + # Build and Test - name: Run task 'build' run: ./build.cmd build - name: Run task 'unit-tests' run: ./build.cmd unit-tests -e - name: Run task 'in-tests-core' run: ./build.cmd in-tests-core -e + # Upload Artifacts with Unique Name - name: Upload test results uses: actions/upload-artifact@v4 if: always() with: - name: test-macos-trx + name: test-macos(${{ matrix.os.arch }})-trx-${{ github.run_id }} path: "**/*.trx" test-pack: @@ -129,9 +148,9 @@ jobs: - uses: actions/setup-node@v4 name: Setup node with: - node-version: "18" + node-version: "22" - name: Install cSpell - run: npm install -g cspell@8.0.0 + run: npm install -g cspell@9.0.2 - name: Copy cSpell config run: cp ./build/cSpell.json ./cSpell.json - name: Run cSpell diff --git a/BenchmarkDotNet.sln.DotSettings b/BenchmarkDotNet.sln.DotSettings index a2d44733f3..53f5cf0a60 100644 --- a/BenchmarkDotNet.sln.DotSettings +++ b/BenchmarkDotNet.sln.DotSettings @@ -166,6 +166,7 @@ True True True + True True True True @@ -177,6 +178,7 @@ True True True + True True True True diff --git a/LICENSE.md b/LICENSE.md index 039248a340..6272f97306 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ ### The MIT License -Copyright (c) 2013–2024 .NET Foundation and contributors +Copyright (c) 2013–2025 .NET Foundation and contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/NuGet.Config b/NuGet.Config index 855812b5a5..7507704b8b 100644 --- a/NuGet.Config +++ b/NuGet.Config @@ -13,5 +13,6 @@ + diff --git a/README.md b/README.md index 3022e0ebb3..1995deff62 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ It's no harder than writing unit tests! Under the hood, it performs a lot of [magic](#automation) that guarantees [reliable and precise](#reliability) results thanks to the [perfolizer](https://github.com/AndreyAkinshin/perfolizer) statistical engine. BenchmarkDotNet protects you from popular benchmarking mistakes and warns you if something is wrong with your benchmark design or obtained measurements. The results are presented in a [user-friendly](#friendliness) form that highlights all the important facts about your experiment. -BenchmarkDotNet is already adopted by [22000+ GitHub projects](https://github.com/dotnet/BenchmarkDotNet/network/dependents) including +BenchmarkDotNet is already adopted by [26400+ GitHub projects](https://github.com/dotnet/BenchmarkDotNet/network/dependents) including [.NET Runtime](https://github.com/dotnet/runtime), [.NET Compiler](https://github.com/dotnet/roslyn), [.NET Performance](https://github.com/dotnet/performance), @@ -135,8 +135,8 @@ If you want to compare benchmarks with each other, mark one of the benchmarks as the [baseline](https://benchmarkdotnet.org/articles/features/baselines.html) via `[Benchmark(Baseline = true)]`: BenchmarkDotNet will compare it with all of the other benchmarks. If you want to compare performance in different environments, use [jobs](https://benchmarkdotnet.org/articles/configs/jobs.html). -For example, you can run all the benchmarks on .NET Core 3.0 and Mono via - `[SimpleJob(RuntimeMoniker.NetCoreApp30)]` and `[SimpleJob(RuntimeMoniker.Mono)]`. +For example, you can run all the benchmarks on .NET 8.0 and Mono via + `[SimpleJob(RuntimeMoniker.Net80)]` and `[SimpleJob(RuntimeMoniker.Mono)]`. If you don't like attributes, you can call most of the APIs via the fluent style and write code like this: @@ -268,13 +268,3 @@ Let's build the best tool for benchmarking together! This project has adopted the code of conduct defined by the [Contributor Covenant](https://www.contributor-covenant.org/) to clarify expected behavior in our community. For more information, see the [.NET Foundation Code of Conduct](https://dotnetfoundation.org/code-of-conduct). - -## Sponsors - -BenchmarkDotNet is supported by the [AWS Open Source Software Fund](https://github.com/aws/dotnet-foss). - -
- - [![](https://gist.githubusercontent.com/AndreyAkinshin/f08c77960c064233348157215781b13b/raw/9d00f0ba4acb8861f9287de5d43b0ec60d7a55ac/aws-logo-small.png)](https://github.com/aws/dotnet-foss) - -
diff --git a/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj b/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj index 6a41ef6e51..c7b97bc90a 100644 --- a/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj +++ b/build/BenchmarkDotNet.Build/BenchmarkDotNet.Build.csproj @@ -6,10 +6,10 @@ enable - - - - - + + + + + diff --git a/build/BenchmarkDotNet.Build/BuildContext.cs b/build/BenchmarkDotNet.Build/BuildContext.cs index 0a165721e0..3e3c42c27d 100644 --- a/build/BenchmarkDotNet.Build/BuildContext.cs +++ b/build/BenchmarkDotNet.Build/BuildContext.cs @@ -157,6 +157,7 @@ public void GenerateFile(FilePath filePath, StringBuilder content) public void GenerateFile(FilePath filePath, string content, bool reportNoChanges = false) { + this.EnsureDirectoryExists(filePath.GetDirectory()); var relativePath = RootDirectory.GetRelativePath(filePath); if (this.FileExists(filePath)) { diff --git a/build/BenchmarkDotNet.Build/Meta/Repo.cs b/build/BenchmarkDotNet.Build/Meta/Repo.cs index 38215fcce8..22a02d51bb 100644 --- a/build/BenchmarkDotNet.Build/Meta/Repo.cs +++ b/build/BenchmarkDotNet.Build/Meta/Repo.cs @@ -12,7 +12,7 @@ public static class Repo public const string HttpsUrlBase = $"https://github.com/{Owner}/{Name}"; public const string HttpsGitUrl = $"{HttpsUrlBase}.git"; - public const string ChangelogDetailsBranch = "docs-changelog-details"; + public const string ChangelogBranch = "docs-changelog"; public const string DocsStableBranch = "docs-stable"; public const string MasterBranch = "master"; diff --git a/build/BenchmarkDotNet.Build/Options/KnownOptions.cs b/build/BenchmarkDotNet.Build/Options/KnownOptions.cs index c5101e1d52..6da4e390ed 100644 --- a/build/BenchmarkDotNet.Build/Options/KnownOptions.cs +++ b/build/BenchmarkDotNet.Build/Options/KnownOptions.cs @@ -29,6 +29,12 @@ public static class KnownOptions Aliases = new[] { "-d" } }; + public static readonly BoolOption ForceClone = new("--force-clone") + { + Description = "Forces re-cloning of the changelog repository, deleting any existing directory.", + Aliases = new[] { "-f" } + }; + public static readonly BoolOption Help = new("--help") { Description = "Prints help information", diff --git a/build/BenchmarkDotNet.Build/Program.cs b/build/BenchmarkDotNet.Build/Program.cs index 51dd875fbd..5fd25cc298 100644 --- a/build/BenchmarkDotNet.Build/Program.cs +++ b/build/BenchmarkDotNet.Build/Program.cs @@ -28,10 +28,10 @@ public HelpInfo GetHelp() { return new HelpInfo { - Examples = new[] - { + Examples = + [ new Example(Name) - } + ] }; } } @@ -48,10 +48,10 @@ public HelpInfo GetHelp() { return new HelpInfo { - Examples = new[] - { + Examples = + [ new Example(Name).WithMsBuildArgument("Configuration", "Debug") - } + ] }; } } @@ -68,10 +68,10 @@ public HelpInfo GetHelp() { return new HelpInfo { - Examples = new[] - { + Examples = + [ new Example(Name) - } + ] }; } } @@ -88,12 +88,12 @@ public HelpInfo GetHelp() { return new HelpInfo { - Examples = new[] - { + Examples = + [ new Example(Name) .WithArgument(KnownOptions.Exclusive) .WithArgument(KnownOptions.Verbosity, "Diagnostic") - } + ] }; } } @@ -113,7 +113,7 @@ public class InTestsFullTask : FrostingTask, IHelpProvider } [TaskName(Name)] -[TaskDescription("Run integration tests using .NET 7 (slow)")] +[TaskDescription("Run integration tests using .NET 8 (slow)")] [IsDependentOn(typeof(BuildTask))] public class InTestsCoreTask : FrostingTask, IHelpProvider { @@ -145,68 +145,69 @@ public HelpInfo GetHelp() { return new HelpInfo { - Examples = new[] - { + Examples = + [ new Example(Name) .WithMsBuildArgument("VersionPrefix", "0.1.1729") .WithMsBuildArgument("VersionSuffix", "preview"), new Example(Name).WithArgument(KnownOptions.Stable) - } + ] }; } } [TaskName(Name)] -[TaskDescription("Update generated documentation files")] -public class DocsUpdateTask : FrostingTask, IHelpProvider +[TaskDescription("Fetch changelog files")] +public class DocsFetchTask : FrostingTask, IHelpProvider { - private const string Name = "docs-update"; - public override void Run(BuildContext context) => context.DocumentationRunner.Update(); + private const string Name = "docs-fetch"; + public override void Run(BuildContext context) => context.DocumentationRunner.Fetch(); public HelpInfo GetHelp() { return new HelpInfo { Description = $"This task updates the following files:\n" + - $"* README.md (the number of dependent projects number)\n" + + $"* Clones branch 'docs-changelog' to docs/_changelog\n" + $"* Last changelog footer (if {KnownOptions.Stable.CommandLineName} is specified)\n" + $"* All changelog details in docs/_changelog\n" + - $" (This dir is a cloned version of this repo from branch {Repo.ChangelogDetailsBranch})", - Options = new IOption[] { KnownOptions.DocsPreview, KnownOptions.DocsDepth }, - EnvironmentVariables = new[] { EnvVar.GitHubToken }, - Examples = new[] - { + $" (This dir is a cloned version of this repo from branch {Repo.ChangelogBranch})", + Options = [KnownOptions.DocsPreview, KnownOptions.DocsDepth, KnownOptions.ForceClone], + EnvironmentVariables = [EnvVar.GitHubToken], + Examples = + [ new Example(Name) .WithArgument(KnownOptions.DocsDepth, "3") .WithArgument(KnownOptions.DocsPreview) - } + .WithArgument(KnownOptions.ForceClone) + ] }; } } [TaskName(Name)] -[TaskDescription("Prepare auxiliary documentation files")] -public class DocsPrepareTask : FrostingTask, IHelpProvider +[TaskDescription("Generate auxiliary documentation files")] +public class DocsGenerateTask : FrostingTask, IHelpProvider { - private const string Name = "docs-prepare"; - public override void Run(BuildContext context) => context.DocumentationRunner.Prepare(); + private const string Name = "docs-generate"; + public override void Run(BuildContext context) => context.DocumentationRunner.Generate(); public HelpInfo GetHelp() { return new HelpInfo { - Options = new IOption[] { KnownOptions.DocsPreview }, - Examples = new[] - { + Options = [KnownOptions.DocsPreview], + Examples = + [ new Example(Name).WithArgument(KnownOptions.DocsPreview) - } + ] }; } } [TaskName(Name)] [TaskDescription("Build final documentation")] -[IsDependentOn(typeof(DocsPrepareTask))] +[IsDependentOn(typeof(DocsGenerateTask))] public class DocsBuildTask : FrostingTask, IHelpProvider { private const string Name = "docs-build"; @@ -215,11 +216,11 @@ public class DocsBuildTask : FrostingTask, IHelpProvider public HelpInfo GetHelp() => new() { Description = "The 'build' task should be run manually to build api docs", - Options = new IOption[] { KnownOptions.DocsPreview }, - Examples = new[] - { + Options = [KnownOptions.DocsPreview], + Examples = + [ new Example(Name).WithArgument(KnownOptions.DocsPreview) - } + ] }; } @@ -227,7 +228,8 @@ public class DocsBuildTask : FrostingTask, IHelpProvider [TaskDescription("Release new version")] [IsDependentOn(typeof(BuildTask))] [IsDependentOn(typeof(PackTask))] -[IsDependentOn(typeof(DocsUpdateTask))] +[IsDependentOn(typeof(DocsFetchTask))] +[IsDependentOn(typeof(DocsGenerateTask))] [IsDependentOn(typeof(DocsBuildTask))] public class ReleaseTask : FrostingTask, IHelpProvider { @@ -236,10 +238,10 @@ public class ReleaseTask : FrostingTask, IHelpProvider public HelpInfo GetHelp() => new() { - Options = new IOption[] { KnownOptions.NextVersion, KnownOptions.Push }, - EnvironmentVariables = new[] { EnvVar.GitHubToken, EnvVar.NuGetToken }, - Examples = new[] - { + Options = [KnownOptions.NextVersion, KnownOptions.Push], + EnvironmentVariables = [EnvVar.GitHubToken, EnvVar.NuGetToken], + Examples = + [ new Example(Name) .WithArgument(KnownOptions.Stable) .WithArgument(KnownOptions.NextVersion, "0.1.1729") @@ -247,6 +249,6 @@ public class ReleaseTask : FrostingTask, IHelpProvider new Example(Name) .WithArgument(KnownOptions.Stable) .WithArgument(KnownOptions.Push) - } + ] }; } \ No newline at end of file diff --git a/build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogBuilder.cs b/build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogBuilder.cs new file mode 100644 index 0000000000..5bfc8c91b2 --- /dev/null +++ b/build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogBuilder.cs @@ -0,0 +1,256 @@ +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using BenchmarkDotNet.Build.Meta; +using BenchmarkDotNet.Build.Options; +using Cake.Common.Diagnostics; +using Cake.Common.IO; +using Cake.Core.IO; +using Cake.FileHelpers; + +namespace BenchmarkDotNet.Build.Runners.Changelog; + +public class ChangelogBuilder +{ + private readonly BuildContext context; + private readonly bool preview; + private readonly string depth; + private readonly bool forceClone; + + /// + /// Directory with original changelog part files from branch 'docs-changelog' + /// + public DirectoryPath SrcDirectory { get; } + + /// + /// Final changelog files to be used by docfx + /// + public DirectoryPath DocfxDirectory { get; } + + public ChangelogBuilder(BuildContext context) + { + this.context = context; + preview = KnownOptions.DocsPreview.Resolve(context); + depth = KnownOptions.DocsDepth.Resolve(context); + forceClone = KnownOptions.ForceClone.Resolve(context); + + var docsDirectory = context.RootDirectory.Combine("docs"); + SrcDirectory = docsDirectory.Combine("_changelog"); + DocfxDirectory = docsDirectory.Combine("changelog"); + } + + public void Fetch() + { + EnvVar.GitHubToken.AssertHasValue(); + + EnsureSrcDirectoryExist(forceClone); + + var history = context.VersionHistory; + var stableVersionCount = history.StableVersions.Length; + + if (depth.Equals("all", StringComparison.OrdinalIgnoreCase)) + { + FetchDetails( + history.StableVersions.First(), + history.FirstCommit); + + for (var i = 1; i < stableVersionCount; i++) + FetchDetails( + history.StableVersions[i], + history.StableVersions[i - 1]); + } + else if (depth != "") + { + if (!int.TryParse(depth, CultureInfo.InvariantCulture, out var depthValue)) + throw new InvalidDataException($"Failed to parse the depth value: '{depth}'"); + + for (var i = Math.Max(stableVersionCount - depthValue, 1); i < stableVersionCount; i++) + FetchDetails( + history.StableVersions[i], + history.StableVersions[i - 1]); + } + + if (preview) + FetchDetails( + history.CurrentVersion, + history.StableVersions.Last(), + "HEAD"); + } + + private void FetchDetails(string version, string versionPrevious, string lastCommit = "") + { + EnsureSrcDirectoryExist(); + context.Information($"Downloading changelog details for v{version}"); + var detailsDirectory = SrcDirectory.Combine("details"); + ChangelogDetailsBuilder.Run(context, detailsDirectory, version, versionPrevious, lastCommit); + } + + public void Generate() + { + GenerateLastFooter(); + + foreach (var version in context.VersionHistory.StableVersions) + GenerateVersion(version); + if (preview) + GenerateVersion(context.VersionHistory.CurrentVersion); + + GenerateIndex(); + GenerateFull(); + GenerateToc(); + } + + public void GenerateLastFooter() + { + var version = context.VersionHistory.CurrentVersion; + var previousVersion = context.VersionHistory.StableVersions.Last(); + var date = KnownOptions.Stable.Resolve(context) + ? DateTime.Now.ToString("MMMM dd, yyyy", CultureInfo.InvariantCulture) + : "TBA"; + + var content = new StringBuilder(); + content.AppendLine($"_Date: {date}_"); + content.AppendLine(""); + content.AppendLine( + $"_Milestone: [v{version}](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av{version})_"); + content.AppendLine( + $"([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v{previousVersion}...v{version}))"); + content.AppendLine(""); + content.AppendLine("_NuGet Packages:_"); + foreach (var packageName in context.NuGetPackageNames) + content.AppendLine($"* https://www.nuget.org/packages/{packageName}/{version}"); + + var fileName = "v" + context.VersionHistory.CurrentVersion + ".md"; + var filePath = SrcDirectory.Combine("footer").CombineWithFilePath(fileName); + context.GenerateFile(filePath, content); + } + + private void GenerateVersion(string version) + { + EnsureSrcDirectoryExist(); + var md = $"v{version}.md"; + var header = SrcDirectory.Combine("header").CombineWithFilePath(md); + var footer = SrcDirectory.Combine("footer").CombineWithFilePath(md); + var details = SrcDirectory.Combine("details").CombineWithFilePath(md); + var release = DocfxDirectory.CombineWithFilePath(md); + + var content = new StringBuilder(); + content.AppendLine("---"); + content.AppendLine("uid: changelog.v" + version); + content.AppendLine("---"); + content.AppendLine(""); + content.AppendLine("# BenchmarkDotNet v" + version); + content.AppendLine(""); + content.AppendLine(""); + + if (context.FileExists(header)) + { + content.AppendLine(context.FileReadText(header)); + content.AppendLine(""); + content.AppendLine(""); + } + + if (context.FileExists(details)) + { + content.AppendLine(context.FileReadText(details)); + content.AppendLine(""); + content.AppendLine(""); + } + + if (context.FileExists(footer)) + { + content.AppendLine("## Additional details"); + content.AppendLine(""); + content.AppendLine(context.FileReadText(footer)); + } + + context.GenerateFile(release, content.ToString()); + } + + private void GenerateIndex() + { + var content = new StringBuilder(); + content.AppendLine("---"); + content.AppendLine("uid: changelog"); + content.AppendLine("---"); + content.AppendLine(""); + content.AppendLine("# ChangeLog"); + content.AppendLine(""); + if (preview) + content.AppendLine($"* @changelog.v{context.VersionHistory.CurrentVersion}"); + foreach (var version in context.VersionHistory.StableVersions.Reverse()) + content.AppendLine($"* @changelog.v{version}"); + content.AppendLine("* @changelog.full"); + + context.GenerateFile(DocfxDirectory.CombineWithFilePath("index.md"), content); + } + + private void GenerateFull() + { + var content = new StringBuilder(); + content.AppendLine("---"); + content.AppendLine("uid: changelog.full"); + content.AppendLine("---"); + content.AppendLine(""); + content.AppendLine("# Full ChangeLog"); + content.AppendLine(""); + if (preview) + content.AppendLine( + $"[!include[v{context.VersionHistory.CurrentVersion}](v{context.VersionHistory.CurrentVersion}.md)]"); + foreach (var version in context.VersionHistory.StableVersions.Reverse()) + content.AppendLine($"[!include[v{version}](v{version}.md)]"); + + context.GenerateFile(DocfxDirectory.CombineWithFilePath("full.md"), content); + } + + private void GenerateToc() + { + var content = new StringBuilder(); + + if (preview) + { + content.AppendLine($"- name: v{context.VersionHistory.CurrentVersion}"); + content.AppendLine($" href: v{context.VersionHistory.CurrentVersion}.md"); + } + + foreach (var version in context.VersionHistory.StableVersions.Reverse()) + { + content.AppendLine($"- name: v{version}"); + content.AppendLine($" href: v{version}.md"); + } + + content.AppendLine("- name: Full ChangeLog"); + content.AppendLine(" href: full.md"); + + context.GenerateFile(DocfxDirectory.CombineWithFilePath("toc.yml"), content); + } + + private void EnsureSrcDirectoryExist(bool forceClone = false) + { + void Log(string message) => context.Information($"[Changelog] {message}"); + + Log($"Preparing git sub-repository for changelog branch '{Repo.ChangelogBranch}'. " + + $"Target directory: '{SrcDirectory}'."); + if (context.DirectoryExists(SrcDirectory) && forceClone) + { + Log($"Directory '{SrcDirectory}' already exists and forceClean is specified. " + + $"Deleting the current directory..."); + context.DeleteDirectory( + SrcDirectory, + new DeleteDirectorySettings { Force = true, Recursive = true }); + Log($"Directory '{SrcDirectory}' deleted successfully."); + } + + if (!context.DirectoryExists(SrcDirectory)) + { + Log($"Cloning branch '{Repo.ChangelogBranch}' from '{Repo.HttpsGitUrl}' to '{SrcDirectory}'."); + context.GitRunner.Clone(SrcDirectory, Repo.HttpsGitUrl, Repo.ChangelogBranch); + Log($"Clone completed: '{Repo.ChangelogBranch}' -> '{SrcDirectory}'."); + } + else + { + Log($"Directory '{SrcDirectory}' already exists. Skipping clone."); + } + } +} \ No newline at end of file diff --git a/build/BenchmarkDotNet.Build/ChangeLogBuilder.cs b/build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogDetailsBuilder.cs similarity index 93% rename from build/BenchmarkDotNet.Build/ChangeLogBuilder.cs rename to build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogDetailsBuilder.cs index e7769e1fbd..27540402e3 100644 --- a/build/BenchmarkDotNet.Build/ChangeLogBuilder.cs +++ b/build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogDetailsBuilder.cs @@ -10,15 +10,15 @@ using Cake.Core.IO; using Octokit; -namespace BenchmarkDotNet.Build; +namespace BenchmarkDotNet.Build.Runners.Changelog; -public static class ChangeLogBuilder +public static class ChangelogDetailsBuilder { - private class Config + private class Config(string currentVersion, string previousVersion, string lastCommit) { - public string CurrentVersion { get; } - public string PreviousVersion { get; } - public string LastCommit { get; } + public string CurrentVersion { get; } = currentVersion; + public string PreviousVersion { get; } = previousVersion; + public string LastCommit { get; } = lastCommit; public void Deconstruct(out string currentMilestone, out string previousMilestone, out string lastCommit) { @@ -26,13 +26,6 @@ public void Deconstruct(out string currentMilestone, out string previousMileston previousMilestone = PreviousVersion; lastCommit = LastCommit; } - - public Config(string currentVersion, string previousVersion, string lastCommit) - { - CurrentVersion = currentVersion; - PreviousVersion = previousVersion; - LastCommit = lastCommit; - } } private class MarkdownBuilder diff --git a/build/BenchmarkDotNet.Build/Runners/DocumentationRunner.cs b/build/BenchmarkDotNet.Build/Runners/DocumentationRunner.cs index 2d2128a86f..e1d1af7469 100644 --- a/build/BenchmarkDotNet.Build/Runners/DocumentationRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/DocumentationRunner.cs @@ -1,11 +1,9 @@ -using System; -using System.Globalization; using System.IO; using System.Linq; using System.Text; using BenchmarkDotNet.Build.Helpers; using BenchmarkDotNet.Build.Meta; -using BenchmarkDotNet.Build.Options; +using BenchmarkDotNet.Build.Runners.Changelog; using Cake.Common.Diagnostics; using Cake.Common.IO; using Cake.Core.IO; @@ -16,85 +14,47 @@ namespace BenchmarkDotNet.Build.Runners; public class DocumentationRunner { private readonly BuildContext context; - private readonly bool preview; - private readonly string depth; - - public DirectoryPath ChangelogDirectory { get; } - public DirectoryPath ChangelogSrcDirectory { get; } - private readonly DirectoryPath changelogDetailsDirectory; + private readonly ChangelogBuilder changelogBuilder; private readonly DirectoryPath docsGeneratedDirectory; private readonly FilePath docfxJsonFile; private readonly FilePath redirectFile; private readonly FilePath readmeFile; private readonly FilePath rootIndexFile; - private readonly FilePath changelogIndexFile; - private readonly FilePath changelogFullFile; - private readonly FilePath changelogTocFile; - private readonly FilePath lastFooterFile; + + public DirectoryPath ChangelogSrcDirectory => changelogBuilder.SrcDirectory; public DocumentationRunner(BuildContext context) { this.context = context; - preview = KnownOptions.DocsPreview.Resolve(context); - depth = KnownOptions.DocsDepth.Resolve(context); + changelogBuilder = new ChangelogBuilder(context); var docsDirectory = context.RootDirectory.Combine("docs"); - ChangelogDirectory = docsDirectory.Combine("changelog"); - ChangelogSrcDirectory = docsDirectory.Combine("_changelog"); - changelogDetailsDirectory = ChangelogSrcDirectory.Combine("details"); docsGeneratedDirectory = docsDirectory.Combine("_site"); - redirectFile = docsDirectory.Combine("_redirects").CombineWithFilePath("_redirects"); docfxJsonFile = docsDirectory.CombineWithFilePath("docfx.json"); readmeFile = context.RootDirectory.CombineWithFilePath("README.md"); rootIndexFile = docsDirectory.CombineWithFilePath("index.md"); - changelogIndexFile = ChangelogDirectory.CombineWithFilePath("index.md"); - changelogFullFile = ChangelogDirectory.CombineWithFilePath("full.md"); - changelogTocFile = ChangelogDirectory.CombineWithFilePath("toc.yml"); - lastFooterFile = ChangelogSrcDirectory.Combine("footer") - .CombineWithFilePath("v" + context.VersionHistory.CurrentVersion + ".md"); } - public void Update() + public void Fetch() { EnvVar.GitHubToken.AssertHasValue(); + changelogBuilder.Fetch(); + } - UpdateReadme(); - UpdateLastFooter(); - - EnsureChangelogDetailsExist(); - - var history = context.VersionHistory; - var stableVersionCount = history.StableVersions.Length; - - if (depth.Equals("all", StringComparison.OrdinalIgnoreCase)) - { - DocfxChangelogDownload( - history.StableVersions.First(), - history.FirstCommit); - - for (var i = 1; i < stableVersionCount; i++) - DocfxChangelogDownload( - history.StableVersions[i], - history.StableVersions[i - 1]); - } - else if (depth != "") - { - if (!int.TryParse(depth, CultureInfo.InvariantCulture, out var depthValue)) - throw new InvalidDataException($"Failed to parse the depth value: '{depth}'"); + public void Generate() + { + changelogBuilder.Generate(); - for (var i = Math.Max(stableVersionCount - depthValue, 1); i < stableVersionCount; i++) - DocfxChangelogDownload( - history.StableVersions[i], - history.StableVersions[i - 1]); - } + UpdateReadme(); + GenerateIndexMd(); + } - if (preview) - DocfxChangelogDownload( - history.CurrentVersion, - history.StableVersions.Last(), - "HEAD"); + public void Build() + { + RunDocfx(); + GenerateRedirects(); } private void UpdateReadme() @@ -108,24 +68,6 @@ private void UpdateReadme() context.GenerateFile(context.ReadmeFile, content, true); } - public void Prepare() - { - foreach (var version in context.VersionHistory.StableVersions) - DocfxChangelogGenerate(version); - if (preview) - DocfxChangelogGenerate(context.VersionHistory.CurrentVersion); - - GenerateIndexMd(); - GenerateChangelogIndex(); - GenerateChangelogFull(); - GenerateChangelogToc(); - } - - public void Build() - { - RunDocfx(); - GenerateRedirects(); - } private void RunDocfx() { @@ -149,124 +91,6 @@ private void GenerateIndexMd() context.GenerateFile(rootIndexFile, content); } - private void GenerateChangelogToc() - { - var content = new StringBuilder(); - - if (preview) - { - content.AppendLine($"- name: v{context.VersionHistory.CurrentVersion}"); - content.AppendLine($" href: v{context.VersionHistory.CurrentVersion}.md"); - } - - foreach (var version in context.VersionHistory.StableVersions.Reverse()) - { - content.AppendLine($"- name: v{version}"); - content.AppendLine($" href: v{version}.md"); - } - - content.AppendLine("- name: Full ChangeLog"); - content.AppendLine(" href: full.md"); - - context.GenerateFile(changelogTocFile, content); - } - - private void GenerateChangelogFull() - { - var content = new StringBuilder(); - content.AppendLine("---"); - content.AppendLine("uid: changelog.full"); - content.AppendLine("---"); - content.AppendLine(""); - content.AppendLine("# Full ChangeLog"); - content.AppendLine(""); - if (preview) - content.AppendLine( - $"[!include[v{context.VersionHistory.CurrentVersion}](v{context.VersionHistory.CurrentVersion}.md)]"); - foreach (var version in context.VersionHistory.StableVersions.Reverse()) - content.AppendLine($"[!include[v{version}](v{version}.md)]"); - - context.GenerateFile(changelogFullFile, content); - } - - private void GenerateChangelogIndex() - { - var content = new StringBuilder(); - content.AppendLine("---"); - content.AppendLine("uid: changelog"); - content.AppendLine("---"); - content.AppendLine(""); - content.AppendLine("# ChangeLog"); - content.AppendLine(""); - if (preview) - content.AppendLine($"* @changelog.v{context.VersionHistory.CurrentVersion}"); - foreach (var version in context.VersionHistory.StableVersions.Reverse()) - content.AppendLine($"* @changelog.v{version}"); - content.AppendLine("* @changelog.full"); - - context.GenerateFile(changelogIndexFile, content); - } - - private void DocfxChangelogGenerate(string version) - { - EnsureChangelogDetailsExist(); - var md = $"v{version}.md"; - var header = ChangelogSrcDirectory.Combine("header").CombineWithFilePath(md); - var footer = ChangelogSrcDirectory.Combine("footer").CombineWithFilePath(md); - var details = ChangelogSrcDirectory.Combine("details").CombineWithFilePath(md); - var release = ChangelogDirectory.CombineWithFilePath(md); - - var content = new StringBuilder(); - content.AppendLine("---"); - content.AppendLine("uid: changelog.v" + version); - content.AppendLine("---"); - content.AppendLine(""); - content.AppendLine("# BenchmarkDotNet v" + version); - content.AppendLine(""); - content.AppendLine(""); - - if (context.FileExists(header)) - { - content.AppendLine(context.FileReadText(header)); - content.AppendLine(""); - content.AppendLine(""); - } - - if (context.FileExists(details)) - { - content.AppendLine(context.FileReadText(details)); - content.AppendLine(""); - content.AppendLine(""); - } - - if (context.FileExists(footer)) - { - content.AppendLine("## Additional details"); - content.AppendLine(""); - content.AppendLine(context.FileReadText(footer)); - } - - context.GenerateFile(release, content.ToString()); - } - - private void EnsureChangelogDetailsExist(bool forceClean = false) - { - if (context.DirectoryExists(changelogDetailsDirectory) && forceClean) - context.DeleteDirectory( - changelogDetailsDirectory, - new DeleteDirectorySettings { Force = true, Recursive = true }); - - if (!context.DirectoryExists(changelogDetailsDirectory)) - context.GitRunner.Clone(changelogDetailsDirectory, Repo.HttpsGitUrl, Repo.ChangelogDetailsBranch); - } - - private void DocfxChangelogDownload(string version, string versionPrevious, string lastCommit = "") - { - EnsureChangelogDetailsExist(); - context.Information($"Downloading changelog details for v{version}"); - ChangeLogBuilder.Run(context, changelogDetailsDirectory, version, versionPrevious, lastCommit); - } - private void GenerateRedirects() { if (!context.FileExists(redirectFile)) @@ -300,27 +124,4 @@ private void GenerateRedirects() context.GenerateFile(fullFilePath, content); } } - - private void UpdateLastFooter() - { - var version = context.VersionHistory.CurrentVersion; - var previousVersion = context.VersionHistory.StableVersions.Last(); - var date = KnownOptions.Stable.Resolve(context) - ? DateTime.Now.ToString("MMMM dd, yyyy", CultureInfo.InvariantCulture) - : "TBA"; - - var content = new StringBuilder(); - content.AppendLine($"_Date: {date}_"); - content.AppendLine(""); - content.AppendLine( - $"_Milestone: [v{version}](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av{version})_"); - content.AppendLine( - $"([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v{previousVersion}...v{version}))"); - content.AppendLine(""); - content.AppendLine("_NuGet Packages:_"); - foreach (var packageName in context.NuGetPackageNames) - content.AppendLine($"* https://www.nuget.org/packages/{packageName}/{version}"); - - context.GenerateFile(lastFooterFile, content); - } } \ No newline at end of file diff --git a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs index 9b10d54ea4..89b777d09f 100644 --- a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs @@ -4,6 +4,7 @@ using Cake.Common.Tools.DotNet; using Cake.Common.Tools.DotNet.Test; using Cake.Core.IO; +using System.Runtime.InteropServices; namespace BenchmarkDotNet.Build.Runners; @@ -47,7 +48,8 @@ private DotNetTestSettings GetTestSettingsParameters(FilePath logFile, string tf private void RunTests(FilePath projectFile, string alias, string tfm) { var os = Utils.GetOs(); - var trxFileName = $"{os}-{alias}-{tfm}.trx"; + var arch = RuntimeInformation.OSArchitecture.ToString().ToLower(); + var trxFileName = $"{os}({arch})-{alias}-{tfm}.trx"; var trxFile = TestOutputDirectory.CombineWithFilePath(trxFileName); var settings = GetTestSettingsParameters(trxFile, tfm); diff --git a/build/cSpell.json b/build/cSpell.json index 6bdc2226cc..a34d415fe3 100644 --- a/build/cSpell.json +++ b/build/cSpell.json @@ -12,6 +12,7 @@ "Cygwin", "Diagnoser", "diagnosers", + "diagsession", "disassemblers", "disassm", "Jits", @@ -29,6 +30,7 @@ "Pseudocode", "runtimes", "Serilog", + "vsprofiler", "vstest", "Tailcall", "toolchains", @@ -37,8 +39,10 @@ "ignoreWords": [ "Akinshin", "Andrey", + "Cassell", "Expecto", "Jint", + "JITted", "LoongArch64", "macrobenchmark", "MediatR", @@ -47,6 +51,9 @@ "NodaTime", "Npgsql", "Sitnik's", + "Sitnik", + "Stepanov", + "Yegor", "Wojciech", "Avalonia", "Gitter" diff --git a/build/common.props b/build/common.props index e74a0b8680..b33bc4d863 100644 --- a/build/common.props +++ b/build/common.props @@ -23,15 +23,20 @@ true annotations + + true - + $(MSBuildThisFileDirectory)strongNameKey.snk true + + + true @@ -40,7 +45,7 @@ - 0.14.0 + 0.15.3 @@ -56,12 +61,13 @@ - + all runtime; build; native; contentfiles; analyzers - + all + runtime; build; native; contentfiles; analyzers diff --git a/build/sdk/global.json b/build/sdk/global.json index 94c8cfbc04..be5c357f30 100644 --- a/build/sdk/global.json +++ b/build/sdk/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.101", + "version": "8.0.410", "rollForward": "disable" } } diff --git a/build/versions.txt b/build/versions.txt index 584dcda7c7..df0365a030 100644 --- a/build/versions.txt +++ b/build/versions.txt @@ -57,4 +57,8 @@ 0.13.10 0.13.11 0.13.12 -0.14.0 \ No newline at end of file +0.14.0 +0.15.0 +0.15.1 +0.15.2 +0.15.3 \ No newline at end of file diff --git a/docs/.gitignore b/docs/.gitignore index aad3551319..ebe91edcfe 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -11,4 +11,6 @@ _exported_templates docfx-bin api/* !api/index.md -/index.md \ No newline at end of file +/index.md +_changelog +changelog diff --git a/docs/_changelog/.gitignore b/docs/_changelog/.gitignore deleted file mode 100644 index 3bc49a8a60..0000000000 --- a/docs/_changelog/.gitignore +++ /dev/null @@ -1 +0,0 @@ -details diff --git a/docs/_changelog/footer/v0.10.0.md b/docs/_changelog/footer/v0.10.0.md deleted file mode 100644 index 823ad6954c..0000000000 --- a/docs/_changelog/footer/v0.10.0.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: November 10, 2016_ - -_Milestone: [v0.10.0](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.0)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.0 - -_Online Documentation:_ https://dotnet.github.io/BenchmarkDotNet/ \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.1.md b/docs/_changelog/footer/v0.10.1.md deleted file mode 100644 index 6a2f16d19d..0000000000 --- a/docs/_changelog/footer/v0.10.1.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: December 04, 2016_ - -_Milestone: [v0.10.1](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.1)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.10.md b/docs/_changelog/footer/v0.10.10.md deleted file mode 100644 index 9dde0a1cc3..0000000000 --- a/docs/_changelog/footer/v0.10.10.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: November 03, 2017_ - -_Milestone: [v0.10.10](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.10)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.9...v0.10.10)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.10 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.11.md b/docs/_changelog/footer/v0.10.11.md deleted file mode 100644 index 0de3715f93..0000000000 --- a/docs/_changelog/footer/v0.10.11.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: December 01, 2017_ - -_Milestone: [v0.10.11](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.11)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.10...v0.10.11)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.11 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.12.md b/docs/_changelog/footer/v0.10.12.md deleted file mode 100644 index 3106385452..0000000000 --- a/docs/_changelog/footer/v0.10.12.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: January 15, 2018_ - -_Milestone: [v0.10.12](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.12)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.11...v0.10.12)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.12 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.13.md b/docs/_changelog/footer/v0.10.13.md deleted file mode 100644 index 6eb70e1fc2..0000000000 --- a/docs/_changelog/footer/v0.10.13.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: March 02, 2018_ - -_Milestone: [v0.10.13](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.13)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.12...v0.10.13)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.13 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.13 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.13 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.13 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.14.md b/docs/_changelog/footer/v0.10.14.md deleted file mode 100644 index 2fd3ea5542..0000000000 --- a/docs/_changelog/footer/v0.10.14.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: April 09, 2018_ - -_Milestone: [v0.10.14](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.14)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.13...v0.10.14)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.14 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.14 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.14 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.14 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.2.md b/docs/_changelog/footer/v0.10.2.md deleted file mode 100644 index 46e50a038f..0000000000 --- a/docs/_changelog/footer/v0.10.2.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: January 21, 2017_ - -_Milestone: [v0.10.2](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.2)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.3.md b/docs/_changelog/footer/v0.10.3.md deleted file mode 100644 index 623439e439..0000000000 --- a/docs/_changelog/footer/v0.10.3.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: March 01, 2017_ - -_Milestone: [v0.10.3](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.3)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.3 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.4.md b/docs/_changelog/footer/v0.10.4.md deleted file mode 100644 index 9f497b878b..0000000000 --- a/docs/_changelog/footer/v0.10.4.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: April 21, 2017_ - -_Milestone: [v0.10.4](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.4)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.4 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.5.md b/docs/_changelog/footer/v0.10.5.md deleted file mode 100644 index b4d8d5b914..0000000000 --- a/docs/_changelog/footer/v0.10.5.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: April 26, 2017_ - -_Milestone: [v0.10.5](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.5)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.5 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.6.md b/docs/_changelog/footer/v0.10.6.md deleted file mode 100644 index b29dfc3939..0000000000 --- a/docs/_changelog/footer/v0.10.6.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: May 12, 2017_ - -_Milestone: [v0.10.6](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.6)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.6 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.7.md b/docs/_changelog/footer/v0.10.7.md deleted file mode 100644 index 5ee21fee21..0000000000 --- a/docs/_changelog/footer/v0.10.7.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: June 05, 2017_ - -_Milestone: [v0.10.7](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.7)_ - -_Overview post: https://aakinshin.net/posts/bdn-v0_10_7/_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.7 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.8.md b/docs/_changelog/footer/v0.10.8.md deleted file mode 100644 index 791022d425..0000000000 --- a/docs/_changelog/footer/v0.10.8.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: June 09, 2017_ - -_Milestone: [v0.10.8](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.8)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.8 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.9.md b/docs/_changelog/footer/v0.10.9.md deleted file mode 100644 index 8fc3fef03e..0000000000 --- a/docs/_changelog/footer/v0.10.9.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: July 28, 2017_ - -_Milestone: [v0.10.9](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.9)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.9 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.0.md b/docs/_changelog/footer/v0.11.0.md deleted file mode 100644 index f9637a79e6..0000000000 --- a/docs/_changelog/footer/v0.11.0.md +++ /dev/null @@ -1,8 +0,0 @@ -_Date: July 23, 2018_ - -_Milestone: [v0.11.0](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.0)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.14...v0.11.0)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.0 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.1.md b/docs/_changelog/footer/v0.11.1.md deleted file mode 100644 index f6c73f9510..0000000000 --- a/docs/_changelog/footer/v0.11.1.md +++ /dev/null @@ -1,8 +0,0 @@ -_Date: August 22, 2018_ - -_Milestone: [v0.11.1](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.1)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.0...v0.11.1)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.2.md b/docs/_changelog/footer/v0.11.2.md deleted file mode 100644 index 338170b8ae..0000000000 --- a/docs/_changelog/footer/v0.11.2.md +++ /dev/null @@ -1,8 +0,0 @@ -_Date: November 1, 2018_ - -_Milestone: [v0.11.2](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.2)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.1...v0.11.2)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.3.md b/docs/_changelog/footer/v0.11.3.md deleted file mode 100644 index befc032994..0000000000 --- a/docs/_changelog/footer/v0.11.3.md +++ /dev/null @@ -1,8 +0,0 @@ -_Date: November 20, 2018_ - -_Milestone: [v0.11.3](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.3)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.2...v0.11.3)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.3 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.4.md b/docs/_changelog/footer/v0.11.4.md deleted file mode 100644 index 5cbac89988..0000000000 --- a/docs/_changelog/footer/v0.11.4.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: February 15, 2019_ - -_Milestone: [v0.11.4](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.4)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.3...v0.11.4)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Tool/0.11.4 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.5.md b/docs/_changelog/footer/v0.11.5.md deleted file mode 100644 index be3282de78..0000000000 --- a/docs/_changelog/footer/v0.11.5.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: April 2, 2019_ - -_Milestone: [v0.11.5](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.11.5)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.4...v0.11.5)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Tool/0.11.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.11.5 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.12.0.md b/docs/_changelog/footer/v0.12.0.md deleted file mode 100644 index 565429ca6a..0000000000 --- a/docs/_changelog/footer/v0.12.0.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: October 24, 2019_ - -_Milestone: [v0.12.0](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.12.0)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.5...v0.12.0)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.12.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.12.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Tool/0.12.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.12.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.12.0 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.12.1.md b/docs/_changelog/footer/v0.12.1.md deleted file mode 100644 index af5fca9dc4..0000000000 --- a/docs/_changelog/footer/v0.12.1.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: April 6, 2020_ - -_Milestone: [v0.12.1](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.12.1)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.12.0...v0.12.1)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.12.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.12.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Tool/0.12.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.12.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.12.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.0.md b/docs/_changelog/footer/v0.13.0.md deleted file mode 100644 index 7a8ed327bc..0000000000 --- a/docs/_changelog/footer/v0.13.0.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: May 19, 2021_ - -_Milestone: [v0.13.0](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.0)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.12.1...v0.13.0)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.0 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.1.md b/docs/_changelog/footer/v0.13.1.md deleted file mode 100644 index eb71d8729f..0000000000 --- a/docs/_changelog/footer/v0.13.1.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: August 11, 2021_ - -_Milestone: [v0.13.1](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.1)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.0...v0.13.1)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.10.md b/docs/_changelog/footer/v0.13.10.md deleted file mode 100644 index 68ef4fd0db..0000000000 --- a/docs/_changelog/footer/v0.13.10.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: November 01, 2023_ - -_Milestone: [v0.13.10](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.10)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.9...v0.13.10)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.10 diff --git a/docs/_changelog/footer/v0.13.11.md b/docs/_changelog/footer/v0.13.11.md deleted file mode 100644 index 03dff00d30..0000000000 --- a/docs/_changelog/footer/v0.13.11.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: December 06, 2023_ - -_Milestone: [v0.13.11](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.11)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.10...v0.13.11)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.11 diff --git a/docs/_changelog/footer/v0.13.12.md b/docs/_changelog/footer/v0.13.12.md deleted file mode 100644 index 246864bb8a..0000000000 --- a/docs/_changelog/footer/v0.13.12.md +++ /dev/null @@ -1,12 +0,0 @@ -_Date: January 05, 2024_ - -_Milestone: [v0.13.12](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.12)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.11...v0.13.12)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.12 -* https://www.nuget.org/packages/BenchmarkDotNet.TestAdapter/0.13.12 diff --git a/docs/_changelog/footer/v0.13.2.md b/docs/_changelog/footer/v0.13.2.md deleted file mode 100644 index c9022e590e..0000000000 --- a/docs/_changelog/footer/v0.13.2.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: August 26, 2022_ - -_Milestone: [v0.13.2](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.2)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.1...v0.13.2)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.3.md b/docs/_changelog/footer/v0.13.3.md deleted file mode 100644 index f0fa6dff23..0000000000 --- a/docs/_changelog/footer/v0.13.3.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: December 26, 2022_ - -_Milestone: [v0.13.3](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.3)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.2...v0.13.3)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.3 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.4.md b/docs/_changelog/footer/v0.13.4.md deleted file mode 100644 index b6ac2ad6af..0000000000 --- a/docs/_changelog/footer/v0.13.4.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: January 13, 2023_ - -_Milestone: [v0.13.4](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.4)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.3...v0.13.4)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.4 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.5.md b/docs/_changelog/footer/v0.13.5.md deleted file mode 100644 index 577eeaea84..0000000000 --- a/docs/_changelog/footer/v0.13.5.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: February 17, 2023_ - -_Milestone: [v0.13.5](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.5)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.4...v0.13.5)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.5 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.6.md b/docs/_changelog/footer/v0.13.6.md deleted file mode 100644 index 7235cd2302..0000000000 --- a/docs/_changelog/footer/v0.13.6.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: July 11, 2023_ - -_Milestone: [v0.13.6](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.6)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.5...v0.13.6)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.6 diff --git a/docs/_changelog/footer/v0.13.7.md b/docs/_changelog/footer/v0.13.7.md deleted file mode 100644 index 1b256837d4..0000000000 --- a/docs/_changelog/footer/v0.13.7.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: August 04, 2023_ - -_Milestone: [v0.13.7](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.7)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.6...v0.13.7)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.7 diff --git a/docs/_changelog/footer/v0.13.8.md b/docs/_changelog/footer/v0.13.8.md deleted file mode 100644 index a8551cc47f..0000000000 --- a/docs/_changelog/footer/v0.13.8.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: September 08, 2023_ - -_Milestone: [v0.13.8](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.8)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.7...v0.13.8)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.8 diff --git a/docs/_changelog/footer/v0.13.9.md b/docs/_changelog/footer/v0.13.9.md deleted file mode 100644 index 962bc252cc..0000000000 --- a/docs/_changelog/footer/v0.13.9.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: October 05, 2023_ - -_Milestone: [v0.13.9](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.9)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.8...v0.13.9)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.9 diff --git a/docs/_changelog/footer/v0.14.0.md b/docs/_changelog/footer/v0.14.0.md deleted file mode 100644 index cf7b8d6984..0000000000 --- a/docs/_changelog/footer/v0.14.0.md +++ /dev/null @@ -1,14 +0,0 @@ -_Date: TBA_ - -_Milestone: [v0.14.0](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.14.0)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.12...v0.14.0)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.14.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.14.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotMemory/0.14.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.14.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.14.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Exporters.Plotting/0.14.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.14.0 -* https://www.nuget.org/packages/BenchmarkDotNet.TestAdapter/0.14.0 diff --git a/docs/_changelog/footer/v0.8.2.md b/docs/_changelog/footer/v0.8.2.md deleted file mode 100644 index cbd405d3ae..0000000000 --- a/docs/_changelog/footer/v0.8.2.md +++ /dev/null @@ -1,4 +0,0 @@ -_Date: January 19, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.8.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.0.md b/docs/_changelog/footer/v0.9.0.md deleted file mode 100644 index 55346f6951..0000000000 --- a/docs/_changelog/footer/v0.9.0.md +++ /dev/null @@ -1,4 +0,0 @@ -_Date: February 9, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.0 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.1.md b/docs/_changelog/footer/v0.9.1.md deleted file mode 100644 index b38f8a6436..0000000000 --- a/docs/_changelog/footer/v0.9.1.md +++ /dev/null @@ -1,4 +0,0 @@ -_Date: February 10, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.2.md b/docs/_changelog/footer/v0.9.2.md deleted file mode 100644 index 73c081a4cd..0000000000 --- a/docs/_changelog/footer/v0.9.2.md +++ /dev/null @@ -1,6 +0,0 @@ -_Milestone: [v0.9.2](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.2)_ - -_Date: March 5, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.3.md b/docs/_changelog/footer/v0.9.3.md deleted file mode 100644 index da9b45b165..0000000000 --- a/docs/_changelog/footer/v0.9.3.md +++ /dev/null @@ -1,7 +0,0 @@ -_Milestone: [v0.9.3](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.3)_ - -_Date: March 13, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.3 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.3-beta \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.4.md b/docs/_changelog/footer/v0.9.4.md deleted file mode 100644 index 0927d0a62b..0000000000 --- a/docs/_changelog/footer/v0.9.4.md +++ /dev/null @@ -1,7 +0,0 @@ -_Milestone: [v0.9.4](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.4)_ - -_Date: March 24, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.4 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.4-beta \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.5.md b/docs/_changelog/footer/v0.9.5.md deleted file mode 100644 index ea7d920994..0000000000 --- a/docs/_changelog/footer/v0.9.5.md +++ /dev/null @@ -1,7 +0,0 @@ -_Milestone: [v0.9.5](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.5)_ - -_Date: May 02, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.5 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.5-beta \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.6.md b/docs/_changelog/footer/v0.9.6.md deleted file mode 100644 index cc71d2d927..0000000000 --- a/docs/_changelog/footer/v0.9.6.md +++ /dev/null @@ -1,8 +0,0 @@ -_Milestone: [v0.9.6](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.6)_ - -_Date: May 11, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.6 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.6-beta -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.9.6 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.7.md b/docs/_changelog/footer/v0.9.7.md deleted file mode 100644 index 21f1d109e5..0000000000 --- a/docs/_changelog/footer/v0.9.7.md +++ /dev/null @@ -1,8 +0,0 @@ -_Milestone: [v0.9.7](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.7)_ - -_Date: May 29, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.7 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.7-beta -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.9.7 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.8.md b/docs/_changelog/footer/v0.9.8.md deleted file mode 100644 index fa5bc33931..0000000000 --- a/docs/_changelog/footer/v0.9.8.md +++ /dev/null @@ -1,7 +0,0 @@ -_Milestone: [v0.9.8](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.8)_ - -_Date: July 07, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.9.8 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.9.md b/docs/_changelog/footer/v0.9.9.md deleted file mode 100644 index 2e6b340c90..0000000000 --- a/docs/_changelog/footer/v0.9.9.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: August 18, 2016_ - -_Milestone: [v0.9.9](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.9)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.9.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.9.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.9.9 - -_Online Documentation:_ https://perfdotnet.github.io/BenchmarkDotNet/ \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.0.md b/docs/_changelog/header/v0.10.0.md deleted file mode 100644 index 48c4103b0a..0000000000 --- a/docs/_changelog/header/v0.10.0.md +++ /dev/null @@ -1,12 +0,0 @@ -* Now BenchmarkDotNet is a part of .NET Foundation -* Job and Column API refactoring (see new documentation) -* Measurement engine improvements -* Horology enhancement (see `TimeInterval` and `Frequency`) -* Introduced `RankColumn` which is based on `WelchTTest` (see [157aabc3](https://github.com/PerfDotNet/BenchmarkDotNet/commit/cf839a0d7ecfdf93da709b63fe324fd2157aabc3)) -* JsonExporters refactoring (see the Exporters/Json section in the documentation) - * Renamed JsonExporters classed and attributes - * JsonExporters with custom settings - * JsonExporters now includes information about the target type namespace (see [#246](https://github.com/PerfDotNet/BenchmarkDotNet/issues/246)). -* Add `JetBrains.Annotations` (see [#253](https://github.com/PerfDotNet/BenchmarkDotNet/pull/253)) -* RFC 4180 support in CSV exporters (see [#241](https://github.com/PerfDotNet/BenchmarkDotNet/issues/241)) -* Many bugfixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.1.md b/docs/_changelog/header/v0.10.1.md deleted file mode 100644 index 70e3be82c8..0000000000 --- a/docs/_changelog/header/v0.10.1.md +++ /dev/null @@ -1,8 +0,0 @@ -* MemoryDiagnoser got improved. The changes: - * Memory Diagnoser is now part of BenchmarkDotNet.Core.dll, and it's **enabled by default** - * MemoryDiagnoser is **100% accurate** about allocated memory when using default settings or Job.ShortRun or any longer job. (see [#284](https://github.com/dotnet/BenchmarkDotNet/pull/284)) - * Memory Diagnoser no longer includes allocations from Cleanup/Setup methods (see [#186](https://github.com/dotnet/BenchmarkDotNet/issues/186)) - * the results are now scaled so they are stable across the runs. (see [#133](https://github.com/dotnet/BenchmarkDotNet/issues/133)) -* .NET Core 1.1+ support, we no longer support 1.0, we target netcoreapp1.1 now. Reason: we wanted to use `GC.GetAllocatedBytesForCurrentThread` in MemoryDiagnoser which is available only in 1.1+ -* Improved information about environment in summary -* Minor bugfixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.10.md b/docs/_changelog/header/v0.10.10.md deleted file mode 100644 index ef34841958..0000000000 --- a/docs/_changelog/header/v0.10.10.md +++ /dev/null @@ -1,10 +0,0 @@ -Highlights: - -* Disassembly Diagnoser (read more here: [Disassembling .NET Code with BenchmarkDotNet](https://adamsitnik.com/Disassembly-Diagnoser/)) -* ParamsSources -* .NET Core x86 support -* Environment variables and Mono args support -* Better environment description -* More: additional sections in the documentation, bug fixes, build script improvements, internal refactoring. - -Overview post: [BenchmarkDotNet v0.10.10](https://aakinshin.net/posts/bdn-v0_10_10/) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.11.md b/docs/_changelog/header/v0.10.11.md deleted file mode 100644 index 3e53cf54cb..0000000000 --- a/docs/_changelog/header/v0.10.11.md +++ /dev/null @@ -1,8 +0,0 @@ -## Highlights - -* ByRef and Stack-only support ([#492](https://github.com/dotnet/BenchmarkDotNet/issues/492), [sample](https://github.com/dotnet/BenchmarkDotNet/blob/edf20758871ec621fdbfd93d862769da46c4bf15/samples/BenchmarkDotNet.Samples/IL/IL_RefReturns.cs)) -* .NET Core 2.1 support ([#587](https://github.com/dotnet/BenchmarkDotNet/issues/587)) -* Improved LINQPad support -* Smart logic for precision in ScaledColumn ([#509](https://github.com/dotnet/BenchmarkDotNet/issues/509), [#590](https://github.com/dotnet/BenchmarkDotNet/issues/590)) -* Better macOS version detection ([15d72388](https://github.com/dotnet/BenchmarkDotNet/commit/15d72388436c1060e87662b5f4519b9e7e071627)) -* Minor fixes and improvements \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.12.md b/docs/_changelog/header/v0.10.12.md deleted file mode 100644 index 43cff33827..0000000000 --- a/docs/_changelog/header/v0.10.12.md +++ /dev/null @@ -1,22 +0,0 @@ -Overview post: [BenchmarkDotNet v0.10.12](https://aakinshin.net/posts/bdn-v0_10_12/) - -### Highlights - -* **Improved DisassemblyDiagnoser:** - BenchmarkDotNet contains an embedded disassembler so that it can print assembly code for all benchmarks; - it's not easy, but the disassembler evolves in every release. -* **Improved MemoryDiagnoser:** - it has a better precision level, and it takes less time to evaluate memory allocations in a benchmark. -* **New TailCallDiagnoser:** - now you get notifications when JIT applies the tail call optimizations to your methods. -* **Better environment info:** - when your share performance results, it's very important to share information about your environment. - The library generates the environment summary for you by default. - Now it contains information about the amount of physical CPU, physical cores, and logic cores. - If you run a benchmark on a virtual machine, you will get the name of the hypervisor - (e.g., Hyper-V, VMware, or VirtualBox). -* **Better summary table:** - one of the greatest features of BenchmarkDotNet is the summary table. - It shows all important information about results in a compact and understandable form. - Now it has better customization options: you can display relative performance of different environments - (e.g., compare .NET Framework and .NET Core) and group benchmarks by categories. \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.13.md b/docs/_changelog/header/v0.10.13.md deleted file mode 100644 index 73f5aa1e53..0000000000 --- a/docs/_changelog/header/v0.10.13.md +++ /dev/null @@ -1 +0,0 @@ -Overview post: [BenchmarkDotNet v0.10.13](https://aakinshin.net/posts/bdn-v0_10_13/) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.14.md b/docs/_changelog/header/v0.10.14.md deleted file mode 100644 index aefb183459..0000000000 --- a/docs/_changelog/header/v0.10.14.md +++ /dev/null @@ -1,4 +0,0 @@ -* Per-method parameterization ([Read more](https://benchmarkdotnet.org/articles/features/parameterization.html)) -* Console histograms and multimodal disribution detection -* Many improvements for Mono disassembly support on Windows ([Read more](https://aakinshin.net/posts/dotnet-crossruntime-disasm/)) -* Many bugfixes diff --git a/docs/_changelog/header/v0.10.2.md b/docs/_changelog/header/v0.10.2.md deleted file mode 100644 index a93c8c36ae..0000000000 --- a/docs/_changelog/header/v0.10.2.md +++ /dev/null @@ -1,9 +0,0 @@ -* Closed [#307](https://github.com/dotnet/BenchmarkDotNet/issues/307): culture invariant statistics output -* Closed [#321](https://github.com/dotnet/BenchmarkDotNet/issues/321): persist optimized, auto-generated dll compiled from url/plain code -* Closed [#322](https://github.com/dotnet/BenchmarkDotNet/issues/332): always restore the console foreground color -* Closed [#337](https://github.com/dotnet/BenchmarkDotNet/issues/337): Better detection of Rscript.exe in RPlotExporter -* Closed [#345](https://github.com/dotnet/BenchmarkDotNet/issues/345): fix bug in WelchTTestPValueColumn for DryJob -* VS 2017 [compatibility fix](https://github.com/dotnet/BenchmarkDotNet/commit/f4bdae5b7e203e0f0a7d283db5faa78107674f31) -* [fix](https://github.com/dotnet/BenchmarkDotNet/commit/24dea483b8312efba669d82a6fac3603e60050f5) bold markup for Atlassian exporter -* Improved precision of nanobenchmarks -* Minor infrastructure changes and misc fixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.3.md b/docs/_changelog/header/v0.10.3.md deleted file mode 100644 index 976a78eeaf..0000000000 --- a/docs/_changelog/header/v0.10.3.md +++ /dev/null @@ -1,8 +0,0 @@ -* **New .csprojs** support for .NET Core. Also for F# ([#366](https://github.com/dotnet/BenchmarkDotNet/issues/366))! -* New plots and RPlotExporter (density plots for each job; cumulative mean plots) -* Fixed exporter order (now RPlotExporer uses the actual measurements instead of previous version) -* Xplat improvments in RuntimeInformation -* Introduced `RunStrategy.Monitoring` -* Possibility to set custom path for Mono ([#306](https://github.com/dotnet/BenchmarkDotNet/issues/306)) -* Possibility to set any .NET Core version >= 1.1 ([#336](https://github.com/dotnet/BenchmarkDotNet/issues/336)) -* **MemoryDiagnoser is now disabled by default (Breaking changes!!)** ([#369](https://github.com/dotnet/BenchmarkDotNet/issues/369)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.4.md b/docs/_changelog/header/v0.10.4.md deleted file mode 100644 index f423a6567a..0000000000 --- a/docs/_changelog/header/v0.10.4.md +++ /dev/null @@ -1,23 +0,0 @@ -* New logo -* Update to Roslyn 2.0, drop .NET 4.5 support ([#303](https://github.com/dotnet/BenchmarkDotNet/issues/303)) -* Initial support of HardwareCounters (Windows only) -* Initial experimental support of in-process benchmarks -* Optional configs for `BenchmarkSwitcher` ([#391](https://github.com/dotnet/BenchmarkDotNet/issues/391), [#392](https://github.com/dotnet/BenchmarkDotNet/pull/392)) -* Host API interface ([#356](https://github.com/dotnet/BenchmarkDotNet/pull/356)) -* Improved measurements for async benchmarks ([#415](https://github.com/dotnet/BenchmarkDotNet/issues/415)) -* Improved precision level (MinIterationTimes is 500ms instead of 200ms; introduced `AccuracyMode.MaxAbsoluteError` and `AccuracyMode.MaxRelativeError` instead of `AccuracyMode.MaxStdErrRelative`; logic which select amount of iterations uses confidence intervals instead of standard errors; the Error column (half of CI99.9%) is shown by default instead of StdErr) -* Introduced `ISummaryStyle`, raw data in CSV reports ([#118](https://github.com/dotnet/BenchmarkDotNet/issues/118), [#146](https://github.com/dotnet/BenchmarkDotNet/issues/146), [#396](https://github.com/dotnet/BenchmarkDotNet/pull/396)) -* Handle cases when report files are existed and locked ([#414](https://github.com/dotnet/BenchmarkDotNet/issues/414), [#416](https://github.com/dotnet/BenchmarkDotNet/pull/416)) -* MarkdownExporter right-justifies numeric columns ([#421](https://github.com/dotnet/BenchmarkDotNet/pull/421)) -* Better colors for console output ([#376](https://github.com/dotnet/BenchmarkDotNet/issues/376)) -* Column legends -* Add information about CPU microarchitecture for well-known processors to summary -* Fix AssemblyInformationalVersionAttribute ([#382](https://github.com/dotnet/BenchmarkDotNet/issues/382)) -* Fix incorrect method filtering in BenchmarkSwitcher ([#365](https://github.com/dotnet/BenchmarkDotNet/issues/365)) -* Fix OS Version in Summary for Windows 10 ([#351](https://github.com/dotnet/BenchmarkDotNet/issues/351)) -* Fix OS Version on Mono -* Fix --class and --method filtering ([#249](https://github.com/dotnet/BenchmarkDotNet/issues/249)) -* Fix --exporters option ([#189](https://github.com/dotnet/BenchmarkDotNet/issues/189)) -* Fix escaping logic in CsvExporter ([#294](https://github.com/dotnet/BenchmarkDotNet/issues/294), [#409](https://github.com/dotnet/BenchmarkDotNet/pull/409)) -* [Fix](https://github.com/dotnet/BenchmarkDotNet/commit/0a251b81b826876179740cc8b79c994a73a5cd51) MacOS detection -* Minor bugfixes and API improvements \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.5.md b/docs/_changelog/header/v0.10.5.md deleted file mode 100644 index 3ed3541c70..0000000000 --- a/docs/_changelog/header/v0.10.5.md +++ /dev/null @@ -1,5 +0,0 @@ -* Fixed SizeUnit presentation in the summary table ([#434](https://github.com/dotnet/BenchmarkDotNet/issues/434)) -* In MemoryDiagnoser, now 1kB = 1024B (instead of 1000 in v0.10.4) ([#434](https://github.com/dotnet/BenchmarkDotNet/issues/434)) -* Fix false allocations detection ([#436](https://github.com/dotnet/BenchmarkDotNet/pull/436) [9b44de70](https://github.com/dotnet/BenchmarkDotNet/commit/9b44de704b96e2333d762b14daa152d859b1917d)) -* Hide ScaledSD column for small values ([da857ad7](https://github.com/dotnet/BenchmarkDotNet/commit/da857ad7eda77db813692d3c3678f8ad04f5af78)) -* Autoselecting amount of digits after the decimal point ([#404](https://github.com/dotnet/BenchmarkDotNet/issues/404)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.6.md b/docs/_changelog/header/v0.10.6.md deleted file mode 100644 index 33948b2f80..0000000000 --- a/docs/_changelog/header/v0.10.6.md +++ /dev/null @@ -1,6 +0,0 @@ -* Removed buggy allocation from Engine which was spoiling the results of MemoryDiagnoser for micro benchmarks. This part of the code is now guarded with very strict integration tests, it should never happen again. We now also exclude the side effects of the Allocation Quantum. **This bug was serious, you must update to `0.10.6`** ([#439](https://github.com/dotnet/BenchmarkDotNet/issues/439)) -* Support of the `PackageTargetFallback` setting which allows to reference components that target old framework monikers (like `dotnet5.4` or `portable-net45+win8`) ([#438](https://github.com/dotnet/BenchmarkDotNet/issues/438)) -* Added `InstructionRetiredPerCycleColumn` which shows up automatically when `HardwareCounter.InstructionRetired` and `HardwareCounter.TotalCycles` are used. -* Support benchmark classes without namespace ([#446](https://github.com/dotnet/BenchmarkDotNet/issues/446)) -* Fix problem with RPlotExporter and quoted directories in %PATH% ([#446](https://github.com/dotnet/BenchmarkDotNet/issues/446)) -* Show Windows brand version in summary \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.7.md b/docs/_changelog/header/v0.10.7.md deleted file mode 100644 index e58ad7f60e..0000000000 --- a/docs/_changelog/header/v0.10.7.md +++ /dev/null @@ -1,6 +0,0 @@ -* LINQPad support (5.22.05+) ([#66](https://github.com/dotnet/BenchmarkDotNet/issues/66), [#445](https://github.com/dotnet/BenchmarkDotNet/issues/445)) -* Benchmark filters and categories ([#248](https://github.com/dotnet/BenchmarkDotNet/issues/248)) -* Updated setup/cleanup attributes: `[GlobalSetup]`, `[GlobalCleanup]`, `[IterationSetup]`, `[IterationCleanup]` ([#270](https://github.com/dotnet/BenchmarkDotNet/issues/270), [#274](https://github.com/dotnet/BenchmarkDotNet/issues/274), [#325](https://github.com/dotnet/BenchmarkDotNet/issues/325), [#456](https://github.com/dotnet/BenchmarkDotNet/issues/456)) -* Better Value Types support ([afa803d0](https://github.com/dotnet/BenchmarkDotNet/commit/afa803d0e38c0e11864b2e4394d4a85d3801d944)) -* Building Sources on Linux: it's possible to build the solution (with unloaded F#/VB projects), run samples (for both net46/netcoreapp1.1), run unit tests (for netcoreapp1.1 only) -* Fix minor bugs in `JsonExporter` ([#451](https://github.com/dotnet/BenchmarkDotNet/pull/451)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.8.md b/docs/_changelog/header/v0.10.8.md deleted file mode 100644 index c93b24ee6e..0000000000 --- a/docs/_changelog/header/v0.10.8.md +++ /dev/null @@ -1,4 +0,0 @@ -* Legend for time units ([#349](https://github.com/dotnet/BenchmarkDotNet/issues/349), [#459](https://github.com/dotnet/BenchmarkDotNet/issues/459), [f14e508e](https://github.com/dotnet/BenchmarkDotNet/commit/f14e508e44b510a26cc3ec5aed30ee7843a92baf)) -* XML exporter ([#157](https://github.com/dotnet/BenchmarkDotNet/issues/157), [#452](https://github.com/dotnet/BenchmarkDotNet/pull/452), [a0148db8](https://github.com/dotnet/BenchmarkDotNet/commit/a0148db80c518a9d255f496534a8d1666be52c69)) -* .NET Framework 4.7 support ([#461](https://github.com/dotnet/BenchmarkDotNet/issues/461), [3f2b5c3c](https://github.com/dotnet/BenchmarkDotNet/commit/3f2b5c3c134c62f34f0ecf1a9c90d91ad37f2c6a), [5513873a](https://github.com/dotnet/BenchmarkDotNet/commit/5513873ac40d07583d6136e431e3b7c8cdf6c851)) -* Public API for AllocationQuantum ([#450](https://github.com/dotnet/BenchmarkDotNet/issues/450), [#462](https://github.com/dotnet/BenchmarkDotNet/pull/462), [a0148db8](https://github.com/dotnet/BenchmarkDotNet/commit/a0148db80c518a9d255f496534a8d1666be52c69)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.9.md b/docs/_changelog/header/v0.10.9.md deleted file mode 100644 index 248e6b007e..0000000000 --- a/docs/_changelog/header/v0.10.9.md +++ /dev/null @@ -1,14 +0,0 @@ -* Migrate from custom build scripts to Cake (C# Make) ([#426](https://github.com/dotnet/BenchmarkDotNet/issues/426), [#475](https://github.com/dotnet/BenchmarkDotNet/pull/475), thanks [@Ky7m](https://github.com/Ky7m)) -* Target Setup methods for specific Benchmarks ([#469](https://github.com/dotnet/BenchmarkDotNet/issues/469), [#501](https://github.com/dotnet/BenchmarkDotNet/pull/501), thanks [@ipjohnson](https://github.com/ipjohnson)) -* Many improvements in XmlExporter ([#476](https://github.com/dotnet/BenchmarkDotNet/pull/476), [#488](https://github.com/dotnet/BenchmarkDotNet/pull/488), thanks [@Teknikaali](https://github.com/Teknikaali)) -* Add MemoryDiagnoser results to JsonExporter output ([#453](https://github.com/dotnet/BenchmarkDotNet/issues/453), [#478](https://github.com/dotnet/BenchmarkDotNet/pull/478), thanks [@Teknikaali](https://github.com/Teknikaali)) -* Detect correct version of .NET Core (+ improved presentation for information about runtime) ([#448](https://github.com/dotnet/BenchmarkDotNet/issues/448), [ed586585...ed586585](https://github.com/dotnet/BenchmarkDotNet/compare/dc6dc411b4d8703d0a1abafe64fb1e0b0a83af1f...cea199f74923c99f88f4bb4d53e37f86b10269b7)) -* Fix UnauthorizedAccessException ([#380](https://github.com/dotnet/BenchmarkDotNet/issues/380), [#390](https://github.com/dotnet/BenchmarkDotNet/issues/390), [#490](https://github.com/dotnet/BenchmarkDotNet/issues/490), [#491](https://github.com/dotnet/BenchmarkDotNet/issues/491), [8505abb5](https://github.com/dotnet/BenchmarkDotNet/commit/8505abb5416bad90cda03f4972b067f9ac44b304)) -* Fix app.config generation ([#499](https://github.com/dotnet/BenchmarkDotNet/issues/499), [dc6dc411](https://github.com/dotnet/BenchmarkDotNet/commit/dc6dc411b4d8703d0a1abafe64fb1e0b0a83af1f)) -* Fix incorrect order of IterationCleanup and Benchmark jitting ([#481](https://github.com/dotnet/BenchmarkDotNet/issues/481), [#503](https://github.com/dotnet/BenchmarkDotNet/pull/503)) -* Fix test scripts for MacOS+zsh ([1177c8](https://github.com/dotnet/BenchmarkDotNet/commit/1177c80e2dbe931439e44bb0ce2ce25cad8b9ba2)) -* Unix-related ProcessorAffinity fixes ([#474](https://github.com/dotnet/BenchmarkDotNet/issues/474), [26d44411](https://github.com/dotnet/BenchmarkDotNet/commit/26d44411ea47f28a9cc7df84b2df0ef89b2bbcf7)) -* Minor fixes in docs ([#465](https://github.com/dotnet/BenchmarkDotNet/pull/465), [#467](https://github.com/dotnet/BenchmarkDotNet/pull/467), [#473](https://github.com/dotnet/BenchmarkDotNet/pull/473), [#480](https://github.com/dotnet/BenchmarkDotNet/pull/480), [#483](https://github.com/dotnet/BenchmarkDotNet/pull/483), thanks [@mtschneiders](https://github.com/mtschneiders), [@davkean](https://github.com/davkean), [@aarondandy](https://github.com/aarondandy), [@AmadeusW](https://github.com/AmadeusW)) -* Temporary hacks for [appveyor connectivity incident](https://appveyor.statuspage.io/incidents/m2vdvw39kdk8) ([#497](https://github.com/dotnet/BenchmarkDotNet/pull/497), [#506](https://github.com/dotnet/BenchmarkDotNet/pull/506)) -* Additional warnings for incorrect Configs ([#482](https://github.com/dotnet/BenchmarkDotNet/issues/482), [eb84825f](https://github.com/dotnet/BenchmarkDotNet/commit/eb84825ff08aa5d23d2d512d4d4bde3e95ca0815)) -* Additional warnings for F# methods with spaces ([#479](https://github.com/dotnet/BenchmarkDotNet/issues/479), [3c2c8dec](https://github.com/dotnet/BenchmarkDotNet/commit/3c2c8dec28d6c570f2901001058cd9c6000e6ca2), [7ba1c809](https://github.com/dotnet/BenchmarkDotNet/commit/7ba1c809004e0b75eaa87724155480eaf623f8a9), [3ca39afe](https://github.com/dotnet/BenchmarkDotNet/commit/3ca39afe9f0d25359f9b092181beb02d57c5ad32)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.11.0.md b/docs/_changelog/header/v0.11.0.md deleted file mode 100644 index 9760c809ba..0000000000 --- a/docs/_changelog/header/v0.11.0.md +++ /dev/null @@ -1,350 +0,0 @@ -This is one of the biggest releases of BenchmarkDotNet ever. -There are so many improvements. -We have - new documentation, - many performance improvements, - Job Mutators, - better user experience, - correct Ctrl+C handling, - better generic benchmarks support, - more scenarios for passing arguments to benchmarks, - awesome support of console arguments, - unicode support, - LLVM support in MonoDisassembler, - and many-many other improvements and bug fixes! - -A big part of the features and bug fixes were implemented to meet the enterprise requirements of Microsoft to make it possible to port CoreCLR, CoreFX, and CoreFXLab to BenchmarkDotNet. - -The release would not be possible without many contributions from amazing community members. This release is a combined effort. We build BenchmarkDotNet together to make benchmarking .NET code easy and available to everyone for free! - -# New documentation - -We have many improvements in our documentation! -The new docs include: - -* [DocFX](https://dotnet.github.io/docfx/) under the hood -* Detailed changelogs which includes all commits, merged pull requests and resolved issues -* API references -* Code samples for main features: - we generate it automatically based on the `BenchmarkDotNet.Samples` project; - it means that all samples can always be compiled - (no more samples with outdated API) -* Better UI -* Documentation versioning: now it's possible to look at the documentation for recent BenchmarkDotNet versions - -# Performance improvements - -BenchmarkDotNet needs to be capable of running few thousands of CoreFX and CoreCLR benchmarks in an acceptable amount of time. The code itself was already optimized so we needed architectural and design changes to meet this requirement. - -## Generate one executable per runtime settings - -To ensure that the side effects of one benchmark run does not affect another benchmark run BenchmarkDotNet generates, builds and runs every benchmark in a dedicated process. So far we were generating and building one executable per benchmark, now we generate and build one executable per runtime settings. So if you want to run ten thousands of benchmarks for .NET Core 2.1 we are going to generate and build single executable, not ten thousand. If you target multiple runtimes the build is going to be executed in parallel. Moreover, if one of the parallel builds fail it's going to be repeated in a sequential way. - -Previously the time to generate and build 650 benchmarks from our Samples project was **one hour**. Now it's something around **13 seconds** which means **276 X** improvement for this particular scenario. You can see the changes [here](https://github.com/dotnet/BenchmarkDotNet/issues/699). - -## Don't execute long operations more than once per iteration - -BenchmarkDotNet was designed to allow for very accurate and stable micro-benchmarking. One of the techniques that we use is manual loop unrolling. In practice, it meant that for every iteration we were executing the benchmark at least 16 times (the default `UnrollFactor` value). It was of course not desired for the very time-consuming benchmarks. - -So far this feature was always enabled by default and users would need to configure `UnrollFactor=1` to disable it. Now BenchmarkDotNet is going to discover such scenario and don't perform manual loop unrolling for the very time-consuming benchmarks. BenchmarkDotNet uses `Job.IterationTime` setting (the default is 0.5s) in the Pilot Experiment stage to determine how many times given benchmark should be executed per iteration. - -Example: - -```cs -public class Program -{ - static void Main() => BenchmarkRunner.Run(); - - [Benchmark] - public void Sleep1s() => Thread.Sleep(TimeSpan.FromSeconds(1)); -} -``` - -Time to run with the previous version: **374 seconds**. With `0.11.0` it's **27 seconds** which gives us almost **14 X** improvement. A good example of benchmarks that are going to benefit from this change are computer game benchmarks and ML.NET benchmarks. You can see the changes [here](https://github.com/dotnet/BenchmarkDotNet/pull/760) and [here](https://github.com/dotnet/BenchmarkDotNet/pull/771). - -## Exposing more configuration settings - -The default settings were configured to work well with every scenario. Before running the benchmark, BenchmarkDotNet does not know anything about it. This is why it performs many warmup iterations before running the benchmarks. - -When you author benchmarks and run them many times you can come up with custom settings that produce similar results but in a shorter manner of time. To allow you to do that we have exposed: - -* `Job.MinIterationCount` (default value is 15) -* `Job.MaxIterationCount` (default value is 100) -* `Job.MinWarmupIterationCount` (default value is 6) -* `Job.MaxWarmupIterationCount` (default value is 50) - -# User Experience - -One of the biggest success factors of BenchmarkDotNet is a great user experience. The tool just works as expected and makes your life easy. We want to make it even better! - -## .NET Standard 2.0 - -We have ported BenchmarkDotNet to .NET Standard 2.0 and thanks to that we were able to not only simplify our code and build process but also merge `BenchmarkDotNet.Core.dll` and `BenchmarkDotNet.Toolchains.Roslyn.dll` into `BenchmarkDotNet.dll`. We still support .NET 4.6 but we have dropped .NET Core 1.1 support. More information and full discussion can be found [here](https://github.com/dotnet/BenchmarkDotNet/pull/688). - -**Note:** Our `BenchmarkDotNet.Diagnostics.Windows` package which uses `EventTrace` to implement ETW-based diagnosers was also ported to .NET Standard 2.0 and you can now use all the ETW diagnosers with .NET Core on Windows. We plan to add EventPipe support and make this page fully cross-platform and Unix compatible soon. - -## Using complex types as benchmark arguments - -So far we have required the users to implement `IParam` interface to make the custom complex types work as benchmark arguments/parameters. This has changed, now the users can use any complex types as arguments and it will just work ([more](https://github.com/dotnet/BenchmarkDotNet/pull/754)). - -```cs -public class Program -{ - static void Main(string[] args) => BenchmarkRunner.Run(); - - public IEnumerable Arguments() - { - yield return new Point2D(10, 200); - } - - [Benchmark] - [ArgumentsSource(nameof(Arguments))] - public int WithArgument(Point2D point) => point.X + point.Y; -} - -public class Point2D -{ - public int X, Y; - - public Point2D(int x, int y) - { - X = x; - Y = y; - } - - public override string ToString() => $"[{X},{Y}]"; -} -``` - -**Note**: If you want to control what will be displayed in the summary you should override `ToString`. - -## If IterationSetup is provided run benchmark once per iteration - -When Stephen Toub says that something is [buggy](https://github.com/dotnet/BenchmarkDotNet/issues/730), it most probably is. BenchmarkDotNet performs multiple invocations of benchmark per every iteration. When we have exposed the `[IterationSetup]` attribute many users were expecting that the `IterationSetup` is going to be invoked before every benchmark execution. - -It was invoked before every iteration, and iteration was more than one benchmark call if the user did not configure that explicitly. We have changed that and now if you provide an `[IterationSetup]` method it is going to be executed before every iteration and iteration will invoke the benchmark just once. - -```cs -public class Test -{ - public static void Main() => BenchmarkRunner.Run(); - - [IterationSetup] - public void MySetup() => Console.WriteLine("MySetup"); - - [Benchmark] - public void MyBenchmark() => Console.WriteLine("MyBenchmark"); -} -``` - -Before: - -```log -MySetup -MyBenchmark -MyBenchmark -MyBenchmark -MyBenchmark -(...) -``` - -After: - -```log -MySetup -MyBenchmark -MySetup -MyBenchmark -MySetup -MyBenchmark -(...) -``` - -**Note:** If you want to configure how many times benchmark should be invoked per iteration you can use the new `[InvocationCountAttribute]`. - -## Job Mutators - -`Job` represents a set of settings to run the benchmarks. We run every benchmark for every job defined by the user. The problem was that so far many jobs were just added to the config instead of being merged with other jobs. - -An example: - -```cs -[ClrJob, CoreJob] -[GcServer(true)] -public class MyBenchmarkClass -``` - -Resulted in 3 jobs and 3 benchmark executions: `ClrJob`, `CoreJob` and `GcServer(true)` for current runtime. - -Now all Jobs and their corresponding attributes marked as mutators are going to be applied to other jobs, not just added to the config. So in this particular scenario, the benchmarks from `MyBenchmarkClass` are going to be executed for .NET with Server GC enabled and .NET Core with Server GC enabled. - -Mutators are great when you want to have a single, global config for all benchmarks and apply given settings only to selected types. You can find out more about mutators [here](https://github.com/dotnet/BenchmarkDotNet/pull/800). - -## Ctrl+C - -When the user: - -* presses `Ctrl+C` -* presses `Ctrl+Break` -* logs off -* closes console window - -We are now going to close any existing ETW session created by BenchmarkDotNet and **restore console colors** ([read more](https://github.com/dotnet/BenchmarkDotNet/pull/761)). - -## Handle OutOfMemoryException more gracefully - -When our benchmark hits `OutOfMemoryException` we print some nice explanation: - -```cs -public class Program -{ - static void Main(string[] args) => BenchmarkRunner.Run(); - - private List list = new List(); - - [Benchmark] - public void AntiPattern() => list.Add(new int[int.MaxValue / 2]); -} -``` - -```log -OutOfMemoryException! -BenchmarkDotNet continues to run additional iterations until desired accuracy level is achieved. It's possible only if the benchmark method doesn't have any side-effects. -If your benchmark allocates memory and keeps it alive, you are creating a memory leak. -You should redesign your benchmark and remove the side-effects. You can use `OperationsPerInvoke`, `IterationSetup` and `IterationCleanup` to do that. -``` - -## Trimming long strings - -We used to display the values "as is" which was bad for long strings. Now the values are trimmed ([more](https://github.com/dotnet/BenchmarkDotNet/issues/748)). - -```cs -public class Long -{ - [Params("text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7")] - public string Text; - - [Benchmark] - public int HashCode() => Text.GetHashCode(); -} -``` - -| Method | Text | -|--------- |--------------------- | -| HashCode | text/(...)q=0.7 [86] | - -# More features - -## Generic benchmarks - -BenchmarkDotNet supports generic benchmarks, all you need to do is to tell it which types should be used as generic arguments ([read more](https://github.com/dotnet/BenchmarkDotNet/pull/758)). - -```cs -[GenericTypeArguments(typeof(int))] -[GenericTypeArguments(typeof(char))] -public class IntroGenericTypeArguments -{ - [Benchmark] public T Create() => Activator.CreateInstance(); -} -``` - -## Arguments - -We now support more scenarios for passing arguments to benchmarks: - -* passing arguments to asynchronous benchmarks ([more](https://github.com/dotnet/BenchmarkDotNet/issues/818)) -* passing generic types -* passing arguments by reference -* passing jagged arrays ([more](https://github.com/dotnet/BenchmarkDotNet/issues/769)) -* types with implicit cast operator to stack only types can be passed as given stack-only types to Benchmarks ([more](https://github.com/dotnet/BenchmarkDotNet/issues/774)) - -Example: - -```cs -public class WithStringToReadOnlySpan -{ - [Benchmark] - [Arguments("some string")] - public void AcceptsReadOnlySpan(ReadOnlySpan notString) -} -``` - -## Console Arguments - -`BenchmarkSwitcher` supports various console arguments ([PR](https://github.com/dotnet/BenchmarkDotNet/pull/824)), to make it work you need to pass the `args` to switcher: - -```cs -class Program -{ - static void Main(string[] args) - => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); -} -``` - -**Note:** to get the most up-to-date info about supported console arguments run the benchmarks with `--help`. - -### Filter - -The `--filter` or just `-f` allows you to filter the benchmarks by their full name (`namespace.typeName.methodName`) using glob patterns. - -Examples: - -1. Run all benchmarks from System.Memory namespace: `-f System.Memory*` -2. Run all benchmarks: `-f *` -3. Run all benchmarks from ClassA and ClassB `-f *ClassA* *ClassB*` - -**Note**: If you would like to **join** all the results into a **single summary**, you need to use `--join`. - -### Categories - -You can also filter the benchmarks by categories: - -* `--anyCategories` - runs all benchmarks that belong to **any** of the provided categories -* `--allCategories`- runs all benchmarks that belong to **all** provided categories - -### Diagnosers - -* `-m`, `--memory` - enables MemoryDiagnoser and prints memory statistics -* `-d`, `--disassm`- enables DisassemblyDiagnoser and exports diassembly of benchmarked code - -### Runtimes - -The `--runtimes` or just `-r` allows you to run the benchmarks for selected Runtimes. Available options are: Clr, Mono, Core and CoreRT. - -Example: run the benchmarks for .NET and .NET Core: - -```log -dotnet run -c Release -- --runtimes clr core -``` - -### More arguments - -* `-j`, `--job` (Default: Default) Dry/Short/Medium/Long or Default -* `-e`, `--exporters` GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML -* `-i`, `--inProcess` (Default: false) Run benchmarks in Process -* `-a`, `--artifacts` Valid path to accessible directory -* `--outliers` (Default: OnlyUpper) None/OnlyUpper/OnlyLower/All -* `--affinity` Affinity mask to set for the benchmark process -* `--allStats` (Default: false) Displays all statistics (min, max & more) -* `--attribute` Run all methods with given attribute (applied to class or method) - -## Other small improvements - -* **Unicode support:** - now you can enable support of Unicode symbols like `μ` or `±` with `[EncodingAttribute.Unicode]`, - an example: BenchmarkDotNet.Samples.IntroEncoding - (see [#735](https://github.com/dotnet/BenchmarkDotNet/pull/735)) -* **Better benchmark validation** - (see [#693](https://github.com/dotnet/BenchmarkDotNet/pull/693), [#737](https://github.com/dotnet/BenchmarkDotNet/pull/737)) -* **Improve .NET Framework version detection**: now we support .NET Framework 4.7.2 - (see [#743](https://github.com/dotnet/BenchmarkDotNet/pull/743)) -* **OutlierModes:** - now it's possible to control how to process outliers, - an example @BenchmarkDotNet.Samples.IntroOutliers - (see [#766](https://github.com/dotnet/BenchmarkDotNet/pull/766)) -* **LLVM support in MonoDisassembler** - (see [a7426e](https://github.com/dotnet/BenchmarkDotNet/commit/a7426e84fde075503f489fdf096a95f694f77b85)) -* **Grand API renaming** - we try not to change public API, but sometimes it's necessary because we want to get a consistent and understandable API in v1.0.0. - (see [#787](https://github.com/dotnet/BenchmarkDotNet/issues/787)) -* **Many-many small improvements and bug fixes** \ No newline at end of file diff --git a/docs/_changelog/header/v0.11.1.md b/docs/_changelog/header/v0.11.1.md deleted file mode 100644 index 97efdd26d2..0000000000 --- a/docs/_changelog/header/v0.11.1.md +++ /dev/null @@ -1,11 +0,0 @@ -This release includes some minor improvements and bug fixes: - -* Fixed `RPlotExporter` ([#852](https://github.com/dotnet/BenchmarkDotNet/issues/852), [#855](https://github.com/dotnet/BenchmarkDotNet/issues/855)). - In v0.11.0, the plot generation was broken because of the huge renaming in [#787](https://github.com/dotnet/BenchmarkDotNet/issues/787). -* [ArgumentsSource](xref:BenchmarkDotNet.Samples.IntroArgumentsSource) now supports additional types like - `Type` ([#840](https://github.com/dotnet/BenchmarkDotNet/issues/840)), - `BigInteger` ([#850](https://github.com/dotnet/BenchmarkDotNet/issues/850)), - `DateTime` ([#853](https://github.com/dotnet/BenchmarkDotNet/issues/853)), - and special double values like `double.NaN` ([#851](https://github.com/dotnet/BenchmarkDotNet/issues/851)) -* Generated projects ignore Directory.Build.props and Directory.Build.targets files [#854](https://github.com/dotnet/BenchmarkDotNet/pull/854) -* Now it's possible to run benchmarks with CoreRun ([de152c](https://github.com/dotnet/BenchmarkDotNet/commit/de152c7acc71eddeaa304c846cc67e6a54ca7a0f), [#857](https://github.com/dotnet/BenchmarkDotNet/pull/857)) diff --git a/docs/_changelog/header/v0.11.2.md b/docs/_changelog/header/v0.11.2.md deleted file mode 100644 index 2e6c435a92..0000000000 --- a/docs/_changelog/header/v0.11.2.md +++ /dev/null @@ -1,657 +0,0 @@ -This release includes many PRs from the Hacktoberfest. -We are very grateful to all the contributors who spent their time to help us make BenchmarkDotNet even better! - -## Highlights - -In this release, we have many improvements in different areas: - -* **Diagnosers** - * EtwProfiler (allows profiling benchmarks on Windows and exporting the data to a trace file) -* **Execution:** - * Comparing NuGet packages (now it's possible to compare different versions of the same package) - * .NET Core 3.0 support - * Deferred Execution Validator -* **Command-line:** - * `--list`: List of benchmarks - * `--info`: Print environment info - * `--runtimes`: Choosing execution runtimes (`--runtimes net472 netcoreapp2.1` will executed a benchmark on .NET 4.7.2 and .NET Core 2.1) - * Options for number of invocations and iterations - * Custom default settings for console argument parser - * Case-insensitive filter - * Benchmarking with different CoreRun instances - * Hardware counters command-line support -* **Exporters:** - * Markdown output for DisassemblyDiagnoser - * Diff view for disassembler output - * Improved LINQPad support (colored monospaced logs) - * Better CPU brand strings -* **Attributes:** - * Async `[GlobalSetup]` and `[GlobalCleanup]` support - * Introduced `[ParamsAllValues]` - * Selecting Baseline across Methods and Jobs -* **Statistics:** - * Better statistical tests (Welch's t-test and Mann-Whitney U-test) - * ZeroMeasurementAnalyser - * RatioColumn -* **Other:** - * Azure Pipelines support for internal builds - * Many minor bug fixes - * Improved documentation - * New tests - ---- - -## Diagnosers - -### EtwProfiler - -`EtwProfiler` allows to profile the benchmarked .NET code on Windows and exports the data to a trace file which can be opened with [PerfView](https://github.com/Microsoft/perfview) or [Windows Performance Analyzer](https://learn.microsoft.com/windows-hardware/test/wpt/windows-performance-analyzer). - -`EtwProfiler` uses `TraceEvent` library which internally uses Event Tracing for Windows (ETW) to capture stack traces and important .NET Runtime events. -Before the process with benchmarked code is started, EtwProfiler starts User and Kernel ETW sessions. Every session writes data to it's own file and captures different data. User session listens for the .NET Runtime events (GC, JIT etc) while the Kernel session gets CPU stacks and Hardware Counter events. After this, the process with benchmarked code is started. During the benchmark execution all the data is captured and written to a trace file. Moreover, BenchmarkDotNet Engine emits it's own events to be able to differentiate jitting, warmup, pilot and actual workload when analyzing the trace file. When the benchmarking is over, both sessions are closed and the two trace files are merged into one. - -![](https://adamsitnik.com/images/etwprofiler/flamegraph.png) - -You can find more details - in the [documentation](xref:docs.etwprofiler) and - in the [blog post](https://adamsitnik.com/ETW-Profiler/) by Adam Sitnik. - -* [#878](https://github.com/dotnet/BenchmarkDotNet/pull/878) EtwProfiler Diagnoser (by [@adamsitnik](https://github.com/adamsitnik)) -* [04a715](https://github.com/dotnet/BenchmarkDotNet/commit/04a71586206a822bca56f0abdacefdc2e5fc1b01) EtwProfiler Diagnoser (#878) (by [@adamsitnik](https://github.com/adamsitnik)) - ---- - -## Execution - -### Comparing NuGet packages - -Now it's possible to compare performance of several versions of the same NuGet package. -An example: - -```cs -[Config(typeof(Config))] -public class IntroNuGet -{ - // Specify jobs with different versions of the same NuGet package to benchmark. - // The NuGet versions referenced on these jobs must be greater or equal to the - // same NuGet version referenced in this benchmark project. - // Example: This benchmark project references Newtonsoft.Json 9.0.1 - private class Config : ManualConfig - { - public Config() - { - var baseJob = Job.MediumRun.With(CsProjCoreToolchain.Current.Value); - Add(baseJob.WithNuGet("Newtonsoft.Json", "11.0.2").WithId("11.0.2")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "11.0.1").WithId("11.0.1")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "10.0.3").WithId("10.0.3")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "10.0.2").WithId("10.0.2")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "10.0.1").WithId("10.0.1")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "9.0.1").WithId("9.0.1")); - } - } - - [Benchmark] - public void SerializeAnonymousObject() - => JsonConvert.SerializeObject( - new { hello = "world", price = 1.99, now = DateTime.UtcNow }); -} -``` - -See also: @BenchmarkDotNet.Samples.IntroNuGet - -* [#290](https://github.com/dotnet/BenchmarkDotNet/issues/290) Question: Any official way to benchmark same method between different assembly versions? -* [#931](https://github.com/dotnet/BenchmarkDotNet/issues/931) Same NuGet version used when benchmarking different packages -* [#922](https://github.com/dotnet/BenchmarkDotNet/pull/922) Enables benchmarking betweeen different Nuget packages (by [@Shazwazza](https://github.com/Shazwazza)) -* [#932](https://github.com/dotnet/BenchmarkDotNet/pull/932) Partition benchmark run info based on added nuget packages (by [@blairconrad](https://github.com/blairconrad)) -* [92a786](https://github.com/dotnet/BenchmarkDotNet/commit/92a7869aaa30aeacaf1da2dcc45bc65c8333ae73) Enables benchmarking betweeen different Nuget packages (#922) fixes #290 (by [@Shazwazza](https://github.com/Shazwazza)) -* [510685](https://github.com/dotnet/BenchmarkDotNet/commit/510685f48ce2baf57682aa82e18c6486989e9625) Partition benchmark run info based on added nuget packages (#932) (by [@blairconrad](https://github.com/blairconrad)) -* [cf84a4](https://github.com/dotnet/BenchmarkDotNet/commit/cf84a44d108d5bf3860129e0a2a78cace9c95626) NuGet casing fix (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - -### .NET Core 3.0 support - -Now it's possible to run benchmarks on preview versions of .NET Core 3.0. - -* [#874](https://github.com/dotnet/BenchmarkDotNet/issues/874) .NET Core 3.0 support (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [2e398c](https://github.com/dotnet/BenchmarkDotNet/commit/2e398c89561b3b1c89ec64b94f656ae20236efd1) detect .NET Core 3.0 and use the appropriate target framework moniker, fixes ... (by [@adamsitnik](https://github.com/adamsitnik)) - -### Deferred Execution Validator - -In LINQ, execution of a query is usually [deferred](https://learn.microsoft.com/dotnet/standard/linq/deferred-execution-example) until the moment when you actually request the data. If your benchmark just returns `IEnumerable` or `IQueryable` it's not measuring the execution of the query, just the creation. - -This is why we decided to warn you about this issue whenever it happens: - -```log -Benchmark IntroDeferredExecution.Wrong returns a deferred execution result (IEnumerable). You need to either change the method declaration to return a materialized result or consume it on your own. You can use .Consume() extension method to do that. -``` - -Don't worry! We are also providing you with a `Consume` extension method which can execute given `IEnumerable` or `IQueryable` and consume its results. All you need to do is to create a [`Consumer`](xref:BenchmarkDotNet.Engines.Consumer) instance, preferably store it in a field (to exclude the cost of creating Consumer from the benchmark itself) and pass it to `Consume` extension method. - -**Do not call `.ToArray()` because it's an expensive operation and it might dominate given benchmark!** - -See also: @BenchmarkDotNet.Samples.IntroDeferredExecution - -* [#858](https://github.com/dotnet/BenchmarkDotNet/issues/858) Should the Engine iterate over and consume IEnumerable and IQueryable results? (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [cebe2a](https://github.com/dotnet/BenchmarkDotNet/commit/cebe2a0f84fa21acb6db9613fe3a4326d635f129) Deferred Execution Validator, fixes #858 (by [@adamsitnik](https://github.com/adamsitnik)) - ---- - -## Command-line - -In this release, we have tons of improvements for command-line experience. - -### `--list`: List of benchmarks - -The `--list` allows you to print all of the available benchmark names. Available options are: - -* `flat` - prints list of the available benchmarks: `--list flat` -```ini -BenchmarkDotNet.Samples.Algo_Md5VsSha256.Md5 -BenchmarkDotNet.Samples.Algo_Md5VsSha256.Sha256 -BenchmarkDotNet.Samples.IntroArguments.Benchmark -BenchmarkDotNet.Samples.IntroArgumentsSource.SingleArgument -BenchmarkDotNet.Samples.IntroArgumentsSource.ManyArguments -BenchmarkDotNet.Samples.IntroArrayParam.ArrayIndexOf -BenchmarkDotNet.Samples.IntroArrayParam.ManualIndexOf -BenchmarkDotNet.Samples.IntroBasic.Sleep -[...] -``` -* `tree` - prints tree of the available benchmarks: `--list tree` -```ini -BenchmarkDotNet - └─Samples - ├─Algo_Md5VsSha256 - │ ├─Md5 - │ └─Sha256 - ├─IntroArguments - │ └─Benchmark - ├─IntroArgumentsSource - │ ├─SingleArgument - │ └─ManyArguments - ├─IntroArrayParam - │ ├─ArrayIndexOf - │ └─ManualIndexOf - ├─IntroBasic - │ ├─Sleep -[...] -``` - -The `--list` option works with the `--filter` option. Examples: - -* `--list flat --filter *IntroSetupCleanup*` prints: -```ini -BenchmarkDotNet.Samples.IntroSetupCleanupGlobal.Logic -BenchmarkDotNet.Samples.IntroSetupCleanupIteration.Benchmark -BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkA -BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkB -BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkC -BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkD -``` -* `--list tree --filter *IntroSetupCleanup*` prints: -```ini -BenchmarkDotNet - └─Samples - ├─IntroSetupCleanupGlobal - │ └─Logic - ├─IntroSetupCleanupIteration - │ └─Benchmark - └─IntroSetupCleanupTarget - ├─BenchmarkA - ├─BenchmarkB - ├─BenchmarkC - └─BenchmarkD -``` - -* [#905](https://github.com/dotnet/BenchmarkDotNet/issues/905) Implement `--list` -* [#914](https://github.com/dotnet/BenchmarkDotNet/pull/914) Implement `--list` - fixes #905 (by [@wojtpl2](https://github.com/wojtpl2)) -* [#916](https://github.com/dotnet/BenchmarkDotNet/pull/916) Update console-args.md - add information about `--list` option (by [@wojtpl2](https://github.com/wojtpl2)) -* [330f66](https://github.com/dotnet/BenchmarkDotNet/commit/330f66c3a3d94d1369d5c0b629bbb0085d5db8eb) Implement `--list` - fixes #905 (#914) (by [@wojtpl2](https://github.com/wojtpl2)) -* [6c7521](https://github.com/dotnet/BenchmarkDotNet/commit/6c7521d4fd6776098667944321c8a65848382ae5) Update console-args.md - add information about `--list` option (#916) (by [@wojtpl2](https://github.com/wojtpl2)) - -### `--info`: Print environment info - -Some of our users really like the info we print about hardware and OS. -Now we have the `--info` console line argument which does not run the benchmarks, but simply prints the info. - -```ini -BenchmarkDotNet=v0.11.1.786-nightly, OS=Windows 10.0.17134.285 (1803/April2018Update/Redstone4) -Intel Xeon CPU E5-1650 v4 3.60GHz, 1 CPU, 12 logical and 6 physical cores -Frequency=3507500 Hz, Resolution=285.1033 ns, Timer=TSC -.NET Core SDK=3.0.100-alpha1-009642 - [Host] : .NET Core 3.0.0-preview1-27004-04 (CoreCLR 4.6.27003.04, CoreFX 4.6.27003.02), 64bit RyuJIT -``` - -* [#904](https://github.com/dotnet/BenchmarkDotNet/issues/904) Implement `--info` -* [#907](https://github.com/dotnet/BenchmarkDotNet/pull/907) fixes #904 Implement `--info` (by [@lahma](https://github.com/lahma)) -* [4be28d](https://github.com/dotnet/BenchmarkDotNet/commit/4be28d25fa9ab79ca194c615783148042738bdad) fixes #904 Implement `--info` (#907) (by [@lahma](https://github.com/lahma)) - -### `--runtimes`: Choosing execution runtimes - -The `--runtimes` or just `-r` allows you to run the benchmarks for selected Runtimes. Available options are: Mono, CoreRT, Core, Clr net46, net461, net462, net47, net471, net472, netcoreapp2.0, netcoreapp2.1, netcoreapp2.2, netcoreapp3.0. - -Example: run the benchmarks for .NET 4.7.2 and .NET Core 2.1: - -```log -dotnet run -c Release -- --runtimes net472 netcoreapp2.1 -``` - -* [#913](https://github.com/dotnet/BenchmarkDotNet/pull/913) .NET Core Toolchains improvements (by [@adamsitnik](https://github.com/adamsitnik)) -* [0f721c](https://github.com/dotnet/BenchmarkDotNet/commit/0f721c8e0e100fc951a54b6045eb7b58c55c2a1f) make it possible to specify runtimes using explicit tfms like net472 or netco... (by [@adamsitnik](https://github.com/adamsitnik)) -* [1c581e](https://github.com/dotnet/BenchmarkDotNet/commit/1c581e5bf5b4ba9f40d113ae09e0731a60523a60) .NET Core Toolchains improvements (#913) (by [@adamsitnik](https://github.com/adamsitnik)) - -### Options for number of invocations and iterations - -* `--launchCount` - how many times we should launch process with target benchmark. The default is 1. -* `--warmupCount` - how many warmup iterations should be performed. If you set it, the minWarmupCount and maxWarmupCount are ignored. By default calculated by the heuristic. -* `--minWarmupCount` - minimum count of warmup iterations that should be performed. The default is 6. -* `--maxWarmupCount` - maximum count of warmup iterations that should be performed. The default is 50. -* `--iterationTime` - desired time of execution of an iteration. Used by Pilot stage to estimate the number of invocations per iteration. 500ms by default. -* `--iterationCount` - how many target iterations should be performed. By default calculated by the heuristic. -* `--minIterationCount` - minimum number of iterations to run. The default is 15. -* `--maxIterationCount` - maximum number of iterations to run. The default is 100. -* `--invocationCount` - invocation count in a single iteration. By default calculated by the heuristic. -* `--unrollFactor` - how many times the benchmark method will be invoked per one iteration of a generated loop. 16 by default -* `--runOncePerIteration` - run the benchmark exactly once per iteration. False by default. - -Example: run single warmup iteration, from 9 to 12 actual workload iterations. - -```log -dotnet run -c Release -- --warmupCount 1 --minIterationCount 9 --maxIterationCount 12 -``` - -* [#902](https://github.com/dotnet/BenchmarkDotNet/pull/902) More command line args (by [@adamsitnik](https://github.com/adamsitnik)) -* [ba0d22](https://github.com/dotnet/BenchmarkDotNet/commit/ba0d22b41fd25022e3a945fe5ef1ae8aea697cf7) allow to configure the number of invocations and iterations from command line (by [@adamsitnik](https://github.com/adamsitnik)) - -### Custom default settings for console argument parser - -If you want to have a possibility to specify custom default Job settings programmatically and optionally overwrite it with console line arguments, then you should create a global config with single job marked as `.AsDefault` and pass it to `BenchmarkSwitcher` together with the console line arguments. - -Example: run single warmup iteration by default. - -```cs -static void Main(string[] args) - => BenchmarkSwitcher - .FromAssembly(typeof(Program).Assembly) - .Run(args, GetGlobalConfig()); - -static IConfig GetGlobalConfig() - => DefaultConfig.Instance - .With(Job.Default - .WithWarmupCount(1) - .AsDefault()); // the KEY to get it working -``` - -Now, the default settings are: `WarmupCount=1` but you might still overwrite it from console args like in the example below: - -```log -dotnet run -c Release -- --warmupCount 2 -``` - -### Case-insensitive filter - -The `--filter` or just `-f` allows you to filter the benchmarks by their full name (`namespace.typeName.methodName`) using glob patterns. - -Examples: - -1. Run all benchmarks from System.Memory namespace: `-f System.Memory*` -2. Run all benchmarks: `-f *` -3. Run all benchmarks from ClassA and ClassB `-f *ClassA* *ClassB*` - -Now this filter expression is case-insensitive. - -* [#864](https://github.com/dotnet/BenchmarkDotNet/issues/864) Make the filter case insensitive (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [106777](https://github.com/dotnet/BenchmarkDotNet/commit/106777f7f575a8535f16292f1de80e8ffba2853a) make the filter case insensitive invariant culture, fixes #864 (by [@adamsitnik](https://github.com/adamsitnik)) - -### Benchmarking with different CoreRun instances - -CoreRun is a simpler version of `dotnet run`, used for developing CoreCLR and CoreFX. - -Typically when working on the performance of .NET Core a developer has more than 1 copy of CoreRun. -Example: CoreRun before my changes, and after my changes. -This change allows to simply run same benchmark for few different CoreRuns to compare the perf in easy way. - -Sample usage: - -```log -dotnet run -c Release -f netcoreapp2.1 -- -f *Empty.method --job dry --coreRun -C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root\CoreRun.exe -C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root_beforeMyChanges\CoreRun.exe -``` - -Sample output: - -![image](https://user-images.githubusercontent.com/6011991/47417561-860f5800-d778-11e8-8dc6-b4e3586a9c90.png) - -* [#925](https://github.com/dotnet/BenchmarkDotNet/issues/925) Make it possible to run the benchmark with multiple CoreRun.exe (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [901616](https://github.com/dotnet/BenchmarkDotNet/commit/90161654725efd5639e0190638a3383d6a49e34c) when user provides CoreRun path and runtime in explicit way, we should use th... (by [@adamsitnik](https://github.com/adamsitnik)) -* [46bebf](https://github.com/dotnet/BenchmarkDotNet/commit/46bebf1497d4e9314c6dfd2d4e10df81332aa4fa) allow the users to run the same benchmarks using few different CoreRun.exe, f... (by [@adamsitnik](https://github.com/adamsitnik)) - -### Hardware counters command-line support - -```log ---counters CacheMisses+InstructionRetired -``` - -* [1e3df7](https://github.com/dotnet/BenchmarkDotNet/commit/1e3df74b2f927f541bed723f65c2d571fa850c53) make it possible to specify hardware counters from command line (by [@adamsitnik](https://github.com/adamsitnik)) -* [a4f91a](https://github.com/dotnet/BenchmarkDotNet/commit/a4f91a392675e4851a785095af162b977d249ba3) better handling of edge cases for parsing hardware counters from the console ... (by [@adamsitnik](https://github.com/adamsitnik)) - ---- - -## Exporters - -### Markdown output for DisassemblyDiagnoser - -Now `DisassemblyDiagnoser` generates markdown version of the assembly listing. - -* [#560](https://github.com/dotnet/BenchmarkDotNet/issues/560) Suggestion: markdown output for DisassemblyDiagnoser (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [1e6235](https://github.com/dotnet/BenchmarkDotNet/commit/1e62355f209a25c7a33f9ab7e7e03b0afe7d851f) github markdown exporter for Disassembler, fixes #560 (by [@adamsitnik](https://github.com/adamsitnik)) - -### Diff view for disassembler output - -Now we have `PrettyGithubMarkdownDiffDisassemblyExporter` which can generates - nice diffs between assembly listings. -This mode can be activated via the `--disasmDiff` command line argument or - the `printDiff: true` argument of `DisassemblyDiagnoserConfig`. -An output example (Diff between SumLocal and SumField on .NET Core 2.1.4 (CoreCLR 4.6.26814.03, CoreFX 4.6.26814.02), 64bit RyuJIT) - -```diff --; BenchmarkDotNet.Samples.IntroDisassemblyRyuJit.SumLocal() -- var local = field; // we use local variable that points to the field -- ^^^^^^^^^^^^^^^^^^ -- mov rax,qword ptr [rcx+8] -+; BenchmarkDotNet.Samples.IntroDisassemblyRyuJit.SumField() - int sum = 0; - ^^^^^^^^^^^^ -- xor edx,edx -- for (int i = 0; i < local.Length; i++) -+ xor eax,eax -+ for (int i = 0; i < field.Length; i++) - ^^^^^^^^^ -- xor ecx,ecx -- for (int i = 0; i < local.Length; i++) -+ xor edx,edx -+ for (int i = 0; i < field.Length; i++) - ^^^^^^^^^^^^^^^^ -- mov r8d,dword ptr [rax+8] -- test r8d,r8d -+ mov rcx,qword ptr [rcx+8] -+ cmp dword ptr [rcx+8],0 - jle M00_L01 -- sum += local[i]; -+ sum += field[i]; - ^^^^^^^^^^^^^^^^ - M00_L00: -- movsxd r9,ecx -- add edx,dword ptr [rax+r9*4+10h] -- for (int i = 0; i < local.Length; i++) -+ mov r8,rcx -+ cmp edx,dword ptr [r8+8] -+ jae 00007ff9`0c412c1f -+ movsxd r9,edx -+ add eax,dword ptr [r8+r9*4+10h] -+ for (int i = 0; i < field.Length; i++) - ^^^ -- inc ecx -- cmp r8d,ecx -+ inc edx -+ cmp dword ptr [rcx+8],edx - jg M00_L00 - return sum; - ^^^^^^^^^^^ - M00_L01: -- mov eax,edx --; Total bytes of code 34 -+ add rsp,28h -+; Total bytes of code 42 -``` - -* [#544](https://github.com/dotnet/BenchmarkDotNet/issues/544) Diff view for disassembler output (assignee: [@wojtpl2](https://github.com/wojtpl2)) -* [#927](https://github.com/dotnet/BenchmarkDotNet/pull/927) Improve Disassembly exporters and add PrettyGithubMarkdownDiffDisassemblyExporter (by [@wojtpl2](https://github.com/wojtpl2)) -* [#936](https://github.com/dotnet/BenchmarkDotNet/issues/936) Producing the asm diff reports on demand -* [#937](https://github.com/dotnet/BenchmarkDotNet/pull/937) Producing the asm diff reports on demand - fix for #936 (by [@wojtpl2](https://github.com/wojtpl2)) -* [1903a1](https://github.com/dotnet/BenchmarkDotNet/commit/1903a1bd96d207ed51611d1dc546920f5bfb0d86) Improve Disassembly exporters and add PrettyGithubMarkdownDiffDisassemblyExpo... (by [@wojtpl2](https://github.com/wojtpl2)) -* [dd103b](https://github.com/dotnet/BenchmarkDotNet/commit/dd103b60a4af0d3b9e7efb523c0923e7cbd8b62d) Producing the asm diff reports on demand - fixes #936 (#937) (by [@wojtpl2](https://github.com/wojtpl2)) - -### Improved LINQPad support - -If you run BenchmarkDotNet v0.11.2+ in LINQPad, your logs will be colored and monospaced: - -![linqpad](https://user-images.githubusercontent.com/2259237/47839043-4ec92880-ddc2-11e8-838d-960f6d5449e5.png) - -* [#447](https://github.com/dotnet/BenchmarkDotNet/issues/447) Implement ColoredLogger for LinqPad -* [#903](https://github.com/dotnet/BenchmarkDotNet/pull/903) Add LINQPad logging (by [@bgrainger](https://github.com/bgrainger)) -* [#915](https://github.com/dotnet/BenchmarkDotNet/pull/915) Use a monospaced font for LINQPad logging output (by [@bgrainger](https://github.com/bgrainger)) -* [c3b609](https://github.com/dotnet/BenchmarkDotNet/commit/c3b6095b933b132c1773ced3af126f282465b980) Add LINQPad logging (#903) (by [@bgrainger](https://github.com/bgrainger)) -* [10fdd0](https://github.com/dotnet/BenchmarkDotNet/commit/10fdd0998b46c4358f6fa38aacc21e57a7730724) Use a monospaced font for LINQPad logging output. (#915) (by [@bgrainger](https://github.com/bgrainger)) - -### Better CPU brand strings - -We did a lot of changes which improve the presentation form of the CPU brand string. -Here is an example of such string in the previous version of BenchmarkDotNet: - -```log -AMD Ryzen 7 2700X Eight-Core Processor (Max: 4.10GHz), 1 CPU, 16 logical and 8 physical cores -``` - -Now it becomes: - -```log -AMD Ryzen 7 2700X 4.10GHz, 1 CPU, 16 logical and 8 physical cores -``` - -As you can see, "Eight-Core Processor" was removed (because we already have "8 physical cores"); - "(Max: 4.10GHz)" was replaced by 4.10GHz (because the original CPU brand string doesn't contain the nominal frequency). - -* [#859](https://github.com/dotnet/BenchmarkDotNet/issues/859) Strange max frequency values on Windows (assignee: [@Rizzen](https://github.com/Rizzen)) -* [#909](https://github.com/dotnet/BenchmarkDotNet/issues/909) Improve CPU Brand Strings without frequency -* [#860](https://github.com/dotnet/BenchmarkDotNet/pull/860) Fix strange CPU Frequency values (by [@Rizzen](https://github.com/Rizzen)) -* [#910](https://github.com/dotnet/BenchmarkDotNet/pull/910) Simplify AMD Ryzen CPU brand info (by [@lahma](https://github.com/lahma)) -* [a78b38](https://github.com/dotnet/BenchmarkDotNet/commit/a78b38b0e89d04ad3fe8934162c7adb42f81eabe) Fix strange CPU Frequency values (#860) (by [@Rizzen](https://github.com/Rizzen)) -* [5df1e6](https://github.com/dotnet/BenchmarkDotNet/commit/5df1e6434b791eb5da6f6ef42505fc6a94ebd008) Simplify AMD Ryzen CPU brand info (#910) (by [@lahma](https://github.com/lahma)) - ---- - -## Attributes - -### Async GlobalSetup and GlobalCleanup - -Now GlobalSetup and GlobalCleanup methods can be async. - -See also: docs.setup-and-cleanup - -* [#521](https://github.com/dotnet/BenchmarkDotNet/issues/521) Support async Setup/Cleanup -* [#892](https://github.com/dotnet/BenchmarkDotNet/pull/892) Added support for async GlobalSetup. (by [@dlemstra](https://github.com/dlemstra)) -* [#923](https://github.com/dotnet/BenchmarkDotNet/pull/923) async GlobalCleanup support (by [@dlemstra](https://github.com/dlemstra)) -* [#926](https://github.com/dotnet/BenchmarkDotNet/pull/926) Added support for async GlobalCleanup. (by [@dlemstra](https://github.com/dlemstra)) -* [e0f7a6](https://github.com/dotnet/BenchmarkDotNet/commit/e0f7a67681860ead87cef76fa0db349460b34eb0) Added support for async GlobalSetup. (#892) (by [@dlemstra](https://github.com/dlemstra)) -* [a971a4](https://github.com/dotnet/BenchmarkDotNet/commit/a971a435ce6e6ca25d246e5e2cd56c5b2cf4739d) async GlobalCleanup support (#923) (by [@dlemstra](https://github.com/dlemstra)) -* [e4c7b8](https://github.com/dotnet/BenchmarkDotNet/commit/e4c7b852e5593bb280881e28ece51d26687c5ba9) Added support for async GlobalCleanup. (#926), fixes #521 (by [@dlemstra](https://github.com/dlemstra)) - -### Introduced ParamsAllValues - -If you want to use all possible values of an `enum` or another type with a small number of values, you can use the [`[ParamsAllValues]`](xref:BenchmarkDotNet.Attributes.ParamsAllValuesAttribute) attribute, instead of listing all the values by hand. The types supported by the attribute are: - -* `bool` -* any `enum` that is not marked with `[Flags]` -* `Nullable`, where `T` is an enum or boolean - -An example: - -```cs -public class IntroParamsAllValues -{ - public enum CustomEnum - { - A, - BB, - CCC - } - - [ParamsAllValues] - public CustomEnum E { get; set; } - - [ParamsAllValues] - public bool? B { get; set; } - - [Benchmark] - public void Benchmark() - { - Thread.Sleep( - E.ToString().Length * 100 + - (B == true ? 20 : B == false ? 10 : 0)); - } -} -``` - -Output: - -```markdown - Method | E | B | Mean | Error | ----------- |---- |------ |---------:|------:| - Benchmark | A | ? | 101.9 ms | NA | - Benchmark | A | False | 111.9 ms | NA | - Benchmark | A | True | 122.3 ms | NA | - Benchmark | BB | ? | 201.5 ms | NA | - Benchmark | BB | False | 211.8 ms | NA | - Benchmark | BB | True | 221.4 ms | NA | - Benchmark | CCC | ? | 301.8 ms | NA | - Benchmark | CCC | False | 312.3 ms | NA | - Benchmark | CCC | True | 322.2 ms | NA | - -// * Legends * - E : Value of the 'E' parameter - B : Value of the 'B' parameter -``` - -* [#658](https://github.com/dotnet/BenchmarkDotNet/issues/658) [Params] for enums should include all values by default -* [#908](https://github.com/dotnet/BenchmarkDotNet/pull/908) Added [ParamsAllValues] (by [@gsomix](https://github.com/gsomix)) -* [922dff](https://github.com/dotnet/BenchmarkDotNet/commit/922dfff62d6cf6fd808865e705a09eee63690a2e) Added [ParamsAllValues] (#908), fixes #658 (by [@gsomix](https://github.com/gsomix)) -* [846d08](https://github.com/dotnet/BenchmarkDotNet/commit/846d0863b6456d3e1e6ccab06d8e61c5cd064194) ParamsAllValuesValidator fixes (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - -### Selecting Baseline across Methods and Jobs - -Now it's possible to mark a method and a job as baselines at the same time: - -```cs -public class TheBaselines -{ - [Benchmark(Baseline = true)] - public void Sleep100ms() => Thread.Sleep(TimeSpan.FromMilliseconds(100)); - - [Benchmark] - public void Sleep50ms() => Thread.Sleep(TimeSpan.FromMilliseconds(50)); -} - -static void Main(string[] args) - => BenchmarkSwitcher - .FromTypes(new[] { typeof(TheBaselines) }) - .Run(args, - DefaultConfig.Instance - .With(Job.Core.AsBaseline()) - .With(Job.Clr.WithId("CLR 4.7.2"))); -``` - -* [#880](https://github.com/dotnet/BenchmarkDotNet/issues/880) Select Baseline across Methods and Jobs (assignee: [@AndreyAkinshin](https://github.com/AndreyAkinshin)) -* [21a007](https://github.com/dotnet/BenchmarkDotNet/commit/21a0073cc8b486f41b2e84deafacd00a1303013a) Support method-job baseline pairs, fixes #880 (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - ---- - -## Statistics - -### Statistical tests - -In this release, statistical testing was significantly improved. -Now it's possible to compare all benchmarks against baseline with the help - of Welch's t-test or Mann–Whitney U test. - -An example: - -```cs -[StatisticalTestColumn( - StatisticalTestKind.Welch, ThresholdUnit.Microseconds, 1, true)] -[StatisticalTestColumn( - StatisticalTestKind.MannWhitney, ThresholdUnit.Microseconds, 1, true)] -[StatisticalTestColumn( - StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.03, true)] -[StatisticalTestColumn( - StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.03, true)] -[SimpleJob(warmupCount: 0, targetCount: 5)] -public class IntroStatisticalTesting -{ - [Benchmark] public void Sleep50() => Thread.Sleep(50); - [Benchmark] public void Sleep97() => Thread.Sleep(97); - [Benchmark] public void Sleep99() => Thread.Sleep(99); - [Benchmark(Baseline = true)] public void Sleep100() => Thread.Sleep(100); - [Benchmark] public void Sleep101() => Thread.Sleep(101); - [Benchmark] public void Sleep103() => Thread.Sleep(103); - [Benchmark] public void Sleep150() => Thread.Sleep(150); -} -``` - -Output: - -| Method | Mean | Error | StdDev | Ratio | Welch(1us)/p-values | Welch(3%)/p-values | MannWhitney(1us)/p-values | MannWhitney(3%)/p-values | -|--------- |----------:|----------:|----------:|------:|---------------------- |---------------------- |-------------------------- |------------------------- | -| Sleep50 | 53.13 ms | 0.5901 ms | 0.1532 ms | 0.51 | Faster: 1.0000/0.0000 | Faster: 1.0000/0.0000 | Faster: 1.0000/0.0040 | Faster: 1.0000/0.0040 | -| Sleep97 | 100.07 ms | 0.9093 ms | 0.2361 ms | 0.97 | Faster: 1.0000/0.0000 | Same: 1.0000/0.1290 | Faster: 1.0000/0.0040 | Same: 1.0000/0.1111 | -| Sleep99 | 102.23 ms | 2.4462 ms | 0.6353 ms | 0.99 | Faster: 0.9928/0.0072 | Same: 1.0000/0.9994 | Faster: 0.9960/0.0079 | Same: 1.0000/1.0000 | -| Sleep100 | 103.34 ms | 0.8180 ms | 0.2124 ms | 1.00 | Base: 0.5029/0.5029 | Base: 1.0000/1.0000 | Base: 0.7262/0.7262 | Base: 1.0000/1.0000 | -| Sleep101 | 103.73 ms | 2.1591 ms | 0.5607 ms | 1.00 | Same: 0.1041/0.8969 | Same: 0.9999/1.0000 | Same: 0.1111/0.9246 | Same: 1.0000/1.0000 | -| Sleep103 | 106.21 ms | 1.2511 ms | 0.3249 ms | 1.03 | Slower: 0.0000/1.0000 | Same: 0.9447/1.0000 | Slower: 0.0040/1.0000 | Same: 0.9246/1.0000 | -| Sleep150 | 153.16 ms | 3.4929 ms | 0.9071 ms | 1.48 | Slower: 0.0000/1.0000 | Slower: 0.0000/1.0000 | Slower: 0.0040/1.0000 | Slower: 0.0040/1.0000 | - -```log -// * Legends * - Mean : Arithmetic mean of all measurements - Error : Half of 99.9% confidence interval - StdDev : Standard deviation of all measurements - Ratio : Mean of the ratio distribution ([Current]/[Baseline]) - Welch(1us)/p-values : Welch-based TOST equivalence test with 1 us threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' - Welch(3%)/p-values : Welch-based TOST equivalence test with 3% threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' - MannWhitney(1us)/p-values : MannWhitney-based TOST equivalence test with 1 us threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' - MannWhitney(3%)/p-values : MannWhitney-based TOST equivalence test with 3% threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' - 1 ms : 1 Millisecond (0.001 sec) -``` - -The statistical testing is a work-in-progress feature. -In future versions of BenchmarkDotNet, we are planning to improve API, fill missed docs, and introduce more parameters for customization. - -See also: @BenchmarkDotNet.Samples.IntroStatisticalTesting - -* [60eca0](https://github.com/dotnet/BenchmarkDotNet/commit/60eca005326970202a33891e5aecd2ef6b7e4cd0) Threshold API for WelchTTest; Improve Student accuracy for small n (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) -* [05cc8d](https://github.com/dotnet/BenchmarkDotNet/commit/05cc8d15ef88e382bbb1827d766d7275c3e42abd) Statistical testing improvements (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - -### ZeroMeasurementAnalyser - -When you have an empty benchmark like this - -```cs -[Benchmark] -public void Empty() { } -``` - -The expected duration of this method is zero. -However, you can get the mean value like `0.0023ns` because of the natural noise. -It's a pretty confusing result for many developers. -Since v0.11.2, we have `ZeroMeasurementAnalyser` which warn you about such methods. -By default, BenchmarkDotNet automatically evaluate overhead. -In this case, `ZeroMeasurementAnalyser` runs Welch's t-test and compare actual and overhead measurements. -If the overhead evaluation is disabled, it runs one-sample Student's t-test against a half of CPU cycle. - -* [#906](https://github.com/dotnet/BenchmarkDotNet/pull/906) Zero measurement analyser (by [@Rizzen](https://github.com/Rizzen)) -* [48d193](https://github.com/dotnet/BenchmarkDotNet/commit/48d193e30c780eb43e65b21f892c48db5dab6f6b) Zero measurement analyser (#906) (by [@Rizzen](https://github.com/Rizzen)) - -### RatioColumn - -The `Ratio` column was formerly known as `Scaled`. -The old title was a source of misunderstanding and confusion because - many developers interpreted it as the ratio of means (e.g., `50.46`/`100.39` for `Time50`). -The ratio of distribution means and the mean of the ratio distribution are pretty close to each other in most cases, - but they are not equal. - -See also: - @BenchmarkDotNet.Samples.IntroBenchmarkBaseline, - @BenchmarkDotNet.Samples.IntroRatioSD, - @docs.baselines. - -* [4e64c9](https://github.com/dotnet/BenchmarkDotNet/commit/4e64c94cfe7b49bbdc06aabb6ee1f262bd370862) Ratio/RatioSD columns (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - diff --git a/docs/_changelog/header/v0.11.3.md b/docs/_changelog/header/v0.11.3.md deleted file mode 100644 index f3be5d81af..0000000000 --- a/docs/_changelog/header/v0.11.3.md +++ /dev/null @@ -1,99 +0,0 @@ - - -## Highlights - -This release is focused mainly on bug fixes that were affecting user experience. But don't worry, we have some new features too! - -* **Diagnosers** - * ConcurrencyVisualizerProfiler (allows profiling benchmarks on Windows and exporting the data to a trace file which can be opened with Concurrency Visualizer) -* **Command-line:** - * `--stopOnFirstError`: Stops the benchmarks execution on first error. [#947](https://github.com/dotnet/BenchmarkDotNet/pull/947) - * `--statisticalTest`: Performs a Mann–Whitney Statistical Test for identifying regressions and improvements. [#960](https://github.com/dotnet/BenchmarkDotNet/pull/960) -* **Bug fixes:** - * Dry mode doesn't work because of the ZeroMeasurementHelper [#943](https://github.com/dotnet/BenchmarkDotNet/issues/943) - * MannWhitneyTest fails when comparing statistics of different sample size [#948](https://github.com/dotnet/BenchmarkDotNet/issues/948) and [#950](https://github.com/dotnet/BenchmarkDotNet/issues/950) - * Improve the dynamic loading of Diagnostics package [#955](https://github.com/dotnet/BenchmarkDotNet/issues/955) - * BenchmarkRunner.RunUrl throws NRE when Config is not provided [#961](https://github.com/dotnet/BenchmarkDotNet/issues/961) - * Don't require the users to do manual installation of TraceEvent when using Diagnostics package [#962](https://github.com/dotnet/BenchmarkDotNet/issues/962) - * Stop benchmark after closing application + Flush log after stopping benchmark [#963](https://github.com/dotnet/BenchmarkDotNet/issues/963) - ---- - -## Diagnosers - -### ConcurrencyVisualizerProfiler - -`ConcurrencyVisualizerProfiler` allows to profile the benchmarked .NET code on Windows and exports the data to a CVTrace file which can be opened with [Concurrency Visualizer](https://learn.microsoft.com/visualstudio/profiling/concurrency-visualizer). - -`ConcurrencyVisualizerProfiler` uses `EtwProfiler` to get a `.etl` file which still can be opened with [PerfView](https://github.com/Microsoft/perfview) or [Windows Performance Analyzer](https://learn.microsoft.com/windows-hardware/test/wpt/windows-performance-analyzer). The difference is that it also enables all Task and Thread related ETW Providers and exports a simple `xml` which can be opened with Visual Studio if you install [Concurrency Visualizer plugin](https://marketplace.visualstudio.com/items?itemName=Diagnostics.ConcurrencyVisualizer2017) - -![open trace](https://user-images.githubusercontent.com/6011991/48638184-2b13fe00-e9d0-11e8-8a94-0e951e4606ae.png) - -![Utilization](https://user-images.githubusercontent.com/6011991/48638108-f6a04200-e9cf-11e8-8f7c-feda0a99138c.png) - -![Threads](https://user-images.githubusercontent.com/6011991/48638120-fb64f600-e9cf-11e8-9e2d-53b615bfe9a8.png) - - -* [#964](https://github.com/dotnet/BenchmarkDotNet/issues/964) Concurrency Visualizer Profiler Diagnoser (by [@adamsitnik](https://github.com/adamsitnik)) -* [dfb3c89](https://github.com/dotnet/BenchmarkDotNet/commit/dfb3c8912505799a76b0eb5ae0c082bb44599fa7) ConcurrencyVisualizerProfiler diagnoser! (by [@adamsitnik](https://github.com/adamsitnik)) - ---- - -## Command-line - -In this release, we have some new command-line arguments! - -### `--stopOnFirstError`: Stops the benchmarks execution on first error - -When provided, BenchmarkDotNet is going to stop the benchmarks execution on first error. - -* [#947](https://github.com/dotnet/BenchmarkDotNet/pull/947) Add option to stop running when the first benchmark fails (by [@wojtpl2](https://github.com/wojtpl2)) - -### `--statisticalTest`: Statistical Test - -To perform a Mann–Whitney U Test and display the results in a dedicated column you need to provide the Threshold via -`--statisticalTest`. Examples: 5%, 10ms, 100ns, 1s. - -Example: run Mann–Whitney U test with relative ratio of 1% for all benchmarks for .NET 4.6 (base), .NET Core 2.0 and .NET Core 2.1. - -```cs -class Program -{ - static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); -} - -public class MySample -{ - [Benchmark] - public void Sleep() - { -#if NETFRAMEWORK - Thread.Sleep(50); -#elif NETCOREAPP2_0 - Thread.Sleep(45); -#elif NETCOREAPP2_1 - Thread.Sleep(55); -#endif - } - - [Benchmark] - public void Same() => Thread.Sleep(50); -} -``` - -```log -dotnet run -c Release -f netcoreapp2.1 --filter * --runtimes net46 netcoreapp2.0 netcoreapp2.1 --statisticalTest 1% -``` - -**Note:** .NET 4.6 will be our baseline because it was provided as first on the runtimes list. - - -| Method | Runtime | Toolchain | Mean | Error | StdDev | Ratio | MannWhitney(1%) | -|------- |-------- |-------------- |---------:|----------:|----------:|------:|---------------- | -| Sleep | Clr | net46 | 50.51 ms | 0.1833 ms | 0.1714 ms | 1.00 | Base | -| Sleep | Core | netcoreapp2.0 | 45.53 ms | 0.1262 ms | 0.1181 ms | 0.90 | Faster | -| Sleep | Core | netcoreapp2.1 | 55.50 ms | 0.1217 ms | 0.1138 ms | 1.10 | Slower | -| | | | | | | | | -| Same | Clr | net46 | 50.47 ms | 0.1795 ms | 0.1679 ms | 1.00 | Base | -| Same | Core | netcoreapp2.0 | 50.55 ms | 0.1873 ms | 0.1752 ms | 1.00 | Same | -| Same | Core | netcoreapp2.1 | 50.55 ms | 0.2162 ms | 0.2022 ms | 1.00 | Same | diff --git a/docs/_changelog/header/v0.11.4.md b/docs/_changelog/header/v0.11.4.md deleted file mode 100644 index 3cf8b79f55..0000000000 --- a/docs/_changelog/header/v0.11.4.md +++ /dev/null @@ -1,114 +0,0 @@ -It's been few months since our last release, but we have been working hard and have some new features for you! - -## Highlights - -* **Features** - * **BenchmarkDotNet as a global tool**: a new global tool which allows you to run benchmarks from given library. - Now you can run benchmarks from the command line via `dotnet benchmark`. - You can find more information about it in the documentation - [#1006](https://github.com/dotnet/BenchmarkDotNet/pull/1006) - [#213](https://github.com/dotnet/BenchmarkDotNet/issues/213) - * **InProcessEmitToolchain**: new, full-featured InProcess toolchain which allows executing benchmarks in the current process - without spawning additional process per benchmark. - It supports `[Arguments]`, `[ArgumentsSource]`, passing the arguments by `out`, `ref` and returning stack-only types like `Span`. - [#919](https://github.com/dotnet/BenchmarkDotNet/issues/919), - [#921](https://github.com/dotnet/BenchmarkDotNet/pull/921) - [#843](https://github.com/dotnet/BenchmarkDotNet/issues/843) - * **ARM support**: BenchmarkDotNet supports now ARM and ARM64. - [#780](https://github.com/dotnet/BenchmarkDotNet/issues/780), - [#979](https://github.com/dotnet/BenchmarkDotNet/pull/979) - [#385](https://github.com/dotnet/BenchmarkDotNet/issues/385) - * **Mono AOT support**: a new toolchain which allows running benchmarks using AOT version of Mono - [#940](https://github.com/dotnet/BenchmarkDotNet/pull/940) - * **NuGet symbol server support**: BenchmarkDotNet publishes now the symbols to NuGet.org symbol server and you can easily debug it. - [#967](https://github.com/dotnet/BenchmarkDotNet/issues/967) - [#968](https://github.com/dotnet/BenchmarkDotNet/pull/968) - * **Experimental support for .NET Core 3.0 WPF benchmarks** - [#1066](https://github.com/dotnet/BenchmarkDotNet/pull/1066) - For a working example please go to [https://github.com/dotMorten/WPFBenchmarkTests](https://github.com/dotMorten/WPFBenchmarkTests) -* **Improvements:** - * CoreRT Toolchain improvements - thanks to help from CoreRT Team we were able to make - the CoreRT Toolchain work with the latest version of CoreRT - [#1001](https://github.com/dotnet/BenchmarkDotNet/pull/1001), - [#1057](https://github.com/dotnet/BenchmarkDotNet/pull/1057) - * Display the number of benchmarks to run: we now display how many benchmarks are going to be executed - before running them and how many remained after running each of them - [#1048](https://github.com/dotnet/BenchmarkDotNet/issues/1048) - * Better list of suggested benchmarks for wrong filter - [#834](https://github.com/dotnet/BenchmarkDotNet/issues/834) - [#957](https://github.com/dotnet/BenchmarkDotNet/pull/957) - * Invalid assembly binding redirects generated by VS were a pain to many of our users, - we have now implemented an approach that tries to work around this issue. - [#895](https://github.com/dotnet/BenchmarkDotNet/issues/895), - [#667](https://github.com/dotnet/BenchmarkDotNet/issues/667), - [#896](https://github.com/dotnet/BenchmarkDotNet/issues/896), - [#942](https://github.com/dotnet/BenchmarkDotNet/issues/942) - * Handling duplicates in IConfig - [#912](https://github.com/dotnet/BenchmarkDotNet/pull/912), - [#938](https://github.com/dotnet/BenchmarkDotNet/issues/938), - [#360](https://github.com/dotnet/BenchmarkDotNet/issues/360), - [#463](https://github.com/dotnet/BenchmarkDotNet/issues/463) - * Disassembly diagnoser should be kept in a separate directory to avoid dependency conflicts - [#1059](https://github.com/dotnet/BenchmarkDotNet/issues/1059) - * Give a warning when the `[Benchmark]` method is static - we now produce an error when users fail into this common issue - [#983](https://github.com/dotnet/BenchmarkDotNet/issues/983) - [#985](https://github.com/dotnet/BenchmarkDotNet/pull/985) - * C# keywords are prohibited as benchmark names - [#849](https://github.com/dotnet/BenchmarkDotNet/issues/849) - * File names should be consistent across all OSes - `<` and `>` are valid on Unix, but not on Windows. - We have unified that and now files produced on Unix and Windows have the same names. - [#981](https://github.com/dotnet/BenchmarkDotNet/issues/981) - * Improve restore, build and publish projects - [#1002](https://github.com/dotnet/BenchmarkDotNet/issues/1002), - [#1013](https://github.com/dotnet/BenchmarkDotNet/pull/1013) - * Make it possible to disable OptimizationsValidator - [#988](https://github.com/dotnet/BenchmarkDotNet/issues/988) - * Sort enum parameters by value instead of name - [#977](https://github.com/dotnet/BenchmarkDotNet/pull/977) - * Detect .NET Core benchmark failures from LINQPad - [#980](https://github.com/dotnet/BenchmarkDotNet/pull/980) - * Improved error logging - [#1008](https://github.com/dotnet/BenchmarkDotNet/pull/1008) - * Improved disassembly diff - [#1022](https://github.com/dotnet/BenchmarkDotNet/pull/1022) - * Using invariant culture for Roslyn Toolchain error messages - [#1042](https://github.com/dotnet/BenchmarkDotNet/pull/1042) - * Use only full names in the auto-generated code to avoid any possible conflicts with user code - [#1007](https://github.com/dotnet/BenchmarkDotNet/issues/1007), - [#1009](https://github.com/dotnet/BenchmarkDotNet/pull/1009) - [#1010](https://github.com/dotnet/BenchmarkDotNet/issues/1010) - * Write the GitHub table format to the console by default - [#1062](https://github.com/dotnet/BenchmarkDotNet/issues/1062) - * Proper cleanup on Ctrl+C/console Window exit - [#1061](https://github.com/dotnet/BenchmarkDotNet/pull/1061) - * Introduce StoppingCriteria - the first step to writing your own heuristic that determines when benchmarking should be stopped - [#984](https://github.com/dotnet/BenchmarkDotNet/pull/984) -* **Breaking changes:** - * .NET Standard 2.0 only - BenchmarkDotNet has a single target now, which should help with some assembly resolving issues. - We had to drop .NET 4.6 support because of that and .NET 4.6.1 is now the oldest supported .NET Framework. - [#1032](https://github.com/dotnet/BenchmarkDotNet/pull/1032) - * CustomCoreClrToolchain has been removed, it's recommended to use CoreRunToolchain instead - [#928](https://github.com/dotnet/BenchmarkDotNet/issues/928) -* **Bug fixes:** - * NRE in `Summary` ctor - [#986](https://github.com/dotnet/BenchmarkDotNet/issues/986) - [#987](https://github.com/dotnet/BenchmarkDotNet/pull/987) - * ArgumentNullException when running benchmarks from published .NET Core app - [#1018](https://github.com/dotnet/BenchmarkDotNet/issues/1018) - * Dry jobs can eat iteration failures - [#1045](https://github.com/dotnet/BenchmarkDotNet/issues/1045) - * NullReferenceException in BenchmarkDotNet.Reports.SummaryTable after iteration failure - [#1046](https://github.com/dotnet/BenchmarkDotNet/issues/1046) - * Running the example throws NullReference - [#1049](https://github.com/dotnet/BenchmarkDotNet/issues/1049) - * Fix race condition in process output reader - [#1051](https://github.com/dotnet/BenchmarkDotNet/issues/1051) - [#1053](https://github.com/dotnet/BenchmarkDotNet/pull/1053) - * Fix a rare but really annoying bug where for some reason we were sometimes setting ForegroundColor - to the same color as BackgroundColor and some parts of the logged output were invisible - [commit](https://github.com/dotnet/BenchmarkDotNet/commit/ea3036810ef60b483d766a097e6f3edfde28a834) - * StopOnFirstError must be respected - [commit](https://github.com/dotnet/BenchmarkDotNet/commit/87d281d7dbf52036819efff52e6661e436648b73) - ---- - diff --git a/docs/_changelog/header/v0.11.5.md b/docs/_changelog/header/v0.11.5.md deleted file mode 100644 index 98e31c0195..0000000000 --- a/docs/_changelog/header/v0.11.5.md +++ /dev/null @@ -1,170 +0,0 @@ -## Highlights - -* **Features and noticeable improvements** - * **Power plan management** - Now BenchmarkDotNet executes all benchmarks with enabled High-Performance power plan (configurable, Windows-only). - You can find some details below. - [#68](https://github.com/dotnet/BenchmarkDotNet/issues/68) - [#952](https://github.com/dotnet/BenchmarkDotNet/pull/952) - * **Better Environment Variables API** - Now we have some additional extension methods which allow defining environment variables in user jobs. - In the previous version, users always had to set an array of environment variables like this: - `job.With(new[] { new EnvironmentVariable("a", "b") })`. - Now it's possible to define an environment variable like - `job.With(new EnvironmentVariable("a", "b"))` or - `job.WithEnvironmentVariable("a", "b")`. - Also, it's possible to clear the list of environment variables via - `job.WithoutEnvironmentVariables()`. - [#1069](https://github.com/dotnet/BenchmarkDotNet/issues/1069) - [#1080](https://github.com/dotnet/BenchmarkDotNet/pull/1080) - * **Better outlier messages** - The previous version of BenchmarkDotNet printed information about detected or removed outliers like this: - "3 outliers were detected". - It was nice, but it didn't provide additional information about these outliers - (users had to read the full log to find the outliers values). - Now BenchmarkDotNet prints additional information about outlier values like this: - "3 outliers were detected (2.50 us..2.70 us)". - [e983cd31](https://github.com/dotnet/BenchmarkDotNet/commit/e983cd3126e64f82fe59bc1bc45d1a870a615e87) - * **Support modern CPU architecture names** - In the environment information section, BenchmarkDotNet prints not only the processor brand string, but also its architecture - (e.g., "Intel Core i7-4770K CPU 3.50GHz (Haswell)"). - However, it failed to recognize some recent processors. - Now it's able to detect the architecture for modern Intel processors correctly - (Kaby Lake, Kaby Lake R, Kaby Lake G, Amber Lake Y, Coffee Lake, Cannon Lake, Whiskey Lake). - [995e053d](https://github.com/dotnet/BenchmarkDotNet/commit/995e053d14a61cdadc417149480f23ebf679bcb7) - * **Introduce BenchmarkDotNet.Annotations** - Currently, BenchmarkDotNet targets .NET Standard 2.0. - It makes some users unhappy because they want to define benchmarks in projects with lower target framework. - We decided to start working on the `BenchmarkDotNet.Annotations` NuGet package which targets .NET Standard 1.0 - and contains classes that users need to define their benchmarks. - However, it's not easy to refactor the full source code base and move all relevant public APIs to this package. - In v0.11.5, we did the first step and moved some of these APIs to `BenchmarkDotNet.Annotations`. - We want to continue moving classes to this package and get full-featured annotation package in the future. - [#1084](https://github.com/dotnet/BenchmarkDotNet/pull/1084) - [#1096](https://github.com/dotnet/BenchmarkDotNet/pull/1096) - * **Use InProcessEmitToolchain by default in InProcess benchmarks** - In BenchmarkDotNet 0.11.4, we introduced `InProcessEmitToolchain`. - It's a new, full-featured InProcess toolchain which allows executing benchmarks in the current process - without spawning additional process per benchmark. - It supports `[Arguments]`, `[ArgumentsSource]`, passing the arguments by `out`, `ref` and returning stack-only types like `Span`. - However, in v0.11.4, it can be activated only if `InProcessEmitToolchain` is declared explicitly. - Now it's enabled by default when `[InProcessAttribute]` is used. - [#1093](https://github.com/dotnet/BenchmarkDotNet/pull/1093) - * **Introduce an option which prevents overwriting results** - Currently, BenchmarkDotNet overwrites results each time when the benchmarks are executed. - It allows avoiding tons of obsolete files in the `BenchmarkDotNet.Artifacts` folder. - However, the behavior doesn't fit all use cases: - sometimes users want to keep results for old benchmark runs. - Now we have a special option for it. - The option can be activated via `--noOverwrite` console line argument or - `DontOverwriteResults` extension method for `IConfig` - [#1074](https://github.com/dotnet/BenchmarkDotNet/issues/1074) - [#1083](https://github.com/dotnet/BenchmarkDotNet/pull/1083) -* **Other improvements and bug fixes** - * **Diagnostics and validation** - * **Better benchmark declaration error processing** - In the previous version, BenchmarkDotNet threw an exception when some benchmark methods had an invalid declaration - (e.g., invalid signature or invalid access modifiers). - Now it prints a nice error message without ugly stack traces. - [#1107](https://github.com/dotnet/BenchmarkDotNet/issues/1107) - * **Better error message for users who want to debug benchmarks** - [#1073](https://github.com/dotnet/BenchmarkDotNet/issues/1073) - * **Don't show the same validation error multiple times** - Now each error will be printed only once. - [#1079](https://github.com/dotnet/BenchmarkDotNet/issues/1079) - * **Restrict MemoryDiagnoserAttribute usage to class** - Now it's impossible to accidentally mark a method with this attribute. - [#1119](https://github.com/dotnet/BenchmarkDotNet/issues/1119) - [#1122](https://github.com/dotnet/BenchmarkDotNet/pull/1122) - * **Export** - * **Better indentation in disassembly listings** - Now DissassemblyDiagnoser correctly process source code which contains tab as the indentation symbol - [#1110](https://github.com/dotnet/BenchmarkDotNet/issues/1110) - * **Fix incorrect indentation for StackOverflow exporter** - Previously, StackOverflow exporter doesn't have a proper indent for job runtimes in the environment information. - Now it's fixed. - [#826](https://github.com/dotnet/BenchmarkDotNet/issues/826) - [#1104](https://github.com/dotnet/BenchmarkDotNet/pull/1104) - * **Fix StackOverflowException in XmlExporter.Full** - [#1086](https://github.com/dotnet/BenchmarkDotNet/issues/1086) - [#1090](https://github.com/dotnet/BenchmarkDotNet/pull/1090) - * **Shortify MemoryDiagnoser column titles** - Now we use the following column titles: - "Allocated" instead of "Allocated Memory/Op", - "Gen 0" instead of "Gen 0/1k Op". - The full description of each column can be found in the legend section below the summary table. - [#1081](https://github.com/dotnet/BenchmarkDotNet/pull/1081) - * **Benchmark generation and execution** - * **Fixed broken Orderers** - The previous version has a nasty bug with custom orderers. - Now it's fixed. - [#1070](https://github.com/dotnet/BenchmarkDotNet/issues/1070) - [#1109](https://github.com/dotnet/BenchmarkDotNet/issues/1109) - * **Better overhead evaluation** - In the previous version, BenchmarkDotNet evaluated the benchmark overhead as a mean value of all overhead iteration. - It was fine in most cases, but in some cases, the mean value can be spoiled by outliers. - Now BenchmarkDotNet uses the median value. - [#1116](https://github.com/dotnet/BenchmarkDotNet/issues/1116) - * **Respect CopyLocalLockFileAssemblies** - Now BenchmarkDotNet respect the `CopyLocalLockFileAssemblies` value and copies it - to the generated benchmark project. - [#1068](https://github.com/dotnet/BenchmarkDotNet/issues/1068) - [#1108](https://github.com/dotnet/BenchmarkDotNet/pull/1108) - * **Disable CodeAnalysisRuleSet for generated benchmarks** - Previously, generated benchmarks may fail if the `CodeAnalysisRuleSet` is defined in `Directory.Build.Props`. - [#1082](https://github.com/dotnet/BenchmarkDotNet/pull/1082) - * **Supported undefined enum values** - [#1020](https://github.com/dotnet/BenchmarkDotNet/issues/1020) - [#1071](https://github.com/dotnet/BenchmarkDotNet/issues/1071) - * **Other minor improvements and bug fixes** - -## Power plans - -In [#952](https://github.com/dotnet/BenchmarkDotNet/pull/952), power plan management was implemented. -It resolves a pretty old issue [#68](https://github.com/dotnet/BenchmarkDotNet/issues/68) which was created more than three years ago. -Now BenchmarkDotNet forces OS to execute a benchmark on the High-Performance power plan. -You can disable this feature by modifying PowerPlanMode property. -Here is an example where we are playing with this value: - -```cs -[Config(typeof(Config))] -public class IntroPowerPlan -{ - private class Config : ManualConfig - { - public Config() - { - Add(Job.MediumRun.WithPowerPlan(PowerPlan.HighPerformance)); - Add(Job.MediumRun.WithPowerPlan(PowerPlan.UserPowerPlan)); - } - } - - [Benchmark] - public int IterationTest() - { - int j = 0; - for (int i = 0; i < short.MaxValue; ++i) - j = i; - return j; - } - - [Benchmark] - public int SplitJoin() - => string.Join(",", new string[1000]).Split(',').Length; -} -``` - -And here is an example of the summary table on plugged-off laptop: - -```md - Method | PowerPlan | Mean | Error | StdDev | --------------- |---------------- |---------:|----------:|----------:| - IterationTest | HighPerformance | 40.80 us | 0.4168 us | 0.6109 us | - SplitJoin | HighPerformance | 13.24 us | 0.2514 us | 0.3763 us | - IterationTest | UserPowerPlan | 79.72 us | 2.5623 us | 3.8352 us | - SplitJoin | UserPowerPlan | 24.54 us | 2.1062 us | 3.1525 us | -``` - -As you can see, the power plan produces a noticeable effect on the benchmark results. - -This feature is available on Windows only. \ No newline at end of file diff --git a/docs/_changelog/header/v0.12.0.md b/docs/_changelog/header/v0.12.0.md deleted file mode 100644 index 0e6a00582f..0000000000 --- a/docs/_changelog/header/v0.12.0.md +++ /dev/null @@ -1,469 +0,0 @@ -It's been several months since our last release, but we have been working hard and have some new features for you! - -## Highlights - -* **Features and major improvements** - * **Advanced multiple target frameworks support** - Now BenchmarkDotNet supports .NET Framework 4.8, .NET Core 3.1, and .NET Core 5.0. - Also, we reworked our API that allows targeting several runtimes from the same config: - the new API is more consistent, flexible, and powerful. - For example, if you want to execute your benchmarking using .NET Framework 4.8 and .NET Core 3.1, - you can use the `SimpleJob(RuntimeMoniker.Net48)`, `[SimpleJob(RuntimeMoniker.NetCoreApp31)]` attributes or - `Job.Default.With(ClrRuntime.Net48)`, `Job.Default.With(CoreRuntime.Core31)` jobs in a manual config. - You can find more details below. - [#1188](https://github.com/dotnet/BenchmarkDotNet/pull/1188), - [#1186](https://github.com/dotnet/BenchmarkDotNet/issues/1186), - [#1236](https://github.com/dotnet/BenchmarkDotNet/issues/1236) - * **Official templates for BenchmarkDotNet-based projects** - With the help of the [BenchmarkDotNet.Templates](https://www.nuget.org/packages/BenchmarkDotNet.Templates/) NuGet package, - you can easily create new projects from the command line via `dotnet new benchmark`. - This command has a lot of useful options, so you can customize your new project as you want. - [#1044](https://github.com/dotnet/BenchmarkDotNet/pull/1044) - * **New NativeMemoryProfiler** - `NativeMemoryProfiler` measure the native memory traffic and adds the extra columns `Allocated native memory` and `Native memory leak` to the summary table. - Internally, it uses `EtwProfiler` to profile the code using ETW. - [#457](https://github.com/dotnet/BenchmarkDotNet/issues/457), - [#1131](https://github.com/dotnet/BenchmarkDotNet/pull/1131), - [#1208](https://github.com/dotnet/BenchmarkDotNet/pull/1208), - [#1214](https://github.com/dotnet/BenchmarkDotNet/pull/1214), - [#1218](https://github.com/dotnet/BenchmarkDotNet/pull/1218), - [#1219](https://github.com/dotnet/BenchmarkDotNet/pull/1219) - * **New ThreadingDiagnoser** - `ThreadingDiagnoser` also adds two extra columns to the summary table: - `Completed Work Items` (the number of work items that have been processed in ThreadPool per single operation) and - `Lock Contentions` (the number of times there *was contention* upon trying to take a Monitor's lock per single operation). - Internally, it uses [new APIs](https://github.com/dotnet/corefx/issues/35500) exposed in .NET Core 3.0. - [#1154](https://github.com/dotnet/BenchmarkDotNet/issues/1154), - [#1227](https://github.com/dotnet/BenchmarkDotNet/pull/1227) - * **Improved MemoryDiagnoser** - Now `MemoryDiagnoser` includes memory allocated by *all threads* that were live during benchmark execution: a new GC API was exposed in .NET Core 3.0 preview6+. - It allows to get the number of allocated bytes for all threads. - [#1155](https://github.com/dotnet/BenchmarkDotNet/pull/1155), - [#1153](https://github.com/dotnet/BenchmarkDotNet/issues/1153), - [#723](https://github.com/dotnet/BenchmarkDotNet/issues/723) - * **LINQPad 6 support** - Now both LINQPad 5 and LINQPad 6 are supported! - [#1241](https://github.com/dotnet/BenchmarkDotNet/issues/1241), - [#1245](https://github.com/dotnet/BenchmarkDotNet/pull/1245) - * **Fast documentation search** - We continue to improve the usability of our documentation. - In this release, we improved the search experience in the documentation: now it works almost instantly with the help of [Algolia](https://www.algolia.com/) engine! - [#1148](https://github.com/dotnet/BenchmarkDotNet/pull/1148), - [#1158](https://github.com/dotnet/BenchmarkDotNet/issues/1158) -* **Minor summary and exporter improvements** - * **Improved presentation of the current architecture in the environment information** - In the previous version of BenchmarkDotNet, the reports always contained "64bit" or "32bit" which did not tell if it was ARM or not. - Now it prints the full architecture name (`x64`, `x86`, `ARM`, or `ARM64`). - For example, instead of `.NET Framework 4.8 (4.8.3815.0), 64bit RyuJIT` you will get `.NET Framework 4.8 (4.8.3815.0), X64 RyuJIT` or `.NET Framework 4.8 (4.8.3815.0), ARM64 RyuJIT`. - [#1213](https://github.com/dotnet/BenchmarkDotNet/pull/1213) - * **Simplified reports for Full .NET Framework version** - Previous version: `.NET Framework 4.7.2 (CLR 4.0.30319.42000), 64bit RyuJIT-v4.7.3324.0`. - Current version: `.NET Framework 4.7.2 (4.7.3362.0), 64bit RyuJIT`. - [#1114](https://github.com/dotnet/BenchmarkDotNet/pull/1114), - [#1111](https://github.com/dotnet/BenchmarkDotNet/issues/1111) - * **More reliable CPU info on Windows** - We added a workaround to for a bug in wmic that uses `\r\r\n` as a line separator. - [#1144](https://github.com/dotnet/BenchmarkDotNet/issues/1144), - [#1145](https://github.com/dotnet/BenchmarkDotNet/pull/1145) - * **Better naming for generated plots** - When `[RPlotExporter]` is used, BenchmarkDotNet generates a lot of useful plots in the `BenchmarkDotNet.Artifacts` folder. - The naming of the plot files was improved: benchmarks without `Params` doesn't include a double dash (`--`) in their names anymore. - [1183](https://github.com/dotnet/BenchmarkDotNet/issues/1183), - [1212](https://github.com/dotnet/BenchmarkDotNet/pull/1212) - * **Better density plot precision** - The previous version of BenchmarkDotNet used the rule-of-thumb bandwidth selector in `RPlotExporter` density plots. - It was fine for unimodal distributions, but sometimes it produced misleading plots for multimodal distributions. - Now, RPlotExporter uses the Sheather&Jones bandwidth selector that significantly improves the presentation of the density plots for complex distributions. - [58fde64](https://github.com/dotnet/BenchmarkDotNet/commit/58fde64c809ceadb3fca9d677a7cec83071b833f) - * **Better alignment in `HtmlExporter`** - Now BenchmarkDotNet aligns the content exported by `HtmlExporter` to the right. - [#1189](https://github.com/dotnet/BenchmarkDotNet/pull/1189) - [dfa074](https://github.com/dotnet/BenchmarkDotNet/commit/dfa074b024cfa8bbe4fe175d3d65a3d9f85127ff) - * **Better precision calculation in SummaryTable** - [4e9eb43](https://github.com/dotnet/BenchmarkDotNet/commit/4e9eb4335eee05a95a3766f2c81ae260508021af) - * **Better summary analysis** - BenchmarkDotNet warns the user when benchmark baseline value is too close to zero and the columns derived from BaselineCustomColumn cannot be computed. - [#1161](https://github.com/dotnet/BenchmarkDotNet/pull/1161), - [#600](https://github.com/dotnet/BenchmarkDotNet/issues/600) - * **Make log file datetime format 24-hour** - [#1149](https://github.com/dotnet/BenchmarkDotNet/issues/1149) - * **Improve AskUser prompt message** - The error messages will surround `*` by quotes on Linux and macOS. - [#1147](https://github.com/dotnet/BenchmarkDotNet/issues/1147) -* **Minor API improvements** - * **ED-PELT algorithm for changepoint detection is now available** - You can find details in [this blog post](https://aakinshin.net/posts/edpelt/). - [f89091](https://github.com/dotnet/BenchmarkDotNet/commit/f89091a2a9c1a4058dd8e32d5aa01271910dd7dc) - * **Improved OutlierMode API** - BenchmarkDotNet performs measurement postprocessing that may remove some of the outlier values (it can be useful to remove upper outliers that we get because of the natural CPU noise). - In the previous version, naming for the `OutlierMode` values was pretty confusing: `None/OnlyUpper/OnlyLower/All`. - Now, these values were renamed to `DontRemove/RemoveUpper/RemoveLower/RemoveAll`. - For example, if you want to remove all the outliers, you can annotate your benchmark with the `[Outliers(OutlierMode.RemoveAll)]` attribute. - The old names still exist (to make sure that the changes are backward compatible), but they are marked as obsolete, and they will be removed in the future versions of the library. - [#1199](https://github.com/dotnet/BenchmarkDotNet/pull/1199), - [0e4b8e](https://github.com/dotnet/BenchmarkDotNet/commit/0e4b8e69d10e73a83fce3ce3980497ee7798bc87) - * **Add the possibility to pass `Config` to `BenchmarkSwitcher.RunAll` and `RunAllJoined`** - [#1194](https://github.com/dotnet/BenchmarkDotNet/issues/1194), - [ae23bd](https://github.com/dotnet/BenchmarkDotNet/commit/ae23bddebe49cd66a9627790c073b7bc45ccbf5c) - * **Improved command line experience** - When user uses `--packages $path`, the `$path` will be sent to the dotnet build command as well. - [1187](https://github.com/dotnet/BenchmarkDotNet/issues/1187) - * **Extend the list of supported power plans.** - Now it supports "ultimate", "balanced", and "power saver" plans. - [#1132](https://github.com/dotnet/BenchmarkDotNet/issues/1132), - [#1139](https://github.com/dotnet/BenchmarkDotNet/pull/1139) - * **Make it possible to not enforce power plan on Windows.** - [1578c5c](https://github.com/dotnet/BenchmarkDotNet/commit/1578c5c60c3f59f9128f680e35d1db219aa60d8d) - * **`Guid` support in benchmark arguments** - Now you can use `Guid` instances as benchmark arguments. - [04ec20b](https://github.com/dotnet/BenchmarkDotNet/commit/04ec20b5e0c0a514e8d158684864e4f9934ae8cc) - * **Make `ArgumentsSource` support `IEnumerable` for benchmarks accepting a single argument to mimic `MemberData` behaviour.** - [ec296dc](https://github.com/dotnet/BenchmarkDotNet/commit/ec296dc45de798f7407852d5ab7febe2b457eca4) - * **Make `FullNameProvider` public** - So it can be reused by the `dotnet/performance` repository. - [6d71308](https://github.com/dotnet/BenchmarkDotNet/commit/6d71308f4c1d96bcf8a526bfcc61bb23307b4041) - * **Extend `Summary` with `LogFilePath`** - [#1135](https://github.com/dotnet/BenchmarkDotNet/issues/1135), - [6e6559](https://github.com/dotnet/BenchmarkDotNet/commit/6e6559499652c312369fd223d0cf4747f65512d6) - * **Allow namespace filtering for `InliningDiagnoser`** - [#1106](https://github.com/dotnet/BenchmarkDotNet/issues/1106), - [#1130](https://github.com/dotnet/BenchmarkDotNet/pull/1130) - * **Option to configure `MaxParameterColumnWidth`** - [#1269](https://github.com/dotnet/BenchmarkDotNet/issues/1269), - [4ec888](https://github.com/dotnet/BenchmarkDotNet/commit/4ec88844547507474ccd0303b31f935b3463318c) -* **Other improvements** - * **Misc improvements in the documentation** - [#1175](https://github.com/dotnet/BenchmarkDotNet/pull/1175), - [#1173](https://github.com/dotnet/BenchmarkDotNet/pull/1173), - [#1180](https://github.com/dotnet/BenchmarkDotNet/pull/1180), - [#1203](https://github.com/dotnet/BenchmarkDotNet/pull/1203), - [#1204](https://github.com/dotnet/BenchmarkDotNet/pull/1204), - [#1206](https://github.com/dotnet/BenchmarkDotNet/pull/1206), - [#1209](https://github.com/dotnet/BenchmarkDotNet/pull/1209), - [#1219](https://github.com/dotnet/BenchmarkDotNet/pull/1219), - [#1225](https://github.com/dotnet/BenchmarkDotNet/pull/1225), - [#1279](https://github.com/dotnet/BenchmarkDotNet/pull/1279) - * **Copy `PreserveCompilationContext` MSBuild setting from the project that defines benchmarks** - [#1152](https://github.com/dotnet/BenchmarkDotNet/issues/1152), - [063d1a](https://github.com/dotnet/BenchmarkDotNet/commit/063d1a56152fd5812cb6e9dd5095dc6e647e6938) - * **Add `System.Buffers.ArrayPoolEventSource` to the list of default .NET Providers of `EtwProfiler`** - [#1179](https://github.com/dotnet/BenchmarkDotNet/issues/1179), - [a106b1](https://github.com/dotnet/BenchmarkDotNet/commit/a106b114b1f04fa1024be84a8969f5a168fa1c8b) - * **Consume CoreRT from the new NuGet feed** - Because CoreRT no longer publishes to MyGet. - [#1129](https://github.com/dotnet/BenchmarkDotNet/pull/1129) -* **Breaking changes:** - * The `[ClrJob]`, `[CoreJob]` and `[CoreRtJob]` attributes got obsoleted and replaced by a `[SimpleJob]` which requires the user to provide target framework moniker in an explicit way. - (See the "Advanced multiple target frameworks support" section for details.) - [#1188](https://github.com/dotnet/BenchmarkDotNet/pull/1188), - [#1182](https://github.com/dotnet/BenchmarkDotNet/issues/1182), - [#1115](https://github.com/dotnet/BenchmarkDotNet/issues/1115), - [#1056](https://github.com/dotnet/BenchmarkDotNet/issues/1056), - [#993](https://github.com/dotnet/BenchmarkDotNet/issues/993), - * The old `InProcessToolchain` is now obsolete. It's recommended to use `InProcessEmitToolchain`. If you want to use the old one on purpose, you have to use `InProcessNoEmitToolchain`. - [#1123](https://github.com/dotnet/BenchmarkDotNet/pull/1123) -* **Bug fixes:** - * Invalid arg passing in StreamLogger constructor. The `append` arg was not passed to the `StreamWriter` .ctor. - [#1185](https://github.com/dotnet/BenchmarkDotNet/pull/1185) - * Improve the output path of `.etl` files produced by `EtwProfiler`. `EtwProfiler` was throwing NRE for users who were using `[ClrJob]` and `[CoreJob]` attributes. - [#1156](https://github.com/dotnet/BenchmarkDotNet/issues/1156), - [#1072](https://github.com/dotnet/BenchmarkDotNet/issues/1072) - * Flush custom loggers at the end of benchmark session. - [#1134](https://github.com/dotnet/BenchmarkDotNet/issues/1134) - * Make ids for tag columns unique - when using multiple `TagColumns` only one `TagColumn` was printed in the results. - [#1146](https://github.com/dotnet/BenchmarkDotNet/issues/1146) - -## Advanced multiple target frameworks support - -Now BenchmarkDotNet supports .NET Framework 4.8, .NET Core 3.1, and .NET Core 5.0. -Also, we reworked our API that allows targeting several runtimes from the same config: - the new API is more consistent, flexible, and powerful. -For example, if you want to execute your benchmarking using .NET Framework 4.8 and .NET Core 3.1, - you can use the `SimpleJob(RuntimeMoniker.Net48)`, `[SimpleJob(RuntimeMoniker.NetCoreApp31)]` attributes or - `Job.Default.With(ClrRuntime.Net48)`, `Job.Default.With(CoreRuntime.Core31)` jobs in a manual config. - -Now let's discuss how to use it in detail. -If you want to test multiple frameworks, your project file **MUST target all of them** and you **MUST install the corresponding SDKs**: - -```xml -netcoreapp3.0;netcoreapp2.1;net48 -``` - -If you run your benchmarks without specifying any custom settings, BenchmarkDotNet is going to run the benchmarks **using the same framework as the host process** (it corresponds to `RuntimeMoniker.HostProcess`): - -```cmd -dotnet run -c Release -f netcoreapp2.1 # is going to run the benchmarks using .NET Core 2.1 -dotnet run -c Release -f netcoreapp3.0 # is going to run the benchmarks using .NET Core 3.0 -dotnet run -c Release -f net48 # is going to run the benchmarks using .NET 4.8 -mono $pathToExe # is going to run the benchmarks using Mono from your PATH -``` - -To run the benchmarks for multiple runtimes with a single command from the command line, you need to specify the runtime moniker names via `--runtimes|-r` console argument: - -```cmd -dotnet run -c Release -f netcoreapp2.1 --runtimes netcoreapp2.1 netcoreapp3.0 # is going to run the benchmarks using .NET Core 2.1 and .NET Core 3.0 -dotnet run -c Release -f netcoreapp2.1 --runtimes netcoreapp2.1 net48 # is going to run the benchmarks using .NET Core 2.1 and .NET 4.8 -``` - -What is going to happen if you provide multiple Full .NET Framework monikers? -Let's say: - -```cmd -dotnet run -c Release -f net461 net472 net48 -``` - -Full .NET Framework always runs every .NET executable using the latest .NET Framework available on a given machine. -If you try to run the benchmarks for a few .NET TFMs, they are all going to be executed using the latest .NET Framework from your machine. -The only difference is that they are all going to have different features enabled depending on the target version they were compiled for. -You can read more about this - [here](https://learn.microsoft.com/dotnet/framework/migration-guide/version-compatibility) and - [here](https://learn.microsoft.com/dotnet/framework/migration-guide/application-compatibility). -This is **.NET Framework behavior which can not be controlled by BenchmarkDotNet or any other tool**. - -**Note:** Console arguments support works only if you pass the `args` to `BenchmarkSwitcher`: - -```cs -class Program -{ - static void Main(string[] args) - => BenchmarkSwitcher - .FromAssembly(typeof(Program).Assembly) - .Run(args); // crucial to make it work -} -``` - -You can achieve the same thing using `[SimpleJobAttribute]`: - -```cs -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Jobs; - -namespace BenchmarkDotNet.Samples -{ - [SimpleJob(RuntimeMoniker.Net48)] - [SimpleJob(RuntimeMoniker.Mono)] - [SimpleJob(RuntimeMoniker.NetCoreApp21)] - [SimpleJob(RuntimeMoniker.NetCoreApp30)] - public class TheClassWithBenchmarks -``` - -Or using a custom config: - -```cs -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Running; - -namespace BenchmarkDotNet.Samples -{ - class Program - { - static void Main(string[] args) - { - var config = DefaultConfig.Instance - .With(Job.Default.With(CoreRuntime.Core21)) - .With(Job.Default.With(CoreRuntime.Core30)) - .With(Job.Default.With(ClrRuntime.Net48)) - .With(Job.Default.With(MonoRuntime.Default)); - - BenchmarkSwitcher - .FromAssembly(typeof(Program).Assembly) - .Run(args, config); - } - } -} -``` - -The recommended way of running the benchmarks for multiple runtimes is to use the `--runtimes` console line argument. -By using the console line argument, you don't need to edit the source code anytime you want to change the list of runtimes. -Moreover, if you share the source code of the benchmark, other people can run it even if they don't have the exact same framework version installed. - -## Official templates for BenchmarkDotNet-based projects - -Since v0.12.0, BenchmarkDotNet provides project templates to setup your benchmarks easily. -The template exists for each major .NET language ([C#](https://learn.microsoft.com/dotnet/csharp/), [F#](https://learn.microsoft.com/dotnet/fsharp/) and [VB](https://learn.microsoft.com/dotnet/visual-basic/)) with equivalent features and structure. -The templates require the [.NET Core SDK](https://www.microsoft.com/net/download). Once installed, run the following command to install the templates: - -```log -dotnet new -i BenchmarkDotNet.Templates -``` - -If you want to uninstall all BenchmarkDotNet templates: - -```log -dotnet new -u BenchmarkDotNet.Templates -``` - -The template is a NuGet package distributed over nuget.org: [BenchmarkDotNet.Templates](https://www.nuget.org/packages/BenchmarkDotNet.Templates/). -To create a new C# benchmark library project from the template, run: - -```log -dotnet new benchmark -``` - -If you'd like to create F# or VB project, you can specify project language with `-lang` option: - -```log -dotnet new benchmark -lang F# -dotnet new benchmark -lang VB -``` - -The template projects have five additional options - all of them are optional. -By default, a class library project targeting netstandard2.0 is created. -You can specify `-f` or `--frameworks` to change target to one or more frameworks: - -```log -dotnet new benchmark -f netstandard2.0;net472 -``` - -The option `--console-app` creates a console app project targeting `netcoreapp3.0` with an entry point: - -```log -dotnet new benchmark --console-app -``` - -This lets you run the benchmarks from a console (`dotnet run`) or from your favorite IDE. -The option `-f` or `--frameworks` will be ignored when `--console-app` is set. -The option `-b` or `--benchmarkName` sets the name of the benchmark class: - -```log -dotnet new benchmark -b Md5VsSha256 -``` - -BenchmarkDotNet lets you create a dedicated configuration class (see [Configs](xref:docs.configs)) to customize the execution of your benchmarks. -To create a benchmark project with a configuration class, use the option `-c` or `--config`: - -```log -dotnet new benchmark -c -``` - -The option `--no-restore` if specified, skips the automatic NuGet restore after the project is created: - -```log -dotnet new benchmark --no-restore -``` - -Use the `-h` or `--help` option to display all possible arguments with a description and the default values: - -```log -dotnet new benchmark --help -``` - -The version of the template NuGet package is synced with the [BenchmarkDotNet](https://www.nuget.org/packages/BenchmarkDotNet/) package. -For instance, the template version `0.12.0` is referencing [BenchmarkDotnet 0.12.0](https://www.nuget.org/packages/BenchmarkDotNet/0.12.0) - there is no floating version behavior. -For more info about the `dotnet new` CLI, please read [the documentation](https://learn.microsoft.com/dotnet/core/tools/dotnet). - -## New NativeMemoryProfiler - -`NativeMemoryProfiler` measure the native memory traffic and adds the extra columns `Allocated native memory` and `Native memory leak` to the summary table. -Internally, it uses `EtwProfiler` to profile the code using ETW. - -Consider the following benchmark: - -```cs -[ShortRunJob] -[NativeMemoryProfiler] -[MemoryDiagnoser] -public class IntroNativeMemory -{ - [Benchmark] - public void BitmapWithLeaks() - { - var flag = new Bitmap(200, 100); - var graphics = Graphics.FromImage(flag); - var blackPen = new Pen(Color.Black, 3); - graphics.DrawLine(blackPen, 100, 100, 500, 100); - } - - [Benchmark] - public void Bitmap() - { - using (var flag = new Bitmap(200, 100)) - { - using (var graphics = Graphics.FromImage(flag)) - { - using (var blackPen = new Pen(Color.Black, 3)) - { - graphics.DrawLine(blackPen, 100, 100, 500, 100); - } - } - } - } - - private const int Size = 20; // Greater value could cause System.OutOfMemoryException for test with memory leaks. - private int ArraySize = Size * Marshal.SizeOf(typeof(int)); - - [Benchmark] - public unsafe void AllocHGlobal() - { - IntPtr unmanagedHandle = Marshal.AllocHGlobal(ArraySize); - Span unmanaged = new Span(unmanagedHandle.ToPointer(), ArraySize); - Marshal.FreeHGlobal(unmanagedHandle); - } - - [Benchmark] - public unsafe void AllocHGlobalWithLeaks() - { - IntPtr unmanagedHandle = Marshal.AllocHGlobal(ArraySize); - Span unmanaged = new Span(unmanagedHandle.ToPointer(), ArraySize); - } -} -``` - -It will produce the summary table like this one: - -| Method | Mean | Error | StdDev | Gen 0 | Gen 1 | Gen 2 | Allocated | Allocated native memory | Native memory leak | -|---------------------- |-------------:|--------------:|-------------:|------:|------:|------:|----------:|------------------------:|-------------------:| -| BitmapWithLeaks | 73,456.43 ns | 22,498.10 ns | 1,233.197 ns | - | - | - | 177 B | 13183 B | 11615 B | -| Bitmap | 91,590.08 ns | 101,468.12 ns | 5,561.810 ns | - | - | - | 180 B | 12624 B | - | -| AllocHGlobal | 79.91 ns | 43.93 ns | 2.408 ns | - | - | - | - | 80 B | - | -| AllocHGlobalWithLeaks | 103.50 ns | 153.21 ns | 8.398 ns | - | - | - | - | 80 B | 80 B | - -As you can see, we have two additional columns `Allocated native memory` and `Native memory leak` that contain some very useful numbers! - -## New ThreadingDiagnoser - -`ThreadingDiagnoser` also adds two extra columns to the summary table: - -* `Completed Work Items`: The number of work items that have been processed in ThreadPool (per single operation) -* `Lock Contentions`: The number of times there **was contention** upon trying to take a Monitor's lock (per single operation) - -Internally, it uses [new APIs](https://github.com/dotnet/corefx/issues/35500) exposed in .NET Core 3.0. - -It can be activated with the help of the `[ThreadingDiagnoser]` attribute: - -```cs -[ThreadingDiagnoser] -public class IntroThreadingDiagnoser -{ - [Benchmark] - public void CompleteOneWorkItem() - { - ManualResetEvent done = new ManualResetEvent(initialState: false); - ThreadPool.QueueUserWorkItem(m => (m as ManualResetEvent).Set(), done); - done.WaitOne(); - } -} -``` - -The above example will print a summary table like this one: - -| Method | Mean | StdDev | Median | Completed Work Items | Lock Contentions | -|-------------------- |--------------:|-----------:|--------------:|---------------------:|-----------------:| -| CompleteOneWorkItem | 8,073.5519 ns | 69.7261 ns | 8,111.6074 ns | 1.0000 | - | - - -## LINQPad 6 support - -Now both LINQPad 5 and LINQPad 6 are supported: - -![](https://user-images.githubusercontent.com/6011991/64907554-5d77b000-d6f4-11e9-9684-50ea678647eb.png) - -## Fast documentation search - -We continue to improve the usability of our documentation. -In this release, we improved the search experience in the documentation: now it works almost instantly with the help of [Algolia](https://www.algolia.com/) engine! -That's how it looks: - -![](https://user-images.githubusercontent.com/2259237/67472874-701dc700-f65a-11e9-85f1-d4f3160f78cf.gif) diff --git a/docs/_changelog/header/v0.12.1.md b/docs/_changelog/header/v0.12.1.md deleted file mode 100644 index 8378e7da70..0000000000 --- a/docs/_changelog/header/v0.12.1.md +++ /dev/null @@ -1,294 +0,0 @@ -## Highlights - -* **.NET 5 support** - As you probably know, .NET Core 5 was officially [rebranded](https://github.com/dotnet/runtime/pull/33694) to .NET 5. - The new version of BenchmarkDotNet supports the new runtime after rebranding. - [#1399](https://github.com/dotnet/BenchmarkDotNet/pull/1399) - [465ebf](https://github.com/dotnet/BenchmarkDotNet/commit/465ebf3fdbf20f0e9219c4c957fb33c13256fdcd) -* **Perfolizer adoption** - The internal statistical engine of BenchmarkDotNet became mature enough to be transformed into an independent project. - Meet [perfolizer](https://github.com/AndreyAkinshin/perfolizer) — a toolkit for performance analysis! - While BenchmarkDotNet focuses on obtaining reliable measurements, perfolizer focuses on the decent analysis of measured data. - You still can use all the statistical algorithms from BenchmarkDotNet, - but you can also install perfolizer as a [standalone NuGet package](https://www.nuget.org/packages/Perfolizer/). - You can find more details in the [official announcement](https://aakinshin.net/posts/introducing-perfolizer/). - [#1386](https://github.com/dotnet/BenchmarkDotNet/pull/1386) - [54a061](https://github.com/dotnet/BenchmarkDotNet/commit/54a06102a6e0cc4169d23c6f9cd2779ee408d2bf) -* **Cross-platform disassembler** - Now the `DisassemblyDiagnoser` is cross-platform! - The disassembling logic was also improved, now it handles runtime helper methods and references to method tables properly. - Internally, it uses the [Iced](https://github.com/0xd4d/iced) library for formatting assembly code. - Special thanks to [@adamsitnik](https://github.com/adamsitnik) for the implementation and [@0xd4d](https://github.com/0xd4d) for Iced! - [#1332](https://github.com/dotnet/BenchmarkDotNet/pull/1332) - [#899](https://github.com/dotnet/BenchmarkDotNet/issues/899) - [#1316](https://github.com/dotnet/BenchmarkDotNet/issues/1316) - [#1364](https://github.com/dotnet/BenchmarkDotNet/issues/1364) - [294320](https://github.com/dotnet/BenchmarkDotNet/commit/294320be9525b0ecfefd0351381756d3a3b77211) -* **EventPipe-based cross-platform profiler** - Now you can easily profiler your benchmarks on Windows, Linux, and macOS! - Just mark your class with the `[EventPipeProfiler(...)]` attribute and get a `.speedscope.json` file that you can browse in [SpeedScope](https://www.speedscope.app/). - Special thanks to [@WojciechNagorski](https://github.com/WojciechNagorski) for the implementation! - [#1321](https://github.com/dotnet/BenchmarkDotNet/pull/1321) - [#1315](https://github.com/dotnet/BenchmarkDotNet/issues/1315) - [c648ff](https://github.com/dotnet/BenchmarkDotNet/commit/c648ff956662abae512c579ffa7f1dc12178f6c3) -* **New fluent API** - We continue to improve our API and make it easier for reading and writing. - Special thanks to [@WojciechNagorski](https://github.com/WojciechNagorski) for the implementation! - [#1273](https://github.com/dotnet/BenchmarkDotNet/pull/1273) - [#1234](https://github.com/dotnet/BenchmarkDotNet/issues/1234) - [640d88](https://github.com/dotnet/BenchmarkDotNet/commit/640d885ae0daddcee7c9ba9b5f1bf5790b9b5ae3) -* **Ref readonly support** - Now you can use `ref readonly` in benchmark signatures. - Special thanks to [@adamsitnik](https://github.com/adamsitnik) for the implementation! - [#1389](https://github.com/dotnet/BenchmarkDotNet/pull/1389) - [#1388](https://github.com/dotnet/BenchmarkDotNet/issues/1388) - [9ac777](https://github.com/dotnet/BenchmarkDotNet/commit/9ac7770682a45afb6cf4ec353f9fa3c69ece67ce) - -## Cross-platform disassembler - -Just mark your benchmark class with the `[DisassemblyDiagnoser]` attribute - and you will get the disassembly listings for all the benchmarks. -The formatting looks pretty nice thanks to [Iced](https://github.com/0xd4d/iced). -It works like a charm on Windows, Linux, and macOS. - -```cs -[DisassemblyDiagnoser] -public class IntroDisassembly -{ - private int[] field = Enumerable.Range(0, 100).ToArray(); - - [Benchmark] - public int SumLocal() - { - var local = field; // we use local variable that points to the field - - int sum = 0; - for (int i = 0; i < local.Length; i++) - sum += local[i]; - - return sum; - } - - [Benchmark] - public int SumField() - { - int sum = 0; - for (int i = 0; i < field.Length; i++) - sum += field[i]; - - return sum; - } -} -``` - -**.NET Core 2.1.16 (CoreCLR 4.6.28516.03, CoreFX 4.6.28516.10), X64 RyuJIT** - -```x86asm -; BenchmarkDotNet.Samples.IntroDisassembly.SumLocal() - mov rax,[rcx+8] - xor edx,edx - xor ecx,ecx - mov r8d,[rax+8] - test r8d,r8d - jle short M00_L01 -M00_L00: - movsxd r9,ecx - add edx,[rax+r9*4+10] - inc ecx - cmp r8d,ecx - jg short M00_L00 -M00_L01: - mov eax,edx - ret -; Total bytes of code 35 -``` - -**.NET Core 2.1.16 (CoreCLR 4.6.28516.03, CoreFX 4.6.28516.10), X64 RyuJIT** - -```x86asm -; BenchmarkDotNet.Samples.IntroDisassembly.SumField() - sub rsp,28 - xor eax,eax - xor edx,edx - mov rcx,[rcx+8] - cmp dword ptr [rcx+8],0 - jle short M00_L01 -M00_L00: - mov r8,rcx - cmp edx,[r8+8] - jae short M00_L02 - movsxd r9,edx - add eax,[r8+r9*4+10] - inc edx - cmp [rcx+8],edx - jg short M00_L00 -M00_L01: - add rsp,28 - ret -M00_L02: - call CORINFO_HELP_RNGCHKFAIL - int 3 -; Total bytes of code 53 -``` - -Now we handle runtime helper methods and references to method tables properly. Example: - -Before: - -```x86asm -; MicroBenchmarks.WithCallsAfter.Benchmark(Int32) - push rsi - sub rsp,20h - mov rsi,rcx - cmp edx,7FFFFFFFh - jne M00_L00 - call MicroBenchmarks.WithCallsAfter.Static() - mov rcx,rsi - call MicroBenchmarks.WithCallsAfter.Instance() - mov rcx,rsi - call MicroBenchmarks.WithCallsAfter.Recursive() - mov rcx,rsi - mov rax,qword ptr [rsi] - mov rax,qword ptr [rax+40h] - call qword ptr [rax+20h] - mov rcx,rsi - mov edx,1 - mov rax,7FF8F4217050h - add rsp,20h - pop rsi - jmp rax -M00_L00: - mov rcx,offset System_Private_CoreLib+0xa31d48 - call coreclr!MetaDataGetDispenser+0x322a0 - mov rsi,rax - mov ecx,0ACFAh - mov rdx,7FF8F42F4680h - call coreclr!MetaDataGetDispenser+0x17140 - mov rdx,rax - mov rcx,rsi - call System.InvalidOperationException..ctor(System.String) - mov rcx,rsi - call coreclr!coreclr_shutdown_2+0x39f0 - int 3 - add byte ptr [rax],al - sbb dword ptr [00007ff9`26284e30],eax - add dword ptr [rax+40h],esp - add byte ptr [rax],al - add byte ptr [rax],al - add byte ptr [rax],al - add byte ptr [rax-70BC4CCh],ah -; Total bytes of code 157 -``` - -After: - -```x86asm -; BenchmarkDotNet.Samples.WithCallsAfter.Benchmark(Int32) - push rsi - sub rsp,20 - mov rsi,rcx - cmp edx,7FFFFFFF - jne M00_L00 - call BenchmarkDotNet.Samples.WithCallsAfter.Static() - mov rcx,rsi - call BenchmarkDotNet.Samples.WithCallsAfter.Instance() - mov rcx,rsi - call BenchmarkDotNet.Samples.WithCallsAfter.Recursive() - mov rcx,rsi - mov rax,[rsi] - mov rax,[rax+40] - call qword ptr [rax+20] - mov rcx,rsi - mov edx,1 - mov rax BenchmarkDotNet.Samples.WithCallsAfter.Benchmark(Boolean) - add rsp,20 - pop rsi - jmp rax -M00_L00: - mov rcx MT_System.InvalidOperationException - call CORINFO_HELP_NEWSFAST - mov rsi,rax - mov ecx,12D - mov rdx,7FF954FF83F0 - call CORINFO_HELP_STRCNS - mov rdx,rax - mov rcx,rsi - call System.InvalidOperationException..ctor(System.String) - mov rcx,rsi - call CORINFO_HELP_THROW - int 3 -; Total bytes of code 134 -``` - -See also: [Cross-runtime .NET disassembly with BenchmarkDotNet](https://aakinshin.net/posts/dotnet-crossruntime-disasm/). - -Special thanks to [@adamsitnik](https://github.com/adamsitnik) for the implementation and [@0xd4d](https://github.com/0xd4d) for Iced! - -## EventPipe-based cross-platform profiler - -Now you can easily profiler your benchmarks on Windows, Linux, and macOS! - -If you want to use the new profiler, you should just mark your benchmark class with the `[EventPipeProfiler(...)]` attribute: - -``` -[EventPipeProfiler(EventPipeProfile.CpuSampling)] // <-- Enables new profiler -public class IntroEventPipeProfiler -{ - [Benchmark] - public void Sleep() => Thread.Sleep(2000); -} -``` - -Once the benchmark run is finished, you get a `.speedscope.json` file that can be opened in [SpeedScope](https://www.speedscope.app/): - -![](https://wojciechnagorski.com/images/EventPipeProfiler/SpeedScope.png#mid) - -The new profiler supports several modes: - -* `CpuSampling` - Useful for tracking CPU usage and general .NET runtime information. This is the default option. -* `GcVerbose` - Tracks GC collections and samples object allocations. -* `GcCollect` - Tracks GC collections only at very low overhead. -* `Jit` - Logging when Just in time (JIT) compilation occurs. Logging of the internal workings of the Just In Time compiler. This is fairly verbose. It details decisions about interesting optimization (like inlining and tail call) - -Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.com/2020/04/cross-platform-profiling-.net-code-with-benchmarkdotnet/) for all the details. - -Special thanks to [@WojciechNagorski](https://github.com/WojciechNagorski) for the implementation! - -## New fluent API - -We continue to improve our API and make it easier for reading and writing. -The old API is still existing, but it is marked as obsolete and will be removed in the further library versions. -The most significant changes: - -**Changes in Job configuration** - -![](https://user-images.githubusercontent.com/17333903/66208963-b49af000-e6b6-11e9-87b3-1e4bb5519273.PNG) - -**Changes in IConfig/ManualConfig** - -![](https://user-images.githubusercontent.com/17333903/66208975-b6fd4a00-e6b6-11e9-94cd-85c73ae0ec27.PNG) - -**Full fluent API** - -![](https://user-images.githubusercontent.com/17333903/66208977-b8c70d80-e6b6-11e9-8e43-6dac3921c121.PNG) - -Special thanks to [@WojciechNagorski](https://github.com/WojciechNagorski) for the implementation! - -## Ref readonly support - -Now you can use `ref readonly` in benchmark signatures. -Here is an example: - -```cs -public class RefReadonlyBenchmark -{ - static readonly int[] array = { 1 }; - - [Benchmark] - public ref readonly int RefReadonly() => ref RefReadonlyMethod(); - - static ref readonly int RefReadonlyMethod() => ref array[0]; -} -``` - -Special thanks to [@adamsitnik](https://github.com/adamsitnik) for the implementation! \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.0.md b/docs/_changelog/header/v0.13.0.md deleted file mode 100644 index d0fca1471d..0000000000 --- a/docs/_changelog/header/v0.13.0.md +++ /dev/null @@ -1,368 +0,0 @@ -It's been a year since our last release. BenchmarkDotNet has been downloaded more than seven million times from [nuget.org](https://www.nuget.org/packages/BenchmarkDotNet/). It's more than we could have ever possibly imagined! Some could say, that it's also more than we can handle ;) That is why we wanted to once again thank [all the contributors](https://github.com/dotnet/BenchmarkDotNet/graphs/contributors) who helped us with `0.13.0` release! - -## Highlights - -In BenchmarkDotNet v0.13.0, we have supported various technologies: - -* .NET 5 and .NET 6 target framework monikers -* .NET SDK installed via snap -* SingleFile deployment -* Xamarin applications -* WASM applications -* Mono AOT - -We have also introduced new features and improvements including: - -* Memory randomization -* Method-specific job attributes -* Sortable parameter columns -* Customizable ratio column -* Improved CoreRun and CoreRT support -* Improved Hardware Counters support - -Of course, this release includes dozens of other improvements and bug fixes! - -## Supported technologies - -### .NET 5, .NET 6, SingleFile and snap - -At some point in time, `netcoreapp5.0` moniker was changed to `net5.0`, which required a fix on our side ([#1479](https://github.com/dotnet/BenchmarkDotNet/pull/1479), btw we love this kind of changes). Moreover, .NET 5 introduced platform-specific TMFs (example: `net5.0-windows10.0.19041.0`) which also required some extra work: [#1560](https://github.com/dotnet/BenchmarkDotNet/pull/1560), [#1691](https://github.com/dotnet/BenchmarkDotNet/pull/1691). - - - In [#1523](https://github.com/dotnet/BenchmarkDotNet/pull/1523) support for .NET 6 was added. - -```xml -net5.0;net5.0-windows10.0.19041.0;net6.0 -``` - -In [#1686](https://github.com/dotnet/BenchmarkDotNet/pull/1686) [@am11](https://github.com/am11) has implemented support for **single file deployment** (supported in .NET 5 onwards). - -Last, but not least in [#1652](https://github.com/dotnet/BenchmarkDotNet/pull/1652) **snap** support has been implemented. - -```log -adam@adsitnik-ubuntu:~/projects/BenchmarkDotNet/samples/BenchmarkDotNet.Samples$ dotnet50 run -c Release -f net5.0 --filter BenchmarkDotNet.Samples.IntroColdStart.Foo -// Validating benchmarks: -// ***** BenchmarkRunner: Start ***** -// ***** Found 1 benchmark(s) in total ***** -// ***** Building 1 exe(s) in Parallel: Start ***** -// start /snap/dotnet-sdk/112/dotnet restore /p:UseSharedCompilation=false /p:BuildInParallel=false /m:1 /p:Deterministic=true /p:Optimize=true in /home/adam/projects/BenchmarkDotNet/samples/BenchmarkDotNet.Samples/bin/Release/net5.0/9a018ee4-0f33-46dd-9093-01d3bf31233b -// command took 1.49s and exited with 0 -// start /snap/dotnet-sdk/112/dotnet build -c Release --no-restore /p:UseSharedCompilation=false /p:BuildInParallel=false /m:1 /p:Deterministic=true /p:Optimize=true in /home/adam/projects/BenchmarkDotNet/samples/BenchmarkDotNet.Samples/bin/Release/net5.0/9a018ee4-0f33-46dd-9093-01d3bf31233b -// command took 2.78s and exited with 0 -// ***** Done, took 00:00:04 (4.37 sec) ***** -// Found 1 benchmarks: -// IntroColdStart.Foo: Job-NECTOD(IterationCount=5, RunStrategy=ColdStart) - -// ************************** -// Benchmark: IntroColdStart.Foo: Job-NECTOD(IterationCount=5, RunStrategy=ColdStart) -// *** Execute *** -// Launch: 1 / 1 -// Execute: /snap/dotnet-sdk/112/dotnet "9a018ee4-0f33-46dd-9093-01d3bf31233b.dll" --benchmarkName "BenchmarkDotNet.Samples.IntroColdStart.Foo" --job "IterationCount=5, RunStrategy=ColdStart" --benchmarkId 0 in /home/adam/projects/BenchmarkDotNet/samples/BenchmarkDotNet.Samples/bin/Release/net5.0/9a018ee4-0f33-46dd-9093-01d3bf31233b/bin/Release/net5.0 -``` - -### Xamarin support - -Thanks to the contributions of the amazing [@jonathanpeppers](https://github.com/jonathanpeppers) BenchmarkDotNet supports Xamarin! The examples can be found in our repo: [iOS](https://github.com/dotnet/BenchmarkDotNet/tree/master/samples/BenchmarkDotNet.Samples.iOS), [Android](https://github.com/dotnet/BenchmarkDotNet/tree/master/samples/BenchmarkDotNet.Samples.Android). - -![](https://user-images.githubusercontent.com/840039/73184210-b105fb80-40e1-11ea-9efb-d29fe4d08c79.png#mid) - -[#1360](https://github.com/dotnet/BenchmarkDotNet/pull/1360), [#1429](https://github.com/dotnet/BenchmarkDotNet/pull/1429), [#1434](https://github.com/dotnet/BenchmarkDotNet/pull/1434), [#1509](https://github.com/dotnet/BenchmarkDotNet/pull/1509) - -### WASM support - -Thanks to the work of [@naricc](https://github.com/naricc) you can now benchmark WASM using Mono Runtime! For more details, please refer to our [docs](https://benchmarkdotnet.org/articles/configs/toolchains.html#Wasm). - -![](https://user-images.githubusercontent.com/6011991/87671577-dd860080-c771-11ea-922f-e26a32a3f831.png#mid) - -[#1483](https://github.com/dotnet/BenchmarkDotNet/pull/1483), [#1498](https://github.com/dotnet/BenchmarkDotNet/pull/1498), [#1500](https://github.com/dotnet/BenchmarkDotNet/pull/1500), [#1501](https://github.com/dotnet/BenchmarkDotNet/pull/1501), [#1507](https://github.com/dotnet/BenchmarkDotNet/pull/1507), [#1592](https://github.com/dotnet/BenchmarkDotNet/pull/1592), [#1689](https://github.com/dotnet/BenchmarkDotNet/pull/1689). - -### Mono AOT support - -In another awesome contribution ([#1662](https://github.com/dotnet/BenchmarkDotNet/pull/1662)) [@naricc](https://github.com/naricc) has implemented Mono AOT support. The new toolchain supports doing Mono AOT runs with both the Mono-Mini compiler and the Mono-LLVM compiler (which uses LLVM on the back end). - - For more details, please go to our [docs](https://benchmarkdotnet.org/articles/configs/toolchains.html#MonoAotLLVM). - -## New features and improvements - -### Memory randomization - -In [#1587](https://github.com/dotnet/BenchmarkDotNet/pull/1587) [@adamsitnik](https://github.com/adamsitnik) has introduced a new, **experimental** feature called "Memory Randomization". - -This feature allows you to ask BenchmarkDotNet to randomize the memory alignment by allocating random-sized byte arrays between iterations and **call [GlobalSetup] before every benchmark iteration** and `[GlobalCleanup]` after every benchmark iteration. - -Sample benchmark: - -```cs -public class IntroMemoryRandomization -{ - [Params(512 * 4)] - public int Size; - - private int[] _array; - private int[] _destination; - - [GlobalSetup] - public void Setup() - { - _array = new int[Size]; - _destination = new int[Size]; - } - - [Benchmark] - public void Array() => System.Array.Copy(_array, _destination, Size); -} -``` - -Without asking for the randomization, the objects are allocated in `[GlobalSetup]` and their unmodified addresses (and alignment) are used for all iterations (as long as they are not promoted to an older generation by the GC). This is typically the desired behavior, as it gives you very nice and flat distributions: - -```cmd -dotnet run -c Release --filter IntroMemoryRandomization -``` - -```ini --------------------- Histogram -------------------- -[502.859 ns ; 508.045 ns) | @@@@@@@@@@@@@@@ ---------------------------------------------------- -``` - -But if for some reason you are interested in getting a distribution that is better reflecting the "real-life" performance you can enable the randomization: - -```cmd -dotnet run -c Release --filter IntroMemoryRandomization --memoryRandomization true -``` - -```ini --------------------- Histogram -------------------- -[108.803 ns ; 213.537 ns) | @@@@@@@@@@@@@@@ -[213.537 ns ; 315.458 ns) | -[315.458 ns ; 446.853 ns) | @@@@@@@@@@@@@@@@@@@@ -[446.853 ns ; 559.259 ns) | @@@@@@@@@@@@@@@ ---------------------------------------------------- -``` - -### Method-specific job attributes - -From now, all attributes that derive from `JobMutatorConfigBaseAttribute` ([full list](https://benchmarkdotnet.org/api/BenchmarkDotNet.Attributes.JobMutatorConfigBaseAttribute.html#BenchmarkDotNet_Attributes_JobMutatorConfigBaseAttribute)) can be applied to methods. You no longer have to move a method to a separate type to customize config for it. - -```cs -[Benchmark] -[WarmupCount(1)] -public void SingleWarmupIteration() - -[Benchmark] -[WarmupCount(9)] -public void NineWarmupIterations() -``` - -### Sortable parameter columns - -In order to sort columns of parameters in the results table, you can use the Property `Priority` inside the params attribute. The priority range is `[Int32.MinValue;Int32.MaxValue]`, lower priorities will appear earlier in the column order. The default priority is set to `0`. - -```cs -public class IntroParamsPriority -{ - [Params(100)] - public int A { get; set; } - - [Params(10, Priority = -100)] - public int B { get; set; } - - [Benchmark] - public void Benchmark() => Thread.Sleep(A + B + 5); -} -``` - -| Method | B | A | Mean | Error | StdDev | -|---------- |--- |---- |---------:|--------:|--------:| -| Benchmark | 10 | 100 | 115.4 ms | 0.12 ms | 0.11 ms | - - -This feature got implemented by [@JohannesDeml](https://github.com/JohannesDeml) in [#1612](https://github.com/dotnet/BenchmarkDotNet/pull/1612). - -### Customizable ratio column - -Now it's possible to customize the format of the ratio column. - -```cs -[Config(typeof(Config))] -public class IntroRatioStyle -{ - [Benchmark(Baseline = true)] - public void Baseline() => Thread.Sleep(1000); - - [Benchmark] - public void Bar() => Thread.Sleep(150); - - [Benchmark] - public void Foo() => Thread.Sleep(1150); - - private class Config : ManualConfig - { - public Config() - { - SummaryStyle = SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend); - } - } -} -``` - -``` -| Method | Mean | Error | StdDev | Ratio | RatioSD | -|--------- |-----------:|--------:|--------:|-------------:|--------:| -| Baseline | 1,000.6 ms | 2.48 ms | 0.14 ms | baseline | | -| Bar | 150.9 ms | 1.30 ms | 0.07 ms | 6.63x faster | 0.00x | -| Foo | 1,150.4 ms | 5.17 ms | 0.28 ms | 1.15x slower | 0.00x | -``` - -This feature was implemented in [#731](https://github.com/dotnet/BenchmarkDotNet/issues/721). - -### Improved CoreRun support - -BenchmarkDotNet was reporting invalid .NET Core version number when comparing performance using CoreRuns built from `dotnet/corefx` and `dotnet/runtime`. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1580](https://github.com/dotnet/BenchmarkDotNet/pull/1580) - -In [#1552](https://github.com/dotnet/BenchmarkDotNet/pull/1552) [@stanciuadrian](https://github.com/stanciuadrian) has implemented support for all `GcMode` characteristics for `CoreRunToolchain`. Previously the settings were just ignored, now they are being translated to corresponding `COMPlus_*` env vars. - -### Improved CoreRT support - -CoreRT has moved from https://github.com/dotnet/corert/ to https://github.com/dotnet/runtimelab/tree/feature/NativeAOT and we had to update the default compiler version and nuget feed address. Moreover, there was a bug in `CoreRtToolchain` which was causing any additional native dependencies to not work. - -Big thanks to [@MichalStrehovsky](https://github.com/MichalStrehovsky), [@jkotas](https://github.com/jkotas) and [@kant2002](https://github.com/kant2002) for their help and support! - -[#1606](https://github.com/dotnet/BenchmarkDotNet/pull/1606), [#1643](https://github.com/dotnet/BenchmarkDotNet/pull/1643), [#1679](https://github.com/dotnet/BenchmarkDotNet/pull/1679) - -### Command-line argument support in BenchmarkRunner - -So far only `BenchmarkSwitcher` was capable of handling console line arguments. Thanks to [@chan18](https://github.com/chan18) `BenchmarkRunner` supports them as well ([#1292](https://github.com/dotnet/BenchmarkDotNet/pull/1292)): - -```cs -public class Program -{ - public static void Main(string[] args) => BenchmarkRunner.Run(typeof(Program).Assembly, args: args); -} -``` - -### New API: ManualConfig.CreateMinimumViable - -`ManualConfig.CreateEmpty` creates a completely empty config. Without adding a column provider and a logger to the config the users won't see any results being printed. In [#1582](https://github.com/dotnet/BenchmarkDotNet/pull/1582) [@adamsitnik](https://github.com/adamsitnik) has introduced a new method that creates minimum viable config: - -```cs -IConfig before = ManualConfig.CreateEmpty() - .AddColumnProvider(DefaultColumnProviders.Instance) - .AddLogger(ConsoleLogger.Default); - -IConfig after = ManualConfig.CreateMinimumViable(); -``` - -### Benchmarking NuGet packages from custom feeds - -In [#1659](https://github.com/dotnet/BenchmarkDotNet/pull/1659/) [@workgroupengineering](https://github.com/workgroupengineering) added the possibility to indicate the source of the tested nuget package and whether it is a pre-release version. - -### Deterministic benchmark builds - -BenchmarkDotNet is now always enforcing Deterministic builds ([#1489](https://github.com/dotnet/BenchmarkDotNet/pull/1489)) and Optimizations enabled ([#1494](https://github.com/dotnet/BenchmarkDotNet/pull/1494)) which is a must-have if you are using custom build configurations. MSBuild enforces optimizations **only** for configurations that are named `Release` (the comparison is case-insensitive). - -```xml - - - - - - -``` - -```cs -var config = DefaultConfig.Instance - .AddJob(Job.Default.WithCustomBuildConfiguration("X").WithId("X").AsBaseline()) - .AddJob(Job.Default.WithCustomBuildConfiguration("Y").WithId("Y")); -``` - -[#1489](https://github.com/dotnet/BenchmarkDotNet/pull/1489), [#1494](https://github.com/dotnet/BenchmarkDotNet/pull/1494) - -### Improved Hardware Counters support - -BenchmarkDotNet is being used by the .NET Team to ensure that .NET is not regressing. More than three thousand benchmarks (they can be found [here](https://github.com/dotnet/performance/tree/main/src/benchmarks/micro)) are being executed multiple times a day on multiple hardware configs. Recently, .NET Team started to use `InstructionsRetired` to help to filter unstable benchmarks that report regressions despite not changing the number of instructions retired. This has exposed few bugs in Hardware Counters support in BenchmarkDotNet, which all got fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1547](https://github.com/dotnet/BenchmarkDotNet/pull/1547) and [#1550](https://github.com/dotnet/BenchmarkDotNet/pull/1550). Moreover, we have **removed** the old [PmcDiagnoser](https://adamsitnik.com/Hardware-Counters-Diagnoser/) and extended [EtwProfiler](https://adamsitnik.com/ETW-Profiler/) with the hardware counters support. It's just much more accurate and futureproof. For details, please go to [#1548](https://github.com/dotnet/BenchmarkDotNet/pull/1548). - -How stable was `PmcDiagnoser` (same benchmarks run twice in a row on the same machine): - -| Method | Runtime | InstructionRetired/Op | -|---------- |--------------------- |----------------------:| -| Burgers_0 | .NET 5.0 | 845,746 | -| Burgers_0 | .NET Core 2.1 | 30,154,151 | -| Burgers_0 | .NET Framework 4.6.1 | 4,230,848 | - -| Method | Runtime | InstructionRetired/Op | -|---------- |--------------------- |----------------------:| -| Burgers_0 | .NET 5.0 | 34,154,524 | -| Burgers_0 | .NET Core 2.1 | 246,534,203 | -| Burgers_0 | .NET Framework 4.6.1 | 2,607,686 | - -How stable is `EtwProfiler`: - -| Method | Runtime | InstructionRetired/Op | -|---------- |--------------------- |----------------------:| -| Burgers_0 | .NET 5.0 | 3,069,978,261 | -| Burgers_0 | .NET Core 2.1 | 3,676,000,000 | -| Burgers_0 | .NET Framework 4.6.1 | 3,468,866,667 | - -| Method | Runtime | InstructionRetired/Op | -|---------- |--------------------- |----------------------:| -| Burgers_0 | .NET 5.0 | 3,066,810,000 | -| Burgers_0 | .NET Core 2.1 | 3,674,666,667 | -| Burgers_0 | .NET Framework 4.6.1 | 3,468,600,000 | - -Moreover, in [#1540](https://github.com/dotnet/BenchmarkDotNet/pull/1540) [@WojciechNagorski](https://github.com/WojciechNagorski) has added the removal of temporary files created by `EtwProfiler`. - -### Improved Troubleshooting - -We have the possibility to ask BDN to stop on the first error: `--stopOnFirstError true`. - -The problem was when the users had not asked for that, tried to run `n` benchmarks, all of them failed to build, and BDN was printing the same build error `n` times. - -In [#1672](https://github.com/dotnet/BenchmarkDotNet/pull/1672) [@adamsitnik](https://github.com/adamsitnik) has changed that, so when all the build fails, BDN stops after printing the first error. - -Moreover, we have also changed the default behavior for the failed builds of the boilerplate code. If the build fails, we don't remove the files. Previously we have required the users to pass `--keepFiles` to keep them. See [#1567](https://github.com/dotnet/BenchmarkDotNet/pull/1567) for more details and don't forget about the [Troubleshooting](https://benchmarkdotnet.org/articles/guides/troubleshooting.html) docs! - -### Docs and Samples improvements - -Big thanks to [@lukasz-pyrzyk](https://github.com/lukasz-pyrzyk), [@fleckert](https://github.com/fleckert), [@MarecekF](https://github.com/MarecekF), [@joostas](https://github.com/joostas), [@michalgalecki](https://github.com/michalgalecki), [@WojciechNagorski](https://github.com/WojciechNagorski), [@MendelMonteiro](https://github.com/MendelMonteiro), [@kevinsalimi](https://github.com/kevinsalimi), [@cedric-cf](https://github.com/cedric-cf), [@YohDeadfall](https://github.com/YohDeadfall), [@jeffhandley](https://github.com/jeffhandley) and [@JohannesDeml](https://github.com/JohannesDeml) who have improved our docs and samples! - -[#1463](https://github.com/dotnet/BenchmarkDotNet/pull/1463), [#1465](https://github.com/dotnet/BenchmarkDotNet/pull/1465), [#1508](https://github.com/dotnet/BenchmarkDotNet/pull/1508), [#1518](https://github.com/dotnet/BenchmarkDotNet/pull/1518), [#1554](https://github.com/dotnet/BenchmarkDotNet/pull/1554), [#1568](https://github.com/dotnet/BenchmarkDotNet/pull/1568), [#1601](https://github.com/dotnet/BenchmarkDotNet/pull/1601), [#1633](https://github.com/dotnet/BenchmarkDotNet/pull/1633), [#1645](https://github.com/dotnet/BenchmarkDotNet/pull/1645), [#1647](https://github.com/dotnet/BenchmarkDotNet/pull/1647), [#1657](https://github.com/dotnet/BenchmarkDotNet/pull/1657), [#1675](https://github.com/dotnet/BenchmarkDotNet/pull/1675), [#1676](https://github.com/dotnet/BenchmarkDotNet/pull/1676), [#1690](https://github.com/dotnet/BenchmarkDotNet/pull/1690). - -### Template improvements - -* Projects created out of our official templates might have been unexpectedly packed when running `dotnet pack` on the entire solution. In [#1584](https://github.com/dotnet/BenchmarkDotNet/pull/1584) [@kendaleiv](https://github.com/kendaleiv) has explicitly disabled packing for the template. -* The template had `netcoreapp3.0` TFM hardcoded. This got fixed by [@https://github.com/ExceptionCaught](https://github.com/ExceptionCaught) in [#1630](https://github.com/dotnet/BenchmarkDotNet/pull/1630) and [#1632](https://github.com/dotnet/BenchmarkDotNet/pull/1632). -* In [#1667](https://github.com/dotnet/BenchmarkDotNet/pull/1667) [@YohDeadfall](https://github.com/YohDeadfall) has changed the default debug type from `portable` to `pdbonly` (required by `DisassemblyDiagnoser`). - -## Bug fixes - - * Very long string `[Arguments]` and `[Params]` were causing BenchmarkDotNet to crash. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1248](https://github.com/dotnet/BenchmarkDotNet/pull/1248) and [#1545](https://github.com/dotnet/BenchmarkDotNet/pull/1545/). So far trace file names were containing full benchmark names and arguments. Now if the name is too long, the trace file name is a hash (consistent for multiple runs of the same benchmark). The same goes for passing benchmark name by the host process to the benchmark process via command-line arguments. -* `LangVersion` set to a non-numeric value like `latest` was crashing the build. Fixed by [@martincostello](https://github.com/martincostello) in [#1420](https://github.com/dotnet/BenchmarkDotNet/pull/1420). -* Windows 10 November 201**9** was being recognized as 201**8**. Fixed by [@kapsiR](https://github.com/kapsiR) in [#1437](https://github.com/dotnet/BenchmarkDotNet/pull/1437). -* Assemblies loaded via streams were not supported. Fixed by [@jeremyosterhoudt](https://github.com/jeremyosterhoudt) in [#1443](https://github.com/dotnet/BenchmarkDotNet/pull/1443). -* [NativeMemoryProfiler](https://wojciechnagorski.com/2019/08/analyzing-native-memory-allocation-with-benchmarkdotnet/) was detecting small leaks that were false positives. Fixed by [@WojciechNagorski](https://github.com/WojciechNagorski) in [#1451](https://github.com/dotnet/BenchmarkDotNet/pull/1451) and [#1600](https://github.com/dotnet/BenchmarkDotNet/pull/1600). -* [DisassemblyDiagnoser](https://adamsitnik.com/Disassembly-Diagnoser/) was crashing on Linux. Fixed by [@damageboy](https://github.com/damageboy) in [#1459](https://github.com/dotnet/BenchmarkDotNet/pull/1459). -* Target framework moniker was being printed as toolchain name for Full .NET Framework benchmarks. Fixed by [@svick](https://github.com/svick) in [#1471](https://github.com/dotnet/BenchmarkDotNet/pull/1471). -* `[ParamsSource]` returning `IEnumerable` was not working properly when combined with `[Arguments]`. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1478](https://github.com/dotnet/BenchmarkDotNet/pull/1478). -* `NullReferenceException` in `MultimodalDistributionAnalyzer`. Fixed by [@suslovk](https://github.com/suslovk) in [#1488](https://github.com/dotnet/BenchmarkDotNet/pull/1488). -* `NotSupportedException` was being thrown if there was an encoding mismatch between host and benchmark process. Diagnosed by [@ChristophLindemann](https://github.com/ChristophLindemann) in [#1487](https://github.com/dotnet/BenchmarkDotNet/issues/1487), fixed by [@lovettchris](https://github.com/lovettchris) in [#1491](https://github.com/dotnet/BenchmarkDotNet/pull/1491). -* `MissingMethodException` was being thrown in projects that referenced a newer version of [Iced](https://github.com/icedland/iced). Fixed by [@Symbai](https://github.com/Symbai) in [#1497](https://github.com/dotnet/BenchmarkDotNet/pull/1497) and in [#1502](https://github.com/dotnet/BenchmarkDotNet/pull/1502). -* `AppendTargetFrameworkToOutputPath` set to `false` was not supported. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1563](https://github.com/dotnet/BenchmarkDotNet/pull/1563) -* A locking finalizer could have hanged benchmark process which would just print `// AfterAll` and never quit. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1571](https://github.com/dotnet/BenchmarkDotNet/pull/1571). To prevent other hangs from happening, a timeout of `250ms` was added. If the process does not quit after running the benchmarks and waiting `250ms`, it's being force killed. -* In some cases, `JsonExporter` was reporting `NaN` for some of the Statistics. This was breaking non-.NET JSON deserializers. Fixed by [@marcnet80](https://github.com/marcnet80) in [#1581](https://github.com/dotnet/BenchmarkDotNet/pull/1581). -* `UnitType.Size` metrics were not using the provided number format. Fixed by [@jodydonetti](https://github.com/jodydonetti) in [#1569](https://github.com/dotnet/BenchmarkDotNet/pull/1569) and [#1618](https://github.com/dotnet/BenchmarkDotNet/pull/1618). -* `MaxColumnWidth` setting was not used for type names. Fixed by [@JohannesDeml](https://github.com/JohannesDeml) in [#1609](https://github.com/dotnet/BenchmarkDotNet/pull/1609). -* Current culture was not respected when formatting `Ratio` column values. Fixed by [@JohannesDeml](https://github.com/JohannesDeml) in [#1610](https://github.com/dotnet/BenchmarkDotNet/pull/1610). -* BenchmarkDotNet was redirecting Standard Error of the benchmark process, which was causing deadlocks for benchmarks that were writing to it. Fixed by [@adamstinik](https://github.com/adamsitnik) in [#1631](https://github.com/dotnet/BenchmarkDotNet/pull/1631) -* `DisassemblyDiagnoser` was failing to disassemble multiline source code. [@YohDeadfall](https://github.com/YohDeadfall) fixed that in [#1674](https://github.com/dotnet/BenchmarkDotNet/pull/1674). -* In [#1644](https://github.com/dotnet/BenchmarkDotNet/pull/1644) [@adamstinik](https://github.com/adamsitnik) has fixed the inconsistency between benchmark filter and hint. - -## Removal of the dotnet global tool - -In [#1006](https://github.com/dotnet/BenchmarkDotNet/pull/1006) (0.11.4) we have introduced a new dotnet global tool. - -By looking at the number of reported bugs we got to the conclusion that the tool has not passed the test of time. - -Why? Because it was based entirely on dynamic assembly loading which is very hard to get right in .NET and the fact that we support all existing .NET Runtimes (.NET, .NET Core, Mono, CoreRT) made it even harder (if not impossible). - -**We have removed it and the old versions are no longer supported**. For more details, please refer to [#1572](https://github.com/dotnet/BenchmarkDotNet/pull/1572). \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.1.md b/docs/_changelog/header/v0.13.1.md deleted file mode 100644 index 5744a4f2e2..0000000000 --- a/docs/_changelog/header/v0.13.1.md +++ /dev/null @@ -1,15 +0,0 @@ -BenchmarkDotNet v0.13.1 is a service update with various bug fixes and improvements. - -## Highlights - -* S390x architecture support ([#1712](https://github.com/dotnet/BenchmarkDotNet/pull/1712)) -* Various WASM toolchain improvements ([#1719](https://github.com/dotnet/BenchmarkDotNet/pull/1719), [#1722](https://github.com/dotnet/BenchmarkDotNet/pull/1722), [#1729](https://github.com/dotnet/BenchmarkDotNet/pull/1729), [#1742](https://github.com/dotnet/BenchmarkDotNet/pull/1742), [#1743](https://github.com/dotnet/BenchmarkDotNet/pull/1743), [#1744](https://github.com/dotnet/BenchmarkDotNet/pull/1744), [#1746](https://github.com/dotnet/BenchmarkDotNet/pull/1746), [#1757](https://github.com/dotnet/BenchmarkDotNet/pull/1757), [#1763](https://github.com/dotnet/BenchmarkDotNet/pull/1763)) -* Support of CoreRT on 5.0.3XX SDK ([#1725](https://github.com/dotnet/BenchmarkDotNet/pull/1725)) -* Using Utf8 for reading from standard input (fixes a nasty encoding-related bug) ([#1735](https://github.com/dotnet/BenchmarkDotNet/pull/1735)) -* Adjusting WaitForExit timeouts ([#1745](https://github.com/dotnet/BenchmarkDotNet/pull/1745)) -* Support for returning unmanaged types from benchmarks with InProcessToolchain ([#1750](https://github.com/dotnet/BenchmarkDotNet/pull/1750)) -* Disabled Tiered JIT ([#1751](https://github.com/dotnet/BenchmarkDotNet/pull/1751)) -* A MemoryDiagnoser bug fix ([#1762](https://github.com/dotnet/BenchmarkDotNet/pull/1762)) -* Allow users to hide Gen X columns ([#1764](https://github.com/dotnet/BenchmarkDotNet/pull/1764)) -* Copy GC settings from host process to the benchmark process ([#1765](https://github.com/dotnet/BenchmarkDotNet/pull/1765)) -* Do not split surrogates in shortified parameter values ([#1705](https://github.com/dotnet/BenchmarkDotNet/pull/1705)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.10.md b/docs/_changelog/header/v0.13.10.md deleted file mode 100644 index 9f55725d19..0000000000 --- a/docs/_changelog/header/v0.13.10.md +++ /dev/null @@ -1,3 +0,0 @@ -## Highlights - -Initial support of .NET 9 and minor bug fixes. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.11.md b/docs/_changelog/header/v0.13.11.md deleted file mode 100644 index ef2d1b9579..0000000000 --- a/docs/_changelog/header/v0.13.11.md +++ /dev/null @@ -1,3 +0,0 @@ -## Highlights - -Small improvements. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.12.md b/docs/_changelog/header/v0.13.12.md deleted file mode 100644 index c1d788166c..0000000000 --- a/docs/_changelog/header/v0.13.12.md +++ /dev/null @@ -1,10 +0,0 @@ -## Highlights - -The biggest highlight of this release if our new VSTest Adapter, - which allows to run benchmarks as unit tests in your favorite IDE! -The detailed guide can be found [here](xref:docs.vstest). - -This release also includes to a minor bug fix that caused incorrect job id generation: - fixed job id generation ([#2491](https://github.com/dotnet/BenchmarkDotNet/pull/2491)). - -Also, the target framework in the BenchmarkDotNet templates was bumped to .NET 8.0. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.2.md b/docs/_changelog/header/v0.13.2.md deleted file mode 100644 index dd5365e56e..0000000000 --- a/docs/_changelog/header/v0.13.2.md +++ /dev/null @@ -1,254 +0,0 @@ -## Highlights - -In BenchmarkDotNet v0.13.2, we have implemented support for: - -* .NET 7 -* NativeAOT -* .NET Framework 4.8.1 - -We have also introduced new features and improvements including: - -* Possibility to hide selected columns, -* Allocation Ratio column, -* Logging progress and estimated finish time, -* ARM64 support for `BenchmarkDotNet.Diagnostics.Windows` package, -* Printing Hardware Intrinsics information, -* Glob filters support for DisassemblyDiagnoser. - -Of course, this release includes dozens of other improvements and bug fixes! - -Our special thanks go to [@mawosoft](https://github.com/mawosoft), [@YegorStepanov](https://github.com/YegorStepanov) and [@radical](https://github.com/radical) who fixed a LOT of really nasty bugs. - -## Supported technologies - -### .NET 7 and .NET Framework 4.8.1 - -.NET 4.8.1 has been [released](https://devblogs.microsoft.com/dotnet/announcing-dotnet-framework-481/) earlier this month, while .NET 7 should land in autumn this year. Now you can use BenchmarkDotNet to compare both! - -```ini -BenchmarkDotNet=v0.13.1.1845-nightly, OS=Windows 11 (10.0.22622.575) -Microsoft SQ1 3.0 GHz, 1 CPU, 8 logical and 8 physical cores -.NET SDK=7.0.100-preview.6.22352.1 - [Host] : .NET 7.0.0 (7.0.22.32404), Arm64 RyuJIT AdvSIMD - Job-QJVIDT : .NET 7.0.0 (7.0.22.32404), Arm64 RyuJIT AdvSIMD - Job-FNNXOY : .NET Framework 4.8.1 (4.8.9032.0), Arm64 RyuJIT -``` - -| Method | Runtime | Mean | Allocated | -|-------------- |--------------------- |---------:|----------:| -| BinaryTrees_2 | .NET 7.0 | 193.6 ms | 227.33 MB | -| BinaryTrees_2 | .NET Framework 4.8.1 | 192.8 ms | 228.01 MB | - -Credit for adding .NET 7 support in [#1816](https://github.com/dotnet/BenchmarkDotNet/pull/1816) goes to [@am11](https://github.com/am11). [@adamsitnik](https://github.com/adamsitnik) implemented .NET 4.8.1 support in [#2044](https://github.com/dotnet/BenchmarkDotNet/pull/2044) and [#2067](https://github.com/dotnet/BenchmarkDotNet/pull/2067). Big thanks to [@MichalPetryka](https://github.com/MichalPetryka) who was using preview versions of BenchmarkDotNet and reported a bug related to .NET 4.8.1 support: [#2059](https://github.com/dotnet/BenchmarkDotNet/issues/2059) that got fixed before we released a new version. - -### NativeAOT - -We are really excited to see the experimental CoreRT project grow and become officially supported by Microsoft (under new name: NativeAOT)! You can read more about it [here](https://devblogs.microsoft.com/dotnet/announcing-dotnet-7-preview-3/#what-is-native-aot). Implementing and improving the support was a combined effort of multiple contributors that spawned across multiple repositories: -* [@MichalStrehovsky](https://github.com/MichalStrehovsky): [#66290 in dotnet/runtime](https://github.com/dotnet/runtime/pull/66290), [#2020 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2020), [#2046 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2046) -* [@hez2010](https://github.com/hez2010): [#66650 in dotnet/runtime](https://github.com/dotnet/runtime/pull/66650) -* [@Beau-Gosse-dev](https://github.com/Beau-Gosse-dev): [#1955 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1955) -* [@adamsitnik](https://github.com/adamsitnik): [#1960 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1960), [#1965 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1965), [#1972 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1972), [#1973 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1973), [#1994 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1994), [#1997 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1997), [#2045 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2045), [#2068 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2068) -* [@kant2002](https://github.com/kant2002): [#1976 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1976), [#1979 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1979) -* [@jkotas](https://github.com/jkotas): [#68038 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68038), [#68142 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68142), [#68249 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68249), [#68308 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68308), [#68375 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68375) -* [@MichalPetryka](https://github.com/MichalPetryka): [#2065 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2065) - - -As every AOT solution, NativeAOT has some [limitations](https://github.com/dotnet/runtime/blob/main/src/coreclr/nativeaot/docs/limitations.md) like limited reflection support or lack of dynamic assembly loading. Because of that, the host process (what you run from command line) is never an AOT process, but just a regular .NET process. This process (called Host process) uses reflection to read benchmarks metadata (find all `[Benchmark]` methods etc.), generates a new project that references the benchmarks and compiles it using ILCompiler. The boilerplate code is not using reflection, so the project is built with `TrimmerDefaultAction=link` (we have greatly reduced build time thanks to that). Such compilation produces a native executable, which is later started by the Host process. This process (called Benchmark or Child process) performs the actual benchmarking and reports the results back to the Host process. By default BenchmarkDotNet uses the latest version of `Microsoft.DotNet.ILCompiler` to build the NativeAOT benchmark according to [this instructions](https://github.com/dotnet/runtime/blob/main/src/coreclr/nativeaot/docs/compiling.md). Moreover, BenchmarkDotNet by default uses current machines CPU features (change: [#1994](https://github.com/dotnet/BenchmarkDotNet/pull/1994), discussion: [#2061](https://github.com/dotnet/BenchmarkDotNet/issues/2061)) and if you don't like this behavior, you can [disable it](https://github.com/dotnet/BenchmarkDotNet/issues/2061#issuecomment-1203602177). - -This is why you need to: -- install [pre-requisites](https://learn.microsoft.com/dotnet/core/deploying/native-aot/#prerequisites) required by NativeAOT compiler -- target .NET to be able to run NativeAOT benchmarks (example: `net7.0` in the .csproj file) -- run the app as a .NET process (example: `dotnet run -c Release -f net7.0`). -- specify the NativeAOT runtime in an explicit way, either by using console line arguments `--runtimes nativeaot7.0` (the recommended approach), or by using`[SimpleJob]` attribute or by using the fluent Job config API `Job.ShortRun.With(NativeAotRuntime.Net70)`: - -```cmd -dotnet run -c Release -f net7.0 --runtimes nativeaot7.0 -``` - -For more examples please go to [docs](https://benchmarkdotnet.org/articles/configs/toolchains.html#nativeaot). - -```ini -BenchmarkDotNet=v0.13.1.1845-nightly, OS=Windows 11 (10.0.22000.856/21H2) -AMD Ryzen Threadripper PRO 3945WX 12-Cores, 1 CPU, 24 logical and 12 physical cores -.NET SDK=7.0.100-rc.1.22423.16 - [Host] : .NET 7.0.0 (7.0.22.42223), X64 RyuJIT AVX2 - Job-KDVXET : .NET 7.0.0 (7.0.22.42223), X64 RyuJIT AVX2 - Job-HFRAGK : .NET 7.0.0-rc.1.22424.9, X64 NativeAOT AVX2 -``` - -| Method | Runtime | Mean | Ratio | Allocated | -|-------------- |-------------- |---------:|------:|----------:| -| BinaryTrees_2 | .NET 7.0 | 95.06 ms | 1.00 | 227.33 MB | -| BinaryTrees_2 | NativeAOT 7.0 | 90.32 ms | 0.96 | 227.33 MB | - - -Some of .NET features are not supported by Native AOT, that is why you may want to filter them out using new `[AotFilter]` attribute: - -```cs -[AotFilter("Currently not supported due to missing metadata.")] -public class Xml_FromStream -``` - -## New features and improvements - -### Hiding Columns - -In [#1621](https://github.com/dotnet/BenchmarkDotNet/pull/1621) [@marcnet80](https://github.com/marcnet80) has reduced the number of columns displayed when multiple runtimes are being compared. - -![](https://user-images.githubusercontent.com/6011991/99952390-defab180-2d7f-11eb-851c-ffef6fbb0647.png#mid) - -In [#1890](https://github.com/dotnet/BenchmarkDotNet/pull/1890) [@YegorStepanov](https://github.com/YegorStepanov) has implemented a set of new APIs that allow for hiding columns. It's also exposed via `-h` and `--hide` command line arguments. - -```cs -[MemoryDiagnoser] // adds Gen0, Gen1, Gen2 and Allocated Bytes columns -[HideColumns(Column.Gen0, Column.Gen1, Column.Gen2)] // dont display GenX columns -public class IntroHidingColumns -{ - [Benchmark] - public byte[] AllocateArray() => new byte[100_000]; -} -``` - -Sample results without `[HideColumns]`: - - -| Method | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------------- |---------:|----------:|----------:|--------:|--------:|--------:|----------:| -| AllocateArray | 3.303 us | 0.0465 us | 0.0435 us | 31.2462 | 31.2462 | 31.2462 | 97.69 KB | - -With: - -| Method | Mean | Error | StdDev | Allocated | -|-------------- |---------:|----------:|----------:|----------:| -| AllocateArray | 3.489 us | 0.0662 us | 0.0763 us | 97.69 KB | - -Imagine how much time [@YegorStepanov](https://github.com/YegorStepanov) has saved to all the people who so far were removing the columns manually from the results before publishing them on GitHub! - -### Allocation Ratio Column - -In [#1859](https://github.com/dotnet/BenchmarkDotNet/pull/1859) [@YegorStepanov](https://github.com/YegorStepanov) has added Allocation Ratio Column. It's enabled by default when `MemoryDiagnoser` is used and one of the benchmarks is marked as `[Benchmark(Baseline = true)]` or when there are multuple jobs defined and one of them is marked as baseline. - -```cs -[MemoryDiagnoser] -public class AllocationColumnSample -{ - [Benchmark(Baseline = true)] - [Arguments("test")] - public string Builder(string value) - { - StringBuilder sb = new (value); - - for (int i = 0; i < 10; i++) - sb.Append(value); - - return sb.ToString(); - } - - [Benchmark] - [Arguments("test")] - public string Concatenation(string value) - { - string result = value; - - for (int i = 0; i < 10; i++) - result += value; - - return result; - } -} -``` - -| Method | value | Mean | Error | StdDev | Ratio | Gen 0 | Allocated | Alloc Ratio | -|-------------- |------ |---------:|--------:|--------:|------:|-------:|----------:|------------:| -| Builder | test | 127.9 ns | 0.49 ns | 0.43 ns | 1.00 | 0.0544 | 456 B | 1.00 | -| Concatenation | test | 120.2 ns | 0.94 ns | 0.88 ns | 0.94 | 0.0908 | 760 B | 1.67 | - -### Progress and estimated finish time - -In [#1909](https://github.com/dotnet/BenchmarkDotNet/pull/1909) [@adamsitnik](https://github.com/adamsitnik) has added logging of progress and estimated finish time. - -```log -// ** Remained 5211 (99.9%) benchmark(s) to run. Estimated finish 2022-08-25 22:26 (9h 7m from now) ** -``` - -### arm64 support for BenchmarkDotNet.Diagnostics.Windows package - -Due to the [update](https://github.com/dotnet/BenchmarkDotNet/pull/2030) to [TraceEvent 3.0](https://www.nuget.org/packages/Microsoft.Diagnostics.Tracing.TraceEvent) `BenchmarkDotNet.Diagnostics.Windows` package has now arm64 support. Which means that you can use `EtwProfiler` and other ETW-based diagnosers on Windows arm64. - -It would not be possible without [@brianrob](https://github.com/brianrob) who implemented arm64 support for TraceEvent in [#1533](https://github.com/microsoft/perfview/pull/1533)! - -### Hardware Intrinsics information - -In [#2051](https://github.com/dotnet/BenchmarkDotNet/pull/2051) [@adamsitnik](https://github.com/adamsitnik) has extended the hardware information printed in the Summary table with Hardware Intrinsics information. - -![](https://user-images.githubusercontent.com/6011991/180951306-bf8c9f8f-469a-4d7c-8ee1-d7fa47acc0e1.png#mid) - -Sine the space in Summary table is quite limited, we full information is printed only in the log: - -![](https://user-images.githubusercontent.com/6011991/180951531-38c6b6d0-ba6f-4766-a305-b2e306e8420b.png#mid) - -Special thanks to [@tannergooding](https://github.com/tannergooding) who provided a lot of [very valuable feedback](https://github.com/dotnet/BenchmarkDotNet/pull/2051#issuecomment-1194368152) and [@MichalPetryka](https://github.com/MichalPetryka) who contributed an improvement [#2066](https://github.com/dotnet/BenchmarkDotNet/pull/2066) for older runtimes. - -### Other improvements - -* WASM toolchain has received a lot of improvements from various .NET Team members: [#1769](https://github.com/dotnet/BenchmarkDotNet/pull/1769), [#1936](https://github.com/dotnet/BenchmarkDotNet/pull/1936), [#1938](https://github.com/dotnet/BenchmarkDotNet/pull/1938), [#1982](https://github.com/dotnet/BenchmarkDotNet/pull/1982). -* Dependencies and TFMs updates: [#1805](https://github.com/dotnet/BenchmarkDotNet/pull/1805), [#1978](https://github.com/dotnet/BenchmarkDotNet/pull/1978), [#2012](https://github.com/dotnet/BenchmarkDotNet/pull/2012), [#2019](https://github.com/dotnet/BenchmarkDotNet/pull/2019), [#2035](https://github.com/dotnet/BenchmarkDotNet/pull/2035). -* Ensure proper SummaryStyle handling implemented by [@mawosoft](https://github.com/mawosoft) in [#1828](https://github.com/dotnet/BenchmarkDotNet/pull/1828). -* Preserving `EnablePreviewFeatures` project setting which gives the possibility to benchmark preview .NET features. Implemented by [@kkokosa](https://github.com/kkokosa) in [#1842](https://github.com/dotnet/BenchmarkDotNet/pull/1842). -* CI: Using non-deprecated macOS pool on Azure Pipelines, implemented by [@akoeplinger](https://github.com/akoeplinger) in [#1847](https://github.com/dotnet/BenchmarkDotNet/pull/1847) -* CI: Updating Cake to 2.0.0, adopting frosting project style. Implemented by [@AndreyAkinshin](https://github.com/AndreyAkinshin) in [#1865](https://github.com/dotnet/BenchmarkDotNet/pull/1865). -* Detecting ReSharper's Dynamic Program Analysis. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1874](https://github.com/dotnet/BenchmarkDotNet/pull/1874). -* Preventing benchmark failure when some of the exporters fail. Implemented by [@epeshk](https://github.com/epeshk) in [#1902](https://github.com/dotnet/BenchmarkDotNet/pull/1902). -* Don't use the diagnosers when benchmarking has failed. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1903](https://github.com/dotnet/BenchmarkDotNet/pull/1903). -* Ensuring the default order of benchmarks is the same as declared in source code. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1907](https://github.com/dotnet/BenchmarkDotNet/pull/1907). -* Making `BuildTimeout` configurable. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1906](https://github.com/dotnet/BenchmarkDotNet/pull/1906). -* Notify users about private methods with Setup/Cleanup attributes. Implemented by [@epeshk](https://github.com/epeshk) in [#1912](https://github.com/dotnet/BenchmarkDotNet/pull/1912). -* Don't run Roslyn Analyzers for the generated code. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1917](https://github.com/dotnet/BenchmarkDotNet/pull/1917). -* Ensure `WorkloadActionUnroll` and similar are optimized if possible. Implemented by [@AndyAyersMS](https://github.com/AndyAyersMS) in [#1935](https://github.com/dotnet/BenchmarkDotNet/pull/1935). -* Don't use blocking acknowledgments when there is no need to. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1940](https://github.com/dotnet/BenchmarkDotNet/pull/1940). -* Executor: Don't use Process.ExitCode, unless the process has exited. Implemented by [@radical](https://github.com/radical) in [#1947](https://github.com/dotnet/BenchmarkDotNet/pull/1947). -* Revise heuristic for initial jitting. Implemented by [@AndyAyersMS](https://github.com/AndyAyersMS) in [#1949](https://github.com/dotnet/BenchmarkDotNet/pull/1949). -* Allow logging build commands output. Implemented by [@radical](https://github.com/radical) in [#1950](https://github.com/dotnet/BenchmarkDotNet/pull/1950). -* Change Mono AOT mode to Normal AOT with LLVM JIT fall back. Implemented by [@fanyang-mono](https://github.com/fanyang-mono) in [#1990](https://github.com/dotnet/BenchmarkDotNet/pull/1990). - -### Glob filters support for DisassemblyDiagnoser - -So far, the disassembler was always loading the type that was generated by BDN, searching for the benchmark method, disassembling it and when encountered direct method calls, disassembling the called methods as well (if their depth was lesser or equal to max configured depth). - -This was working fine, but only for direct method calls. For indirect, the disassembly was incomplete. - -In [#2072](https://github.com/dotnet/BenchmarkDotNet/pull/2072) [@adamsitnik](https://github.com/adamsitnik) has added the possibility to filter methods disassembled by the `DisassemblyDiagnoser`. - -The users can now pass `--disasmFilter $globPattern` and it's going to be applied to full signatures of all methods available for disassembling. Examples: -* `--disasmFilter *System.Text*` - disassemble all `System.Text` methods. -* `--disasmFilter *` - disassemble all possible methods. - -Moreover, [ClrMD](https://github.com/microsoft/clrmd) was updated to v2 ([#2040](https://github.com/dotnet/BenchmarkDotNet/pull/2040)) and few disassembler bugs have been fixed ([#2075](https://github.com/dotnet/BenchmarkDotNet/pull/2075), [#2078](https://github.com/dotnet/BenchmarkDotNet/pull/2078)). We are expecting that the disassembler will be more reliable now. - -### Docs and Samples improvements - -Big thanks to [@SnakyBeaky](https://github.com/SnakyBeaky), [@Distinctlyminty](https://github.com/Distinctlyminty), [@asaf92](https://github.com/asaf92), [@adamsitnik](https://github.com/adamsitnik) and [@eiriktsarpalis](https://github.com/eiriktsarpalis) who have improved our docs, samples and error messages! - -[#1776](https://github.com/dotnet/BenchmarkDotNet/pull/1776), [#1797](https://github.com/dotnet/BenchmarkDotNet/pull/1797), [#1850](https://github.com/dotnet/BenchmarkDotNet/pull/1850), [#1861](https://github.com/dotnet/BenchmarkDotNet/pull/1861), [#1939](https://github.com/dotnet/BenchmarkDotNet/pull/1939), [#1974](https://github.com/dotnet/BenchmarkDotNet/pull/1974), [#1997](https://github.com/dotnet/BenchmarkDotNet/pull/1997), [#2042](https://github.com/dotnet/BenchmarkDotNet/pull/2042), [#2050](https://github.com/dotnet/BenchmarkDotNet/pull/2050), [#2068](https://github.com/dotnet/BenchmarkDotNet/pull/2068). - -## Bug fixes - -* WASM: [#1811](https://github.com/dotnet/BenchmarkDotNet/pull/1811), [#1846](https://github.com/dotnet/BenchmarkDotNet/pull/1846), [#1916](https://github.com/dotnet/BenchmarkDotNet/pull/1916), [#1926](https://github.com/dotnet/BenchmarkDotNet/pull/1926), [#1932](https://github.com/dotnet/BenchmarkDotNet/pull/1932). -* Diagnoser-provided Analysers weren't automatically added to Config. Fixed by [@mawosoft](https://github.com/mawosoft) in [#1790](https://github.com/dotnet/BenchmarkDotNet/pull/1790). -* Exportes could been duplicated. Fixed by [@workgroupengineering](https://github.com/workgroupengineering) in [#1796](https://github.com/dotnet/BenchmarkDotNet/pull/1796). -* Small bug in SummaryStyle. Fixed by [@mawosoft](https://github.com/mawosoft) in [#1801](https://github.com/dotnet/BenchmarkDotNet/pull/1801). -* `InvalidOperationException/NullReferenceException` in `SmartParaemter`. Fixed by [@mawosoft](https://github.com/mawosoft) in [#1810](https://github.com/dotnet/BenchmarkDotNet/pull/1810). -* Failures caused by colons in benchmark name. Fixed by [@ronbrogan](https://github.com/ronbrogan) in [#1823](https://github.com/dotnet/BenchmarkDotNet/pull/1823). -* Some benchmark arugments were not properly escaped and were causing process launcher to crush. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1841](https://github.com/dotnet/BenchmarkDotNet/pull/1841) -* Invalid size specifiers for Memory and Disassembly diagnosers. Fixed by [@YegorStepanov](https://github.com/YegorStepanov) in [#1854](https://github.com/dotnet/BenchmarkDotNet/pull/1854) and [#1855](https://github.com/dotnet/BenchmarkDotNet/pull/1855). -* Respect LogicalGroup order in DefaultOrderer. Fixed by [@AndreyAkinshin](https://github.com/AndreyAkinshin) in [#1866](https://github.com/dotnet/BenchmarkDotNet/pull/1866). -* Endless loop in user interaction with redirected input. Fixed by [@tmds](https://github.com/tmds) in [#](https://github.com/dotnet/BenchmarkDotNet/pull/1870). -* Broken power plan support. Fixed by [@YegorStepanov](https://github.com/YegorStepanov) in [#1885](https://github.com/dotnet/BenchmarkDotNet/pull/1885). -* `BytesAllocatedPerOperation` was not being output by the JSON and XML exporters. Fixed by [#martincostello](https://github.com/martincostello) in [#1919](https://github.com/dotnet/BenchmarkDotNet/pull/1919). -* Incorrect default InvocationCount in the summary table. Fixed by [@AndreyAkinshin](https://github.com/AndreyAkinshin) in [#1929](https://github.com/dotnet/BenchmarkDotNet/issues/1929). -* Failed build output was printed in reverse order. Fixed by [@radical](https://github.com/radical) in [#1945](https://github.com/dotnet/BenchmarkDotNet/pull/1945). -* Build failures due to `NETSDK1150`. Fixed by [@OlegOLK](https://github.com/OlegOLK) in [#1981](https://github.com/dotnet/BenchmarkDotNet/pull/1981). -* `MetricCoumn` was not respecting provided units when formatting values. Fixed by [@mawosoft](https://github.com/mawosoft) in [#2033](https://github.com/dotnet/BenchmarkDotNet/pull/2033). -* Generating invalid code that was causing benchmark failures. Fixed by [@mawosoft](https://github.com/mawosoft) in [#2041](https://github.com/dotnet/BenchmarkDotNet/pull/2041). -* CI: non-master build branches were publishing artifacts to the CI feed. Fixed by [@mawosoft](https://github.com/mawosoft) in [#2047](https://github.com/dotnet/BenchmarkDotNet/pull/2047). -* Comments in the project files were causing build failures. Fixed by [@mawosoft](https://github.com/mawosoft) in [#2056](https://github.com/dotnet/BenchmarkDotNet/pull/2056). diff --git a/docs/_changelog/header/v0.13.3.md b/docs/_changelog/header/v0.13.3.md deleted file mode 100644 index db1514edf6..0000000000 --- a/docs/_changelog/header/v0.13.3.md +++ /dev/null @@ -1,91 +0,0 @@ -## Highlights - -* New supported technologies - * Add arm64 disassembler - [#1422](https://github.com/dotnet/BenchmarkDotNet/issues/1422) - [#2127](https://github.com/dotnet/BenchmarkDotNet/pull/2127) - [#2107](https://github.com/dotnet/BenchmarkDotNet/pull/2107) - [#2123](https://github.com/dotnet/BenchmarkDotNet/pull/2123) - [#2070](https://github.com/dotnet/BenchmarkDotNet/issues/2070) - [#2118](https://github.com/dotnet/BenchmarkDotNet/pull/2118) - [#2119](https://github.com/dotnet/BenchmarkDotNet/pull/2119) - [#2234](https://github.com/dotnet/BenchmarkDotNet/pull/2234) - [#2222](https://github.com/dotnet/BenchmarkDotNet/pull/2222) - [#2212](https://github.com/dotnet/BenchmarkDotNet/pull/2212) - [9ee1/Capstone.NET#37](https://github.com/9ee1/Capstone.NET/pull/37) - * Initial .NET 8 support - [#2192](https://github.com/dotnet/BenchmarkDotNet/pull/2192) - * .NET 6/7 MonoVM support - [#2064](https://github.com/dotnet/BenchmarkDotNet/issues/2064) - [#2142](https://github.com/dotnet/BenchmarkDotNet/pull/2142) - [#2227](https://github.com/dotnet/BenchmarkDotNet/pull/2227) - [#2230](https://github.com/dotnet/BenchmarkDotNet/pull/2230) - * Armv6 and Ppc64le architectures support - [#2216](https://github.com/dotnet/BenchmarkDotNet/issues/2216) - [#2219](https://github.com/dotnet/BenchmarkDotNet/pull/2219) -* Improved support - * Improved WASM support - [#2201](https://github.com/dotnet/BenchmarkDotNet/pull/2201) - [#2099](https://github.com/dotnet/BenchmarkDotNet/issues/2099) - [#2154](https://github.com/dotnet/BenchmarkDotNet/pull/2154) - [#2112](https://github.com/dotnet/BenchmarkDotNet/pull/2112) - * Improved NativeAOT support - [#2095](https://github.com/dotnet/BenchmarkDotNet/pull/2095) - [#2221](https://github.com/dotnet/BenchmarkDotNet/pull/2221) - * Improved Android support - [#2231](https://github.com/dotnet/BenchmarkDotNet/pull/2231) - * 32-bit benchmarks can now handle addresses larger than 2GB with the help of `LargeAddressAware` - [#1469](https://github.com/dotnet/BenchmarkDotNet/issues/1469) - [#2145](https://github.com/dotnet/BenchmarkDotNet/pull/2145) - * Support 64bit affinity masks - [#2211](https://github.com/dotnet/BenchmarkDotNet/issues/2211) - [#2228](https://github.com/dotnet/BenchmarkDotNet/pull/2228) -* New features - * Add `ExceptionDiagnoser` - [#1736](https://github.com/dotnet/BenchmarkDotNet/issues/1736) - [#2169](https://github.com/dotnet/BenchmarkDotNet/pull/2169) - [#2182](https://github.com/dotnet/BenchmarkDotNet/pull/2182) - * Add `PerfCollectProfiler` - [#2117](https://github.com/dotnet/BenchmarkDotNet/pull/2117) - * Incremental benchmark execution with the help of `--resume` - [#1799](https://github.com/dotnet/BenchmarkDotNet/issues/1799) - [#2164](https://github.com/dotnet/BenchmarkDotNet/pull/2164) - * Taskbar progress - [#2102](https://github.com/dotnet/BenchmarkDotNet/issues/2102) - [#2158](https://github.com/dotnet/BenchmarkDotNet/pull/2158) - [#2140](https://github.com/dotnet/BenchmarkDotNet/pull/2140) - * Support `--noForcedGCs` to avoid forced GC between benchmark runs - [#2101](https://github.com/dotnet/BenchmarkDotNet/pull/2101) - * Added apples to apples comparison mode - [#2116](https://github.com/dotnet/BenchmarkDotNet/pull/2116) - [#2193](https://github.com/dotnet/BenchmarkDotNet/pull/2193) - * Communication between the host process and the benchmark process is now using pipes instead of output parsing - [#2092](https://github.com/dotnet/BenchmarkDotNet/pull/2092) - [#1933](https://github.com/dotnet/BenchmarkDotNet/issues/1933) - [#2189](https://github.com/dotnet/BenchmarkDotNet/issues/2189) - [#2207](https://github.com/dotnet/BenchmarkDotNet/pull/2207) - [#2200](https://github.com/dotnet/BenchmarkDotNet/pull/2200) -* Dozens of bugfixes - -## Special Thanks - -We would like to highlight some important contributors who helped us with this release: - -1. [OpenHack'22 (devexperts.com)](https://code.devexperts.com/event/openhack22) hackathon sponsored by the DevExperts company. - As part of this hackathon we have received following PRs: - * [#2132](https://github.com/dotnet/BenchmarkDotNet/pull/2132) fix: include argument and param names in --filter (by [@blouflashdb](https://github.com/blouflashdb)) - * [#2140](https://github.com/dotnet/BenchmarkDotNet/pull/2140) Update console title with benchmark information (by [@franciscomoloureiro](https://github.com/franciscomoloureiro)) - * [#2142](https://github.com/dotnet/BenchmarkDotNet/pull/2142) Issue 2064: Mono70 moniker (by [@Serg046](https://github.com/Serg046)) - * [#2148](https://github.com/dotnet/BenchmarkDotNet/pull/2148) adding validation errors when the benchmarks are unsupported (by [@emanuel-v-r](https://github.com/emanuel-v-r)) - * [#2160](https://github.com/dotnet/BenchmarkDotNet/pull/2160) Corrected logic to restore foreground color in ConsoleLogger.cs (by [@farQtech](https://github.com/farQtech)) - * [#2164](https://github.com/dotnet/BenchmarkDotNet/pull/2164) 1799 adding resume arg (by [@melias](https://github.com/melias)) - * [#2169](https://github.com/dotnet/BenchmarkDotNet/pull/2169) Issue #1736: Add ExceptionDiagnoser (by [@Serg046](https://github.com/Serg046)) - * [#2161](https://github.com/dotnet/BenchmarkDotNet/pull/2161) add quiet logger (by [@franciscomoloureiro](https://github.com/franciscomoloureiro)) **(not merged yet)** - * [#2171](https://github.com/dotnet/BenchmarkDotNet/pull/2171) Issue #1024: Calculate baseline by the fastest benchmark (by [@Serg046](https://github.com/Serg046)) **(not merged yet)** - -2. Jan Vorlicek helped to implement arm64 disassembler during an internal Microsoft Hackathon: - * [#2107](https://github.com/dotnet/BenchmarkDotNet/pull/2107) Implement TryGetReferencedAddress for relative branches (by [@janvorli](https://github.com/janvorli)) - * [#2123](https://github.com/dotnet/BenchmarkDotNet/pull/2123) Added other arm64 constant form extraction plus other changes (by [@janvorli](https://github.com/janvorli)) - -3. Ahmed Garhy (maintainer of Capstone.NET) helped to improve Capstone.NET, which was need to implement arm64 disassembler: - * [9ee1/Capstone.NET#37](https://github.com/9ee1/Capstone.NET/pull/37) Sign Assembly with a Strong Name (by [@9ee1](https://github.com/9ee1)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.4.md b/docs/_changelog/header/v0.13.4.md deleted file mode 100644 index 3ae8407ee4..0000000000 --- a/docs/_changelog/header/v0.13.4.md +++ /dev/null @@ -1,26 +0,0 @@ -## Highlights - -* Fixed LINQPad support - [#2237](https://github.com/dotnet/BenchmarkDotNet/issues/2237) - [#2238](https://github.com/dotnet/BenchmarkDotNet/pull/2238) -* New `JitStatsDiagnoser` - [#2243](https://github.com/dotnet/BenchmarkDotNet/pull/2243) -* Minor documentation improvements - [#2206](https://github.com/dotnet/BenchmarkDotNet/pull/2206) - [#2218](https://github.com/dotnet/BenchmarkDotNet/pull/2218) - -## JitStatsDiagnoser - -This new diagnoser introduced in ([#2243](https://github.com/dotnet/BenchmarkDotNet/pull/2243)) allows getting advanced JIT statistics. - -Sample usage: - -```cmd -dotnet run -c Release -f net7.0 --filter *IntroBasic.Sleep --profiler jit -``` - -Result: - -| Method | Mean | Error | StdDev | Methods JITted | Methods Tiered | JIT allocated memory | -|------- |---------:|---------:|---------:|---------------:|---------------:|---------------------:| -| Sleep | 15.53 ms | 0.034 ms | 0.032 ms | 1,102 | 15 | 221,736 B | \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.5.md b/docs/_changelog/header/v0.13.5.md deleted file mode 100644 index c3915a4f6d..0000000000 --- a/docs/_changelog/header/v0.13.5.md +++ /dev/null @@ -1,34 +0,0 @@ -## Highlights - -* Improved `JitStatsDiagnoser`. - This diagnoser was added in v0.13.4, it shows various stats from the JIT compiler that were collected during entire benchmark run - (amount of JITted methods, - amount of [tiered methods](https://learn.microsoft.com/en-us/dotnet/core/whats-new/dotnet-core-3-0#tiered-compilation), - how much memory JIT allocated during the benchmark). - In this release, we improved metric collection - ([#2246](https://github.com/dotnet/BenchmarkDotNet/pull/2246), - [e715d5](https://github.com/dotnet/BenchmarkDotNet/commit/e715d5bb63984fca65120d9a497f7d16395f9e5b)) - and added the `[JitStatsDiagnoser]` attribute - ([#2250](https://github.com/dotnet/BenchmarkDotNet/pull/2250) - [512413](https://github.com/dotnet/BenchmarkDotNet/commit/512413ceb24077154bdf6d6306138accffe64c7a)). -* Enable strong-named assemblies in the released NuGet packages - [#2258](https://github.com/dotnet/BenchmarkDotNet/issues/2258) - [#2263](https://github.com/dotnet/BenchmarkDotNet/pull/2263) - [5cd288](https://github.com/dotnet/BenchmarkDotNet/commit/5cd288996ca13292fcf638be299c097a600aea7b) -* Avoid keeping referenced values returned from a benchmark in memory - [#1942](https://github.com/dotnet/BenchmarkDotNet/issues/1942) - [#2191](https://github.com/dotnet/BenchmarkDotNet/pull/2191) - [ff5dbe](https://github.com/dotnet/BenchmarkDotNet/commit/ff5dbe662478f547e4be8d734eaeb6a106f40875) -* Keep generated files when MSBuild bin log is requested - [#2252](https://github.com/dotnet/BenchmarkDotNet/issues/2252) - [#2254](https://github.com/dotnet/BenchmarkDotNet/pull/2254) - [d3fbc0](https://github.com/dotnet/BenchmarkDotNet/commit/d3fbc03d6dabeb52f23c6b7e50287150e66957cc) -* Add `Id` for `UnresolvedDiagnoser` (an exception fix) - [#2251](https://github.com/dotnet/BenchmarkDotNet/pull/2251) - [a992b5](https://github.com/dotnet/BenchmarkDotNet/commit/a992b57490e844acf587bc2e01b08a7040dbc8e2) -* Add brand names for Windows 22H2 and macOS 13 - [86f212](https://github.com/dotnet/BenchmarkDotNet/commit/86f212b79e297d87d3942e4c50130fe6e214f3c8) - [0c2699](https://github.com/dotnet/BenchmarkDotNet/commit/0c26996ea685a99068aca71e7ae547b0851d3c64) -* Remove deprecated `InProcessToolchain` - [#2248](https://github.com/dotnet/BenchmarkDotNet/pull/2248) - [615384](https://github.com/dotnet/BenchmarkDotNet/commit/615384d2553434d7f35c03ab3174d761f82c6c2d) diff --git a/docs/_changelog/header/v0.13.6.md b/docs/_changelog/header/v0.13.6.md deleted file mode 100644 index 7667d404ce..0000000000 --- a/docs/_changelog/header/v0.13.6.md +++ /dev/null @@ -1,23 +0,0 @@ -## Highlights - -* New [BenchmarkDotNet.Diagnostics.dotTrace](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace) NuGet package. - Once this package is installed, you can annotate your benchmarks with the `[DotTraceDiagnoser]` and get a [dotTrace](https://www.jetbrains.com/profiler/) performance snapshot at the end of the benchmark run. - [#2328](https://github.com/dotnet/BenchmarkDotNet/pull/2328) -* Updated documentation website. - We migrated to [docfx](https://dotnet.github.io/docfx/) 2.67 and got the refreshed modern template based on bootstrap 5 with dark/light theme switcher. -* Updated [BenchmarkDotNet.Templates](https://www.nuget.org/packages/BenchmarkDotNet.Templates). - Multiple issues were resolved, now you can create new benchmark projects from terminal or your favorite IDE. - [#1658](https://github.com/dotnet/BenchmarkDotNet/issues/1658) - [#1881](https://github.com/dotnet/BenchmarkDotNet/issues/1881) - [#2149](https://github.com/dotnet/BenchmarkDotNet/issues/2149) - [#2338](https://github.com/dotnet/BenchmarkDotNet/pull/2338) -* Response file support. - Now it's possible to pass additional arguments to BenchmarkDotNet using `@filename` syntax. - [#2320](https://github.com/dotnet/BenchmarkDotNet/pull/2320) - [#2348](https://github.com/dotnet/BenchmarkDotNet/pull/2348) -* Custom runtime support. - [#2285](https://github.com/dotnet/BenchmarkDotNet/pull/2285) -* Introduce CategoryDiscoverer, see [`IntroCategoryDiscoverer`](xref:BenchmarkDotNet.Samples.IntroCategoryDiscoverer). - [#2306](https://github.com/dotnet/BenchmarkDotNet/issues/2306) - [#2307](https://github.com/dotnet/BenchmarkDotNet/pull/2307) -* Multiple bug fixes. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.7.md b/docs/_changelog/header/v0.13.7.md deleted file mode 100644 index 5a8487b639..0000000000 --- a/docs/_changelog/header/v0.13.7.md +++ /dev/null @@ -1,3 +0,0 @@ -## Highlights - -This release contains important bug fixes listed below. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.8.md b/docs/_changelog/header/v0.13.8.md deleted file mode 100644 index b9222853e2..0000000000 --- a/docs/_changelog/header/v0.13.8.md +++ /dev/null @@ -1,3 +0,0 @@ -## Highlights - -This release contains important bug fixes. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.9.md b/docs/_changelog/header/v0.13.9.md deleted file mode 100644 index 07e4813c9f..0000000000 --- a/docs/_changelog/header/v0.13.9.md +++ /dev/null @@ -1,3 +0,0 @@ -## Highlights - -This release contains bug fixes. \ No newline at end of file diff --git a/docs/_changelog/header/v0.14.0.md b/docs/_changelog/header/v0.14.0.md deleted file mode 100644 index 1c2f6693ce..0000000000 --- a/docs/_changelog/header/v0.14.0.md +++ /dev/null @@ -1,15 +0,0 @@ -## Highlights - -* Introduce `BenchmarkDotNet.Diagnostics.dotMemory` [#2549](https://github.com/dotnet/BenchmarkDotNet/pull/2549): memory allocation profile of your benchmarks using [dotMemory](https://www.jetbrains.com/dotmemory/), see @BenchmarkDotNet.Samples.IntroDotMemoryDiagnoser -* Introduce `BenchmarkDotNet.Exporters.Plotting` [#2560](https://github.com/dotnet/BenchmarkDotNet/pull/2560): plotting via [ScottPlot](https://scottplot.net/) (initial version) -* Multiple bugfixes -* The default build toolchains have been updated to pass `IntermediateOutputPath`, `OutputPath`, and `OutDir` properties to the `dotnet build` command. This change forces all build outputs to be placed in a new directory generated by BenchmarkDotNet, and fixes many issues that have been reported with builds. You can also access these paths in your own `.csproj` and `.props` from those properties if you need to copy custom files to the output. - -## Bug fixes - -* Fixed multiple build-related bugs including passing MsBuildArguments and .Net 8's `UseArtifactsOutput`. - -## Breaking Changes - -* `DotNetCliBuilder` removed `retryFailedBuildWithNoDeps` constructor option. -* `DotNetCliCommand` removed `RetryFailedBuildWithNoDeps` property and `BuildNoRestoreNoDependencies()` and `PublishNoBuildAndNoRestore()` methods (replaced with `PublishNoRestore()`). \ No newline at end of file diff --git a/docs/_changelog/header/v0.8.2.md b/docs/_changelog/header/v0.8.2.md deleted file mode 100644 index b06d5ec1c2..0000000000 --- a/docs/_changelog/header/v0.8.2.md +++ /dev/null @@ -1,19 +0,0 @@ -* Changes in the Summary table - * Summary table now supports ResultExtenders that can add new column to the table - * Now we use [StandardError](https://en.wikipedia.org/wiki/Standard_error) (aka `Error`) as the main accuracy metric - * Columns `op/s`, `StdDev` are disabled by default (you can add it via ResultExtenders) -* Statistic improvements, now you have detailed statistic in the console log like follows: -``` -Mean = 118.5298 us, StdError = 1.2863 us (N = 30, StdDev = 7.0454 us) -Min = 109.1602 us, Median = 117.1794 us, Max = 132.5764 us -IQR = 10.1244 us, LowerFence = 98.0834 us, UpperFence = 138.5810 us -ConfidenceInterval = [116.0086 us; 121.0510 us] (CI 95%) -``` -* Added the `Baseline` feature, see [#64](https://github.com/PerfDotNet/BenchmarkDotNet/issues/64) -* Export improvements, now you have files `-report-github.md`, `-report-stackoverflow.md` for easy publishing results on GitHub and StackOverflow. -* Basic plotting. Added new `BenchmarkRPlotExporter` that creates `BuildPlots.R` in the bin directory. It is an R script that generates boxplot and barplot for your benchmarks (you should have installed R with defined `R_HOME` environment variable) -* Updated environment info - * Added Stopwatch `Frequency` and `Resolution` - * Split common benchmark properties (like `Mode`, `Platform`, `Runtime`) in several lines (3 properties per line) -* Log improvements: add total time, statistics, list of exported files -* Bug fixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.0.md b/docs/_changelog/header/v0.9.0.md deleted file mode 100644 index 3d14c4b30c..0000000000 --- a/docs/_changelog/header/v0.9.0.md +++ /dev/null @@ -1,2 +0,0 @@ -* New API -* Autodetermination of amount iteration for warmup/target idle/main iterations, duration of iteration, amount of CLR launches. \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.2.md b/docs/_changelog/header/v0.9.2.md deleted file mode 100644 index 44d8bb5fe0..0000000000 --- a/docs/_changelog/header/v0.9.2.md +++ /dev/null @@ -1 +0,0 @@ -* Dnx451 support (Closed [#51](https://github.com/PerfDotNet/BenchmarkDotNet/issues/51), Merged [#87](https://github.com/PerfDotNet/BenchmarkDotNet/issues/87)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.3.md b/docs/_changelog/header/v0.9.3.md deleted file mode 100644 index f7614fbe00..0000000000 --- a/docs/_changelog/header/v0.9.3.md +++ /dev/null @@ -1 +0,0 @@ -* CoreCLR support (Closed [#52](https://github.com/PerfDotNet/BenchmarkDotNet/issues/52), Merged [#113](https://github.com/PerfDotNet/BenchmarkDotNet/issues/113)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.4.md b/docs/_changelog/header/v0.9.4.md deleted file mode 100644 index 63c47b49d0..0000000000 --- a/docs/_changelog/header/v0.9.4.md +++ /dev/null @@ -1,9 +0,0 @@ -* Improved messages about error in benchmarks, see [#104](https://github.com/PerfDotNet/BenchmarkDotNet/issues/104) -* Natural sort order, see [#92](https://github.com/PerfDotNet/BenchmarkDotNet/issues/92), [#95](https://github.com/PerfDotNet/BenchmarkDotNet/issues/95), [#97](https://github.com/PerfDotNet/BenchmarkDotNet/issues/97) -* Improved `double`/`float`/`decimal`/`enum` support for Params, see [#96](https://github.com/PerfDotNet/BenchmarkDotNet/issues/96), [#105](https://github.com/PerfDotNet/BenchmarkDotNet/issues/105), [#116](https://github.com/PerfDotNet/BenchmarkDotNet/issues/116) -* Now environment info includes information about `HardwareTimerKind` and `JitModules` -* Added `DryConfig` -* Improved export performance, closed [#119](https://github.com/PerfDotNet/BenchmarkDotNet/issues/119), merged [#124](https://github.com/PerfDotNet/BenchmarkDotNet/pull/124) -* Better cmd-line discoverability (see [#78](https://github.com/PerfDotNet/BenchmarkDotNet/issues/78)), e.g. run `Benchmark.exe --help` and some useful information will be printed -* Supporting all kinds of references for generated project (exact version, custom paths, GAC, sub-folders, dependent assemblies not copied), closed [#41](https://github.com/PerfDotNet/BenchmarkDotNet/issues/41), [#49](https://github.com/PerfDotNet/BenchmarkDotNet/issues/49), [#72](https://github.com/PerfDotNet/BenchmarkDotNet/issues/72), [#123](https://github.com/PerfDotNet/BenchmarkDotNet/issues/123), merged [#125](https://github.com/PerfDotNet/BenchmarkDotNet/pull/125) -* Friendliness to LinqPad restored, closed [#66](https://github.com/PerfDotNet/BenchmarkDotNet/issues/66), merged [#125](https://github.com/PerfDotNet/BenchmarkDotNet/pull/125) \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.5.md b/docs/_changelog/header/v0.9.5.md deleted file mode 100644 index 1d429dcc35..0000000000 --- a/docs/_changelog/header/v0.9.5.md +++ /dev/null @@ -1,11 +0,0 @@ -* Added validators, JitOptimizationsValidator detects all non-optimzied dlls that were referenced [#134](https://github.com/PerfDotNet/BenchmarkDotNet/issues/134) -* **Strong naming** [#101](https://github.com/PerfDotNet/BenchmarkDotNet/issues/101) -* Add `IOrderProvider` [#107](https://github.com/PerfDotNet/BenchmarkDotNet/issues/107) -* **Putting all the generated artifacts in a separate folder: ./BenchmarkDotNet.Artifacts/results** and ./BenchmarkDotNet.Artifacts/bin [#94](https://github.com/PerfDotNet/BenchmarkDotNet/issues/94) -* Printing dotnet cli version for .NET Core and Dnx451, informing user when not installed. Closed [#128](https://github.com/PerfDotNet/BenchmarkDotNet/issues/128) -* Supporting assembly redirects [#67](https://github.com/PerfDotNet/BenchmarkDotNet/issues/67) -* Changed used msbuild version: 12 for .NET 4.5 (VS 2013), 14 for .NET 4.6 (VS 2015). Closed [#132](https://github.com/PerfDotNet/BenchmarkDotNet/issues/132) and [#137](https://github.com/PerfDotNet/BenchmarkDotNet/issues/137) -* Switched to new ‘dotnet’ target framework monikers (dotnet5.4 instead of dnxcore50), [why](https://github.com/aspnet/Announcements/issues/98) -* dnx452, dnx46, net462 support added -* Executing single Benchmark for multiple Runtimes also with Diagnoser attached (see [#117](https://github.com/PerfDotNet/BenchmarkDotNet/pull/117)) -* Misc minor changes \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.6.md b/docs/_changelog/header/v0.9.6.md deleted file mode 100644 index b69b6591bd..0000000000 --- a/docs/_changelog/header/v0.9.6.md +++ /dev/null @@ -1,3 +0,0 @@ -* Added Percentiles (see [#164](https://github.com/PerfDotNet/BenchmarkDotNet/pull/164)) -* Added support for Json export (see [#84](https://github.com/PerfDotNet/BenchmarkDotNet/issues/84)) -* Bugfixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.7.md b/docs/_changelog/header/v0.9.7.md deleted file mode 100644 index 560d956baf..0000000000 --- a/docs/_changelog/header/v0.9.7.md +++ /dev/null @@ -1,2 +0,0 @@ -* .NET Core RC2 support (see [#187](https://github.com/PerfDotNet/BenchmarkDotNet/pull/187)) -* Bugfixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.8.md b/docs/_changelog/header/v0.9.8.md deleted file mode 100644 index 055ca19bf6..0000000000 --- a/docs/_changelog/header/v0.9.8.md +++ /dev/null @@ -1,8 +0,0 @@ -* CoreCLR RTM support (see [#216](https://github.com/PerfDotNet/BenchmarkDotNet/issues/216)). **Breaking change:** we have dropped dnx451 and dnxcore50 support. -* Migration from MSBuild to Roslyn, which supports Mono on Linux and MacOS (see [#149](https://github.com/PerfDotNet/BenchmarkDotNet/issues/149)). **Breaking change:** we have dropped .NET 4.0 support. -* Ability to manage GC mode: turn on/off the Server/Concurrent GC modes, extend to CPU groups, set gcAllowVeryLargeObjects and avoid BenchmarkDotNet from forcing GC.Collect (see [#188](https://github.com/PerfDotNet/BenchmarkDotNet/issues/188), [#76](https://github.com/PerfDotNet/BenchmarkDotNet/issues/76) and [#211](https://github.com/PerfDotNet/BenchmarkDotNet/issues/211)) -* Support CopyToOutput (see [#212](https://github.com/PerfDotNet/BenchmarkDotNet/issues/212)). Now you can use native dependencies as well as custom files. -* Copying custom settings from app.config (see [#108](https://github.com/PerfDotNet/BenchmarkDotNet/issues/108)). It means we support assembly binding redirects as well as custom connection strings etc. -* AsciiDocExporter (see [#169](https://github.com/PerfDotNet/BenchmarkDotNet/pull/169)) -* Framework setting in Job has been removed (see [#194](https://github.com/PerfDotNet/BenchmarkDotNet/issues/194)) -* Minor bugfixes and improvements \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.9.md b/docs/_changelog/header/v0.9.9.md deleted file mode 100644 index a3352ca136..0000000000 --- a/docs/_changelog/header/v0.9.9.md +++ /dev/null @@ -1,9 +0,0 @@ -* Attribute config style (see [#166](https://github.com/PerfDotNet/BenchmarkDotNet/issues/166)) -* [Online documentation](https://perfdotnet.github.io/BenchmarkDotNet/index.htm) (see [#219](https://github.com/PerfDotNet/BenchmarkDotNet/issues/219)) -* Mono LLVM support (see [#226](https://github.com/PerfDotNet/BenchmarkDotNet/issues/226)) -* Async method support (see [#236](https://github.com/PerfDotNet/BenchmarkDotNet/issues/236)) -* NuGet packages and repo layout restructuring (see [#225](https://github.com/PerfDotNet/BenchmarkDotNet/issues/225), [#228](https://github.com/PerfDotNet/BenchmarkDotNet/issues/228)) -* `[Cleanup]` attribute (see [#215](https://github.com/PerfDotNet/BenchmarkDotNet/issues/215)) -* New statistics columns: `Skewness`, `Kurtosis`, `WelchTTestPValue`, Improved math for the `Scaled` column -* Now current default branch is `master` -* Minor improvements and bug fixes \ No newline at end of file diff --git a/docs/articles/configs/diagnosers.md b/docs/articles/configs/diagnosers.md index cb0facc3a9..e81943310a 100644 --- a/docs/articles/configs/diagnosers.md +++ b/docs/articles/configs/diagnosers.md @@ -35,10 +35,10 @@ The current Diagnosers are: Please see Adam Sitnik's [blog post](https://adamsitnik.com/ConcurrencyVisualizer-Profiler/) for all the details. - Native Memory Profiler (`NativeMemoryProfiler`) It uses `EtwProfiler` to profile the code using ETW and adds the extra columns `Allocated native memory` and `Native memory leak`. - Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.com/2019/08/analyzing-native-memory-allocation-with-benchmarkdotnet/) for all the details. + Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.github.io/2019/08/analyzing-native-memory-allocation-with-benchmarkdotnet/) for all the details. - Event Pipe Profiler (`EventPipeProfiler`). It is a cross-platform profiler that allows profile .NET code on every platform - Windows, Linux, macOS. - Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.com/2020/04/cross-platform-profiling-.net-code-with-benchmarkdotnet/) for all the details. + Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.github.io/2020/04/cross-platform-profiling-.net-code-with-benchmarkdotnet/) for all the details. - Threading Diagnoser (`ThreadingDiagnoser`) - .NET Core 3.0+ diagnoser that reports some Threading statistics. - Exception Diagnoser (`ExceptionDiagnoser`) - a diagnoser that reports the frequency of exceptions thrown during the operation. @@ -59,11 +59,11 @@ private class Config : ManualConfig { public Config() { - Add(MemoryDiagnoser.Default); - Add(new InliningDiagnoser()); - Add(new EtwProfiler()); - Add(ThreadingDiagnoser.Default); - Add(ExceptionDiagnoser.Default); + AddDiagnoser(MemoryDiagnoser.Default); + AddDiagnoser(new InliningDiagnoser()); + AddDiagnoser(new EtwProfiler()); + AddDiagnoser(ThreadingDiagnoser.Default); + AddDiagnoser(ExceptionDiagnoser.Default); } } ``` diff --git a/docs/articles/configs/jobs.md b/docs/articles/configs/jobs.md index a6085e0bdb..ad0cf4559c 100644 --- a/docs/articles/configs/jobs.md +++ b/docs/articles/configs/jobs.md @@ -98,20 +98,20 @@ public class MyBenchmarks { public Config() { - Add( - new Job("MySuperJob", RunMode.Dry, EnvMode.RyuJitX64) + AddJob( + new Job("MySuperJob", RunMode.Dry, EnvironmentMode.RyuJitX64) { - Env = { Runtime = Runtime.Core }, + Environment = { Runtime = CoreRuntime.Core90 }, Run = { LaunchCount = 5, IterationTime = TimeInterval.Millisecond * 200 }, - Accuracy = { MaxStdErrRelative = 0.01 } + Accuracy = { MaxRelativeError = 0.01 } }); // The same, using the .With() factory methods: - Add( + AddJob( Job.Dry .WithPlatform(Platform.X64) .WithJit(Jit.RyuJit) - .WithRuntime(Runtime.Core) + .WithRuntime(CoreRuntime.Core90) .WithLaunchCount(5) .WithIterationTime(TimeInterval.Millisecond * 200) .WithMaxRelativeError(0.01) @@ -122,26 +122,26 @@ public class MyBenchmarks } ``` -Basically, it's a good idea to start with predefined values (e.g. `EnvMode.RyuJitX64` and `RunMode.Dry` passed as constructor args) and modify rest of the properties using property setters or with help of object initializer syntax. +Basically, it's a good idea to start with predefined values (e.g. `EnvironmentMode.RyuJitX64` and `RunMode.Dry` passed as constructor args) and modify rest of the properties using property setters or with help of object initializer syntax. Note that the job cannot be modified after it's added into config. Trying to set a value on property of the frozen job will throw an `InvalidOperationException`. Use the `Job.Frozen` property to determine if the code properties can be altered. If you do want to create a new job based on frozen one (all predefined job values are frozen) you can use the `.With()` extension method ```cs - var newJob = Job.Dry.With(Platform.X64); + var newJob = Job.Dry.WithPlatform(Platform.X64); ``` or pass the frozen value as a constructor argument ```c# - var newJob = new Job(Job.Dry) { Env = { Platform = Platform.X64 } }; + var newJob = new Job(Job.Dry) { Environment = { Platform = Platform.X64 } }; ``` or use the `.Apply()` method on unfrozen job ```c# - var newJob = new Job() { Env = { Platform = Platform.X64 } }.Apply(Job.Dry); + var newJob = new Job() { Environment = { Platform = Platform.X64 } }.Apply(Job.Dry); ``` in any case the Id property will not be transfered and you must pass it explicitly (using the .ctor id argument or the `.WithId()` extension method). @@ -152,7 +152,9 @@ You can also add new jobs via attributes. Examples: ```cs [DryJob] -[ClrJob, CoreJob, MonoJob] +[MonoJob] +[SimpleJob(RuntimeMoniker.Net90)] +[SimpleJob(RuntimeMoniker.NetCoreApp31)] [LegacyJitX86Job, LegacyJitX64Job, RyuJitX64Job] [SimpleJob(RunStrategy.ColdStart, launchCount: 1, warmupCount: 5, iterationCount: 5, id: "FastAndDirtyJob")] public class MyBenchmarkClass @@ -212,7 +214,7 @@ public class MySuperJobAttribute : Attribute, IConfigSource { var job = new Job("MySuperJob", RunMode.Core); job.Env.Platform = Platform.X64; - Config = ManualConfig.CreateEmpty().With(job); + Config = ManualConfig.CreateEmpty().AddJob(job); } public IConfig Config { get; } diff --git a/docs/articles/configs/toolchains.md b/docs/articles/configs/toolchains.md index 0bae3ffdff..b47e994a94 100644 --- a/docs/articles/configs/toolchains.md +++ b/docs/articles/configs/toolchains.md @@ -18,14 +18,14 @@ When you run your benchmarks without specifying the toolchain in an explicit way If you want to test multiple frameworks, your project file **MUST target all of them** and you **MUST install the corresponding SDKs**: ```xml -netcoreapp3.0;netcoreapp2.1;net48 +netcoreapp3.1;net8.0;net48 ``` If you run your benchmarks without specifying any custom settings, BenchmarkDotNet is going to run the benchmarks **using the same framework as the host process**: ```cmd -dotnet run -c Release -f netcoreapp2.1 # is going to run the benchmarks using .NET Core 2.1 -dotnet run -c Release -f netcoreapp3.0 # is going to run the benchmarks using .NET Core 3.0 +dotnet run -c Release -f netcoreapp3.1 # is going to run the benchmarks using .NET Core 3.1 +dotnet run -c Release -f net8.0 # is going to run the benchmarks using .NET 8.0 dotnet run -c Release -f net48 # is going to run the benchmarks using .NET 4.8 mono $pathToExe # is going to run the benchmarks using Mono from your PATH ``` @@ -33,8 +33,8 @@ mono $pathToExe # is going to run the benchmarks using Mo To run the benchmarks for multiple runtimes with a single command, you need to specify the target framework moniker names via `--runtimes|-r` console argument: ```cmd -dotnet run -c Release -f netcoreapp2.1 --runtimes netcoreapp2.1 netcoreapp3.0 # is going to run the benchmarks using .NET Core 2.1 and .NET Core 3.0 -dotnet run -c Release -f netcoreapp2.1 --runtimes netcoreapp2.1 net48 # is going to run the benchmarks using .NET Core 2.1 and .NET 4.8 +dotnet run -c Release -f net8.0 --runtimes net8.0 netcoreapp3.1 # is going to run the benchmarks using .NET 8.0 and .NET Core 3.1 +dotnet run -c Release -f net8.0 --runtimes net8.0 net48 # is going to run the benchmarks using .NET 8.0 and .NET 4.8 ``` What is going to happen if you provide multiple Full .NET Framework monikers? Let's say: @@ -67,8 +67,8 @@ namespace BenchmarkDotNet.Samples { [SimpleJob(RuntimeMoniker.Net48)] [SimpleJob(RuntimeMoniker.Mono)] - [SimpleJob(RuntimeMoniker.NetCoreApp21)] - [SimpleJob(RuntimeMoniker.NetCoreApp30)] + [SimpleJob(RuntimeMoniker.NetCoreApp31)] + [SimpleJob(RuntimeMoniker.Net80)] public class TheClassWithBenchmarks ``` @@ -87,10 +87,9 @@ namespace BenchmarkDotNet.Samples static void Main(string[] args) { var config = DefaultConfig.Instance - .With(Job.Default.With(CoreRuntime.Core21)) - .With(Job.Default.With(CoreRuntime.Core30)) - .With(Job.Default.With(ClrRuntime.Net48)) - .With(Job.Default.With(MonoRuntime.Default)); + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80)) + .AddJob(Job.Default.WithRuntime(ClrRuntime.Net48)) + .AddJob(Job.Default.WithRuntime(MonoRuntime.Default)); BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) @@ -115,9 +114,9 @@ public class MyConfig : ManualConfig Add(Job.Default.With( CsProjCoreToolchain.From( new NetCoreAppSettings( - targetFrameworkMoniker: "netcoreapp2.1", - runtimeFrameworkVersion: "2.1.0-preview2-25628-01", - name: ".NET Core 2.1")))); + targetFrameworkMoniker: "net8.0-windows", + runtimeFrameworkVersion: "8.0.101", + name: ".NET 8.0 Windows")))); } } ``` @@ -130,8 +129,8 @@ It's possible to benchmark a private build of .NET Runtime. All you need to do i BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) .Run(args, - DefaultConfig.Instance.With( - Job.ShortRun.With(ClrRuntime.CreateForLocalFullNetFrameworkBuild(version: "4.0")))); + DefaultConfig.Instance.AddJob( + Job.ShortRun.WithRuntime(ClrRuntime.CreateForLocalFullNetFrameworkBuild(version: "4.0")))); ``` This sends the provided version as a `COMPLUS_Version` env var to the benchmarked process. @@ -146,11 +145,11 @@ public class CustomPathsConfig : ManualConfig public CustomPathsConfig() { var dotnetCli32bit = NetCoreAppSettings - .NetCoreApp20 + .NetCoreApp31 .WithCustomDotNetCliPath(@"C:\Program Files (x86)\dotnet\dotnet.exe", "32 bit cli"); var dotnetCli64bit = NetCoreAppSettings - .NetCoreApp20 + .NetCoreApp31 .WithCustomDotNetCliPath(@"C:\Program Files\dotnet\dotnet.exe", "64 bit cli"); AddJob(Job.RyuJitX86.WithToolchain(CsProjCoreToolchain.From(dotnetCli32bit)).WithId("32 bit cli")); @@ -208,7 +207,7 @@ or: ```cs var config = DefaultConfig.Instance - .With(Job.Default.With(NativeAotRuntime.Net70)); // compiles the benchmarks as net7.0 and uses the latest NativeAOT to build a native app + .AddJob(Job.Default.WithRuntime(NativeAotRuntime.Net70)); // compiles the benchmarks as net7.0 and uses the latest NativeAOT to build a native app BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) @@ -231,8 +230,8 @@ If you want to benchmark some particular version of NativeAOT (or from a differe ```cs var config = DefaultConfig.Instance - .With(Job.ShortRun - .With(NativeAotToolchain.CreateBuilder() + .AddJob(Job.ShortRun + .WithToolchain(NativeAotToolchain.CreateBuilder() .UseNuGet( microsoftDotNetILCompilerVersion: "7.0.0-*", // the version goes here nuGetFeedUrl: "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json") // this address might change over time @@ -338,8 +337,8 @@ or explicitly in the code: ```cs var config = DefaultConfig.Instance - .With(Job.ShortRun - .With(NativeAotToolchain.CreateBuilder() + .AddJob(Job.ShortRun + .WithToolchain(NativeAotToolchain.CreateBuilder() .UseLocalBuild(@"C:\Projects\runtime\artifacts\packages\Release\Shipping\") .DisplayName("NativeAOT local build") .TargetFrameworkMoniker("net7.0") diff --git a/docs/articles/features/event-pipe-profiler.md b/docs/articles/features/event-pipe-profiler.md index 3b0ed69bc6..25d1e90386 100644 --- a/docs/articles/features/event-pipe-profiler.md +++ b/docs/articles/features/event-pipe-profiler.md @@ -7,7 +7,7 @@ name: EventPipeProfiler `EventPipeProfiler` is a cross-platform profiler that allows profile .NET code on every platform - Windows, Linux, macOS. Collected data are exported to trace files (`.speedscope.json` and `.nettrace`) which can be analyzed using [SpeedScope](https://www.speedscope.app/), [PerfView](https://github.com/Microsoft/perfview), and [Visual Studio Profiler](https://learn.microsoft.com/visualstudio/profiling/profiling-feature-tour). This new profiler is available from the 0.12.1 version. -![](https://wojciechnagorski.com/images/EventPipeProfiler/SpeedScopeAdvance.png) +![](https://wojciechnagorski.github.io/images/EventPipeProfiler/SpeedScopeAdvance.png) # Configuration diff --git a/docs/articles/features/toc.yml b/docs/articles/features/toc.yml index ac29397743..61e3a2cd54 100644 --- a/docs/articles/features/toc.yml +++ b/docs/articles/features/toc.yml @@ -12,5 +12,7 @@ href: etwprofiler.md - name: EventPipeProfiler href: event-pipe-profiler.md +- name: VSProfiler + href: vsprofiler.md - name: VSTest href: vstest.md \ No newline at end of file diff --git a/docs/articles/features/vsprofiler.md b/docs/articles/features/vsprofiler.md new file mode 100644 index 0000000000..9e8f52712d --- /dev/null +++ b/docs/articles/features/vsprofiler.md @@ -0,0 +1,71 @@ +--- +uid: docs.vsprofiler +name: VS Profiler +--- + +# Running with Visual Studio profiler +Visual Studio supports [profiler integration with BenchmarkDotNet](https://learn.microsoft.com/visualstudio/profiling/profiling-with-benchmark-dotnet) on Windows through its [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package. Once installed, Visual Studio specific diagnosers will capture performance data in runs and automatically open traces if launched through Visual Studio + +![](../../images/vs-profiler-demo.png) + +## How it works + +First, install the [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package in your benchmarking project. Next add one or more of the Visual Studio diagnosers to your benchmark to capture the relevant profiling information while benchmarking. Lastly, run your benchmarks and a diagsession will be generated. If run from Visual Studio the diagsession will automatically be opened. + +## Available Diagnosers + +* `[CPUUsageDiagnoser]` - Enables the [CPU Usage tool](https://learn.microsoft.com/visualstudio/profiling/cpu-usage). +* `[DatabaseDiagnoser]` - Enables the [Database tool](https://learn.microsoft.com/visualstudio/profiling/analyze-database) +* `[DotNetCountersDiagnoser]` - Enables the [.NET Counters tool](https://learn.microsoft.com/visualstudio/profiling/dotnet-counters-tool) +* `[DotNetObjectAllocDiagnoser]` - Enables the [.NET Object Allocation tool](https://learn.microsoft.com/visualstudio/profiling/dotnet-alloc-tool). When using this tool, you must also specify `[DotNetObjectAllocJobConfiguration]` on the benchmark. If this is missing the run will fail and you will receive an error indicating you need to add it. +* `[EventsDiagnoser]` - Enables the [Events tool](https://learn.microsoft.com/visualstudio/profiling/events-viewer) +* `[FileIODiagnoser]` - Enables the [File IO tool](https://learn.microsoft.com/visualstudio/profiling/use-file-io) + +## How to use it? + +After installing the [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package add the following code as a benchmark: + +```cs +using System; +using System.Security.Cryptography; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Running; +using Microsoft.VSDiagnostics; + +namespace MyBenchmarks +{ + [CPUUsageDiagnoser] + public class Md5VsSha256 + { + private const int N = 10000; + private readonly byte[] data; + + private readonly SHA256 sha256 = SHA256.Create(); + private readonly MD5 md5 = MD5.Create(); + + public Md5VsSha256() + { + data = new byte[N]; + new Random(42).NextBytes(data); + } + + [Benchmark] + public byte[] Sha256() => sha256.ComputeHash(data); + + [Benchmark] + public byte[] Md5() => md5.ComputeHash(data); + } + + public class Program + { + public static void Main(string[] args) + { + var summary = BenchmarkRunner.Run(typeof(Program).Assembly); + } + } +} +``` + +In this case we have added the `[CpuUsageDiagnoser]` to capture a CPU sampling trace. From here run the benchmark in Visual Studio (Ctrl+F5), and after the benchmark run the resulting diagsession will be displayed. Double clicking on one of the benchmark rows shown under the Benchmarks tab will filter the time selection to the specific benchmark allowing you to better isolate and investigate. + +![](../../images/vs-profiler-demo.png) \ No newline at end of file diff --git a/docs/articles/guides/console-args.md b/docs/articles/guides/console-args.md index 6fc5ddb245..765ad2e2bb 100644 --- a/docs/articles/guides/console-args.md +++ b/docs/articles/guides/console-args.md @@ -117,28 +117,29 @@ You can also filter the benchmarks by categories: The `--runtimes` or just `-r` allows you to run the benchmarks for selected Runtimes. Available options are: * Clr - BDN will either use Roslyn (if you run it as .NET app) or latest installed .NET SDK to build the benchmarks (if you run it as .NET Core app). -* Core - if you run it as .NET Core app, BDN will use the same target framework moniker, if you run it as .NET app it's going to use netcoreapp2.1. +* Core - if you run it as .NET Core app, BDN will use the same target framework moniker, if you run it as .NET app it's going to use net8.0. * Mono - it's going to use the Mono from `$Path`, you can override it with `--monoPath`. -* net46, net461, net462, net47, net471, net472 - to build and run benchmarks against specific .NET framework version. -* netcoreapp2.0, netcoreapp2.1, netcoreapp2.2, netcoreapp3.0, netcoreapp3.1, net5.0, net6.0, net7.0 - to build and run benchmarks against specific .NET Core version. -* nativeaot5.0, nativeaot6.0, nativeaot7.0 - to build and run benchmarks using NativeAOT. Can be customized with additional options: `--ilcPath`, `--ilCompilerVersion`. +* net46, net461, net462, net47, net471, net472, net48, net481 - to build and run benchmarks against specific .NET Framework version. +* netcoreapp3.1, net5.0, net6.0, net7.0, net8.0 - to build and run benchmarks against specific .NET (Core) version. +* nativeaot5.0, nativeaot6.0, nativeaot7.0, nativeaot8.0 - to build and run benchmarks using NativeAOT. Can be customized with additional options: `--ilcPackages`, `--ilCompilerVersion`. +* mono6.0, mono7.0, mono8.0 - to build and run benchmarks with .Net 6+ using MonoVM. -Example: run the benchmarks for .NET 4.7.2 and .NET Core 2.1: +Example: run the benchmarks for .NET 4.7.2 and .NET 8.0: ```log -dotnet run -c Release -- --runtimes net472 netcoreapp2.1 +dotnet run -c Release -- --runtimes net472 net8.0 ``` -Example: run the benchmarks for .NET Core 3.0 and latest .NET SDK installed on your PC: +Example: run the benchmarks for .NET Core 3.1 and latest .NET SDK installed on your PC: ```log -dotnet run -c Release -f netcoreapp3.0 -- --runtimes clr core +dotnet run -c Release -f netcoreapp3.1 -- --runtimes clr core ``` -But same command executed with `-f netcoreapp2.0` is going to run the benchmarks for .NET Core 2.0: +But same command executed with `-f net6.0` is going to run the benchmarks for .NET 6.0: ```log -dotnet run -c Release -f netcoreapp2.0 -- --runtimes clr core +dotnet run -c Release -f net6.0 -- --runtimes clr core ``` ## Number of invocations and iterations @@ -207,10 +208,10 @@ To perform a Mann–Whitney U Test and display the results in a dedicated column * `--statisticalTest`- Threshold for Mann–Whitney U Test. Examples: 5%, 10ms, 100ns, 1s -Example: run Mann–Whitney U test with relative ratio of 5% for all benchmarks for .NET Core 2.0 (base) vs .NET Core 2.1 (diff). .NET Core 2.0 will be baseline because it was first. +Example: run Mann–Whitney U test with relative ratio of 5% for all benchmarks for .NET 6.0 (base) vs .NET 8.0 (diff). .NET 6.0 will be baseline because it was first. ```log -dotnet run -c Release -- --filter * --runtimes netcoreapp2.0 netcoreapp2.1 --statisticalTest 5% +dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5% ``` ## More @@ -228,7 +229,7 @@ dotnet run -c Release -- --filter * --runtimes netcoreapp2.0 netcoreapp2.1 --sta * `--cli` path to dotnet cli (optional). * `--packages` the directory to restore packages to (optional). * `--coreRun` path(s) to CoreRun (optional). -* `--ilcPath` path to ILCompiler for NativeAOT. +* `--ilcPackages` path to ILCompiler for NativeAOT. * `--info` prints environment configuration including BenchmarkDotNet, OS, CPU and .NET version * `--stopOnFirstError` stop on first error. * `--help` display this help screen. @@ -242,6 +243,7 @@ dotnet run -c Release -- --filter * --runtimes netcoreapp2.0 netcoreapp2.1 --sta * `--platform` the Platform that should be used. If not specified, the host process platform is used (default). AnyCpu/X86/X64/Arm/Arm64/LoongArch64. * `--runOncePerIteration` run the benchmark exactly once per iteration. * `--buildTimeout` build timeout in seconds. +* `--wakeLock` prevents the system from entering sleep or turning off the display. None/System/Display. * `--wasmEngine` full path to a java script engine used to run the benchmarks, used by Wasm toolchain. * `--wasmMainJS` path to the test-main.js file used by Wasm toolchain. Mandatory when using \"--runtimes wasm\" * `--expose_wasm` arguments for the JavaScript engine used by Wasm toolchain. diff --git a/docs/articles/guides/good-practices.md b/docs/articles/guides/good-practices.md index 854ead5679..da4329de2b 100644 --- a/docs/articles/guides/good-practices.md +++ b/docs/articles/guides/good-practices.md @@ -4,8 +4,8 @@ Never use the Debug build for benchmarking. *Never*. The debug version of the target method can run 10–100 times slower. The release mode means that you should have `true` in your csproj file -or use [/optimize](https://learn.microsoft.com/dotnet/csharp/language-reference/compiler-options/) for `csc`. Also your never -should use an attached debugger (e.g. Visual Studio or WinDbg) during the benchmarking. The best way is +or use [/optimize](https://learn.microsoft.com/dotnet/csharp/language-reference/compiler-options/) for `csc`. Also, never +use an attached debugger (e.g. Visual Studio or WinDbg) during the benchmarking. The best way is build our benchmark in the Release mode and run it from the command line. ## Try different environments diff --git a/docs/articles/license.md b/docs/articles/license.md index 039248a340..6272f97306 100644 --- a/docs/articles/license.md +++ b/docs/articles/license.md @@ -1,6 +1,6 @@ ### The MIT License -Copyright (c) 2013–2024 .NET Foundation and contributors +Copyright (c) 2013–2025 .NET Foundation and contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/docs/articles/samples/IntroArgumentsSource.md b/docs/articles/samples/IntroArgumentsSource.md index 9a0bff5cf4..a5b3692870 100644 --- a/docs/articles/samples/IntroArgumentsSource.md +++ b/docs/articles/samples/IntroArgumentsSource.md @@ -11,7 +11,7 @@ You can mark one or several fields or properties in your class by the [`[ArgumentsSource]`](xref:BenchmarkDotNet.Attributes.ArgumentsSourceAttribute) attribute. In this attribute, you have to specify the name of public method/property which is going to provide the values (something that implements `IEnumerable`). - The source must be within benchmarked type! +The source may be instance or static. If the source is not in the same type as the benchmark, the type containing the source must be specified in the attribute constructor. ### Source code diff --git a/docs/articles/samples/IntroNuGet.md b/docs/articles/samples/IntroNuGet.md index 53d9ac54f8..1cc9af84a2 100644 --- a/docs/articles/samples/IntroNuGet.md +++ b/docs/articles/samples/IntroNuGet.md @@ -1,10 +1,10 @@ ---- +--- uid: BenchmarkDotNet.Samples.IntroNuGet --- ## Sample: IntroNuGet -You can set specific versions of NuGet dependencies for each job. +You can set specific versions of NuGet dependencies for each job using MsBuild properties in your csproj. It allows comparing different versions of the same package (if there are no breaking changes in API). ### Source code @@ -13,14 +13,11 @@ It allows comparing different versions of the same package (if there are no brea ### Output -| Method | Job | NuGetReferences | Mean | Error | StdDev | -|------------------------- |------- |----------------------- |---------:|----------:|----------:| -| SerializeAnonymousObject | 10.0.1 | Newtonsoft.Json 10.0.1 | 2.926 us | 0.0795 us | 0.0283 us | -| SerializeAnonymousObject | 10.0.2 | Newtonsoft.Json 10.0.2 | 2.877 us | 0.5928 us | 0.2114 us | -| SerializeAnonymousObject | 10.0.3 | Newtonsoft.Json 10.0.3 | 2.706 us | 0.1251 us | 0.0446 us | -| SerializeAnonymousObject | 11.0.1 | Newtonsoft.Json 11.0.1 | 2.778 us | 0.5037 us | 0.1796 us | -| SerializeAnonymousObject | 11.0.2 | Newtonsoft.Json 11.0.2 | 2.644 us | 0.0609 us | 0.0217 us | -| SerializeAnonymousObject | 9.0.1 | Newtonsoft.Json 9.0.1 | 2.722 us | 0.3552 us | 0.1267 us | +| Method | Job | Arguments | Mean | Error | StdDev | +|-------------------------- |------- |-------------------- |---------:|----------:|----------:| +| ToImmutableArrayBenchmark | v9.0.0 | /p:SciVersion=9.0.0 | 1.173 μs | 0.0057 μs | 0.0086 μs | +| ToImmutableArrayBenchmark | v9.0.3 | /p:SciVersion=9.0.3 | 1.173 μs | 0.0038 μs | 0.0058 μs | +| ToImmutableArrayBenchmark | v9.0.5 | /p:SciVersion=9.0.5 | 1.172 μs | 0.0107 μs | 0.0157 μs | ### Links diff --git a/docs/articles/samples/IntroParamsSource.md b/docs/articles/samples/IntroParamsSource.md index 5164d2514a..e631fbf1db 100644 --- a/docs/articles/samples/IntroParamsSource.md +++ b/docs/articles/samples/IntroParamsSource.md @@ -10,7 +10,7 @@ You can mark one or several fields or properties in your class by the [`[Params]`](xref:BenchmarkDotNet.Attributes.ParamsAttribute) attribute. In this attribute, you have to specify the name of public method/property which is going to provide the values (something that implements `IEnumerable`). -The source must be within benchmarked type! +The source may be instance or static. If the source is not in the same type as the benchmark, the type containing the source must be specified in the attribute constructor. ### Source code diff --git a/docs/articles/samples/IntroVisualStudioProfiler.md b/docs/articles/samples/IntroVisualStudioProfiler.md new file mode 100644 index 0000000000..8d35498f21 --- /dev/null +++ b/docs/articles/samples/IntroVisualStudioProfiler.md @@ -0,0 +1,23 @@ +--- +uid: BenchmarkDotNet.Samples.IntroVisualStudioProfiler +--- + +## Sample: Visual Studio Profiler + +Using the [Microsoft.VisualStudio.BenchmarkDotNetDiagnosers](https://www.nuget.org/packages/Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers) NuGet package you can capture performance profiles of your benchmarks that can be opened in Visual Studio. + +### Source code + +[!code-csharp[IntroVisualStudioDiagnoser.cs](../../../samples/BenchmarkDotNet.Samples/IntroVisualStudioDiagnoser.cs)] + +### Output +The output will contain a path to the collected diagsession and automatically open in Visual Studio when launched from it. + +```markdown +// * Diagnostic Output - VSDiagnosticsDiagnoser * +Collection result moved to 'C:\Work\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\net8.0\BenchmarkDotNet.Artifacts\BenchmarkDotNet_IntroVisualStudioProfiler_20241205_192056.diagsession'. +Session : {d54ebddb-2d6d-404f-b1da-10acbc89635f} + Stopped +Exported diagsession file: C:\Work\BenchmarkDotNet\samples\BenchmarkDotNet.Samples\bin\Release\net8.0\BenchmarkDotNet.Artifacts\BenchmarkDotNet_IntroVisualStudioProfiler_20241205_192056.diagsession. +Opening diagsession in VisualStudio: 15296 +``` \ No newline at end of file diff --git a/docs/articles/samples/IntroWakeLock.md b/docs/articles/samples/IntroWakeLock.md new file mode 100644 index 0000000000..333b3e4345 --- /dev/null +++ b/docs/articles/samples/IntroWakeLock.md @@ -0,0 +1,32 @@ +--- +uid: BenchmarkDotNet.Samples.IntroWakeLock +--- + +## Sample: IntroWakeLock + +Running benchmarks may sometimes take enough time such that the system enters sleep or turns off the display. + +Using a WakeLock prevents the Windows system doing so. + +### Source code + +[!code-csharp[IntroWakeLock.cs](../../../samples/BenchmarkDotNet.Samples/IntroWakeLock.cs)] + +### Command line + +``` +--wakeLock None +``` +``` +--wakeLock System +``` +``` +--wakeLock Display +``` + +### Links + +* @BenchmarkDotNet.Attributes.WakeLockAttribute +* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroWakeLock + +--- diff --git a/docs/articles/samples/toc.yml b/docs/articles/samples/toc.yml index 7e4c7671e1..8dab3b5c6b 100644 --- a/docs/articles/samples/toc.yml +++ b/docs/articles/samples/toc.yml @@ -124,6 +124,10 @@ href: IntroTagColumn.md - name: IntroTailcall href: IntroTailcall.md +- name: IntroVisualStudioProfiler + href: IntroVisualStudioProfiler.md +- name: IntroWakeLock + href: IntroWakeLock.md - name: IntroWasm href: IntroWasm.md - name: IntroUnicode diff --git a/docs/changelog/.gitignore b/docs/changelog/.gitignore deleted file mode 100644 index 9f626be1d8..0000000000 --- a/docs/changelog/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.md -*.yml \ No newline at end of file diff --git a/docs/images/vs-profiler-demo.png b/docs/images/vs-profiler-demo.png new file mode 100644 index 0000000000..928f012df1 Binary files /dev/null and b/docs/images/vs-profiler-demo.png differ diff --git a/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj b/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj index 2ae53f94d8..6359340234 100644 --- a/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj +++ b/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj @@ -7,13 +7,15 @@ Exe net462;net8.0 false + + false - + @@ -21,8 +23,8 @@ - - - + + + diff --git a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj index 36c3f60b32..948849aa05 100644 --- a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj +++ b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj @@ -13,16 +13,25 @@ $(NoWarn);CA1018;CA5351;CA1825 false + + false - + + + + 9.0.0 + + + - - + - + + + diff --git a/samples/BenchmarkDotNet.Samples/IntroArgumentsSource.cs b/samples/BenchmarkDotNet.Samples/IntroArgumentsSource.cs index 33a8d90c0c..493422e51f 100644 --- a/samples/BenchmarkDotNet.Samples/IntroArgumentsSource.cs +++ b/samples/BenchmarkDotNet.Samples/IntroArgumentsSource.cs @@ -20,10 +20,13 @@ public class IntroArgumentsSource } [Benchmark] - [ArgumentsSource(nameof(TimeSpans))] + [ArgumentsSource(typeof(BenchmarkArguments), nameof(BenchmarkArguments.TimeSpans))] // when the arguments come from a different type, specify that type here public void SingleArgument(TimeSpan time) => Thread.Sleep(time); + } - public IEnumerable TimeSpans() // for single argument it's an IEnumerable of objects (object) + public static class BenchmarkArguments + { + public static IEnumerable TimeSpans() // for single argument it's an IEnumerable of objects (object) { yield return TimeSpan.FromMilliseconds(10); yield return TimeSpan.FromMilliseconds(100); diff --git a/samples/BenchmarkDotNet.Samples/IntroDotMemoryDiagnoser.cs b/samples/BenchmarkDotNet.Samples/IntroDotMemoryDiagnoser.cs index 846e2178f4..894bfc6c34 100644 --- a/samples/BenchmarkDotNet.Samples/IntroDotMemoryDiagnoser.cs +++ b/samples/BenchmarkDotNet.Samples/IntroDotMemoryDiagnoser.cs @@ -4,16 +4,10 @@ namespace BenchmarkDotNet.Samples { - // Enables dotMemory profiling for all jobs + // Profile benchmarks via dotMemory SelfApi profiling for all jobs [DotMemoryDiagnoser] - // Adds the default "external-process" job - // Profiling is performed using dotMemory Command-Line Profiler - // See: https://www.jetbrains.com/help/dotmemory/Working_with_dotMemory_Command-Line_Profiler.html - [SimpleJob] - // Adds an "in-process" job - // Profiling is performed using dotMemory SelfApi - // NuGet reference: https://www.nuget.org/packages/JetBrains.Profiler.SelfApi - [InProcess] + [SimpleJob] // external-process execution + [InProcess] // in-process execution public class IntroDotMemoryDiagnoser { [Params(1024)] diff --git a/samples/BenchmarkDotNet.Samples/IntroDotTraceDiagnoser.cs b/samples/BenchmarkDotNet.Samples/IntroDotTraceDiagnoser.cs index 351207c78b..047e6ee059 100644 --- a/samples/BenchmarkDotNet.Samples/IntroDotTraceDiagnoser.cs +++ b/samples/BenchmarkDotNet.Samples/IntroDotTraceDiagnoser.cs @@ -3,16 +3,11 @@ namespace BenchmarkDotNet.Samples { - // Enables dotTrace profiling for all jobs + // Profile benchmarks via dotTrace SelfApi profiling for all jobs + // See: https://www.nuget.org/packages/JetBrains.Profiler.SelfApi [DotTraceDiagnoser] - // Adds the default "external-process" job - // Profiling is performed using dotTrace command-line Tools - // See: https://www.jetbrains.com/help/profiler/Performance_Profiling__Profiling_Using_the_Command_Line.html - [SimpleJob] - // Adds an "in-process" job - // Profiling is performed using dotTrace SelfApi - // NuGet reference: https://www.nuget.org/packages/JetBrains.Profiler.SelfApi - [InProcess] + [SimpleJob] // external-process execution + [InProcess] // in-process execution public class IntroDotTraceDiagnoser { [Benchmark] diff --git a/samples/BenchmarkDotNet.Samples/IntroEnvVars.cs b/samples/BenchmarkDotNet.Samples/IntroEnvVars.cs index 59bd2db82f..66f5197119 100644 --- a/samples/BenchmarkDotNet.Samples/IntroEnvVars.cs +++ b/samples/BenchmarkDotNet.Samples/IntroEnvVars.cs @@ -10,13 +10,14 @@ public class IntroEnvVars { private class ConfigWithCustomEnvVars : ManualConfig { - private const string JitNoInline = "COMPlus_JitNoInline"; - public ConfigWithCustomEnvVars() { - AddJob(Job.Default.WithRuntime(CoreRuntime.Core21).WithId("Inlining enabled")); - AddJob(Job.Default.WithRuntime(CoreRuntime.Core21) - .WithEnvironmentVariables(new EnvironmentVariable(JitNoInline, "1")) + AddJob(Job.Default.WithRuntime(CoreRuntime.Core80).WithId("Inlining enabled")); + AddJob(Job.Default.WithRuntime(CoreRuntime.Core80) + .WithEnvironmentVariables([ + new EnvironmentVariable("DOTNET_JitNoInline", "1"), + new EnvironmentVariable("COMPlus_JitNoInline", "1") + ]) .WithId("Inlining disabled")); } } @@ -27,4 +28,4 @@ public void Foo() // Benchmark body } } -} \ No newline at end of file +} diff --git a/samples/BenchmarkDotNet.Samples/IntroFluentConfigBuilder.cs b/samples/BenchmarkDotNet.Samples/IntroFluentConfigBuilder.cs index 94c34c1cbd..4ac29919ee 100644 --- a/samples/BenchmarkDotNet.Samples/IntroFluentConfigBuilder.cs +++ b/samples/BenchmarkDotNet.Samples/IntroFluentConfigBuilder.cs @@ -38,7 +38,7 @@ public static void Run() .Run( DefaultConfig.Instance .AddJob(Job.Default.WithRuntime(ClrRuntime.Net462)) - .AddJob(Job.Default.WithRuntime(CoreRuntime.Core21)) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80)) .AddValidator(ExecutionValidator.FailOnError)); } } diff --git a/samples/BenchmarkDotNet.Samples/IntroNativeMemory.cs b/samples/BenchmarkDotNet.Samples/IntroNativeMemory.cs index f968696b43..75f636e87e 100644 --- a/samples/BenchmarkDotNet.Samples/IntroNativeMemory.cs +++ b/samples/BenchmarkDotNet.Samples/IntroNativeMemory.cs @@ -3,6 +3,7 @@ using System.Runtime.InteropServices; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Diagnostics.Windows.Configs; +using BenchmarkDotNet.Filters; namespace BenchmarkDotNet.Samples { @@ -11,7 +12,8 @@ namespace BenchmarkDotNet.Samples [MemoryDiagnoser] public class IntroNativeMemory { - [Benchmark] +#pragma warning disable CA1416 + [Benchmark, WindowsOnly] public void BitmapWithLeaks() { var flag = new Bitmap(200, 100); @@ -20,7 +22,7 @@ public void BitmapWithLeaks() graphics.DrawLine(blackPen, 100, 100, 500, 100); } - [Benchmark] + [Benchmark, WindowsOnly] public void Bitmap() { using (var flag = new Bitmap(200, 100)) @@ -34,6 +36,7 @@ public void Bitmap() } } } +#pragma warning restore CA1416 private const int Size = 20; // Greater value could cause System.OutOfMemoryException for test with memory leaks. private int ArraySize = Size * Marshal.SizeOf(typeof(int)); @@ -52,5 +55,13 @@ public unsafe void AllocHGlobalWithLeaks() IntPtr unmanagedHandle = Marshal.AllocHGlobal(ArraySize); Span unmanaged = new Span(unmanagedHandle.ToPointer(), ArraySize); } + + private class WindowsOnlyAttribute : FilterConfigBaseAttribute + { + public WindowsOnlyAttribute() + : base(new SimpleFilter(_ => RuntimeInformation.IsOSPlatform(OSPlatform.Windows))) + { + } + } } } diff --git a/samples/BenchmarkDotNet.Samples/IntroNuGet.cs b/samples/BenchmarkDotNet.Samples/IntroNuGet.cs index 2eb31cd89a..850a9ceebf 100644 --- a/samples/BenchmarkDotNet.Samples/IntroNuGet.cs +++ b/samples/BenchmarkDotNet.Samples/IntroNuGet.cs @@ -1,8 +1,9 @@ using System; +using System.Collections.Immutable; +using System.Linq; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Jobs; -using Newtonsoft.Json; namespace BenchmarkDotNet.Samples { @@ -10,33 +11,49 @@ namespace BenchmarkDotNet.Samples /// Benchmarks between various versions of a NuGet package /// /// - /// Only supported with the CsProjCoreToolchain toolchain + /// Only supported with CsProj toolchains. /// [Config(typeof(Config))] public class IntroNuGet { - // Specify jobs with different versions of the same NuGet package to benchmark. - // The NuGet versions referenced on these jobs must be greater or equal to the - // same NuGet version referenced in this benchmark project. - // Example: This benchmark project references Newtonsoft.Json 9.0.1 + // Setup your csproj like this: + /* + + + 9.0.0 + + + + + */ + // All versions of the package must be source-compatible with your benchmark code. private class Config : ManualConfig { public Config() { - var baseJob = Job.MediumRun; + string[] targetVersions = [ + "9.0.0", + "9.0.3", + "9.0.5", + ]; - AddJob(baseJob.WithNuGet("Newtonsoft.Json", "11.0.2").WithId("11.0.2")); - AddJob(baseJob.WithNuGet("Newtonsoft.Json", "11.0.1").WithId("11.0.1")); - AddJob(baseJob.WithNuGet("Newtonsoft.Json", "10.0.3").WithId("10.0.3")); - AddJob(baseJob.WithNuGet("Newtonsoft.Json", "10.0.2").WithId("10.0.2")); - AddJob(baseJob.WithNuGet("Newtonsoft.Json", "10.0.1").WithId("10.0.1")); - AddJob(baseJob.WithNuGet("Newtonsoft.Json", "9.0.1").WithId("9.0.1")); + foreach (var version in targetVersions) + { + AddJob(Job.MediumRun + .WithMsBuildArguments($"/p:SciVersion={version}") + .WithId($"v{version}") + ); + } } } + private static readonly Random rand = new Random(Seed: 0); + private static readonly double[] values = Enumerable.Range(1, 10_000).Select(x => rand.NextDouble()).ToArray(); + [Benchmark] - public void SerializeAnonymousObject() - => JsonConvert.SerializeObject( - new { hello = "world", price = 1.99, now = DateTime.UtcNow }); + public void ToImmutableArrayBenchmark() + { + var results = values.ToImmutableArray(); + } } } \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroParamsSource.cs b/samples/BenchmarkDotNet.Samples/IntroParamsSource.cs index cfd00d50f4..8257439ddf 100644 --- a/samples/BenchmarkDotNet.Samples/IntroParamsSource.cs +++ b/samples/BenchmarkDotNet.Samples/IntroParamsSource.cs @@ -20,7 +20,16 @@ public class IntroParamsSource // public static method public static IEnumerable ValuesForB() => new[] { 10, 20 }; + // public field getting its params from a method in another type + [ParamsSource(typeof(ParamsValues), nameof(ParamsValues.ValuesForC))] + public int C; + [Benchmark] - public void Benchmark() => Thread.Sleep(A + B + 5); + public void Benchmark() => Thread.Sleep(A + B + C + 5); + } + + public static class ParamsValues + { + public static IEnumerable ValuesForC() => new[] { 1000, 2000 }; } } \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroSmokeEmptyBasic.cs b/samples/BenchmarkDotNet.Samples/IntroSmokeEmptyBasic.cs new file mode 100644 index 0000000000..39783dc74a --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroSmokeEmptyBasic.cs @@ -0,0 +1,82 @@ +using BenchmarkDotNet.Attributes; + +namespace BenchmarkDotNet.Samples; + +[DisassemblyDiagnoser] +public class IntroSmokeEmptyBasic +{ + [Benchmark] public void Void1() {} + [Benchmark] public void Void2() {} + [Benchmark] public void Void3() {} + [Benchmark] public void Void4() {} + + [Benchmark] public byte Byte1() => 0; + [Benchmark] public byte Byte2() => 0; + [Benchmark] public byte Byte3() => 0; + [Benchmark] public byte Byte4() => 0; + + [Benchmark] public sbyte Sbyte1() => 0; + [Benchmark] public sbyte Sbyte2() => 0; + [Benchmark] public sbyte Sbyte3() => 0; + [Benchmark] public sbyte Sbyte4() => 0; + + [Benchmark] public short Short1() => 0; + [Benchmark] public short Short2() => 0; + [Benchmark] public short Short3() => 0; + [Benchmark] public short Short4() => 0; + + [Benchmark] public ushort Ushort1() => 0; + [Benchmark] public ushort Ushort2() => 0; + [Benchmark] public ushort Ushort3() => 0; + [Benchmark] public ushort Ushort4() => 0; + + [Benchmark] public int Int1() => 0; + [Benchmark] public int Int2() => 0; + [Benchmark] public int Int3() => 0; + [Benchmark] public int Int4() => 0; + + [Benchmark] public uint Uint1() => 0u; + [Benchmark] public uint Uint2() => 0u; + [Benchmark] public uint Uint3() => 0u; + [Benchmark] public uint Uint4() => 0u; + + [Benchmark] public bool Bool1() => false; + [Benchmark] public bool Bool2() => false; + [Benchmark] public bool Bool3() => false; + [Benchmark] public bool Bool4() => false; + + [Benchmark] public char Char1() => 'a'; + [Benchmark] public char Char2() => 'a'; + [Benchmark] public char Char3() => 'a'; + [Benchmark] public char Char4() => 'a'; + + [Benchmark] public float Float1() => 0f; + [Benchmark] public float Float2() => 0f; + [Benchmark] public float Float3() => 0f; + [Benchmark] public float Float4() => 0f; + + [Benchmark] public double Double1() => 0d; + [Benchmark] public double Double2() => 0d; + [Benchmark] public double Double3() => 0d; + [Benchmark] public double Double4() => 0d; + + [Benchmark] public long Long1() => 0L; + [Benchmark] public long Long2() => 0L; + [Benchmark] public long Long3() => 0L; + [Benchmark] public long Long4() => 0L; + + [Benchmark] public ulong Ulong1() => 0uL; + [Benchmark] public ulong Ulong2() => 0uL; + [Benchmark] public ulong Ulong3() => 0uL; + [Benchmark] public ulong Ulong4() => 0uL; + + [Benchmark] public string String1() => ""; + [Benchmark] public string String2() => ""; + [Benchmark] public string String3() => ""; + [Benchmark] public string String4() => ""; + + [Benchmark] public object? Object1() => null; + [Benchmark] public object? Object2() => null; + [Benchmark] public object? Object3() => null; + [Benchmark] public object? Object4() => null; +} \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroSmokeIncrements.cs b/samples/BenchmarkDotNet.Samples/IntroSmokeIncrements.cs new file mode 100644 index 0000000000..6dfd15433d --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroSmokeIncrements.cs @@ -0,0 +1,138 @@ +using BenchmarkDotNet.Attributes; + +namespace BenchmarkDotNet.Samples; + +public class IntroSmokeIncrements +{ + public int Field; + + [Benchmark] + public void Increment01() + { + Field++; + } + + [Benchmark] + public void Increment02() + { + Field++; + Field++; + } + + [Benchmark] + public void Increment03() + { + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment04() + { + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment05() + { + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment06() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment07() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment08() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment09() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment10() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } + + [Benchmark] + public void Increment20() + { + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + Field++; + } +} \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroSmokeValueTypes.cs b/samples/BenchmarkDotNet.Samples/IntroSmokeValueTypes.cs new file mode 100644 index 0000000000..66bce4d921 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroSmokeValueTypes.cs @@ -0,0 +1,70 @@ +using System; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Environments; + +namespace BenchmarkDotNet.Samples; + +[MemoryDiagnoser, DisassemblyDiagnoser] +public class IntroSmokeValueTypes +{ + [Benchmark] public Jit ReturnEnum() => Jit.RyuJit; + + [Benchmark] public DateTime ReturnDateTime() => new DateTime(); + + [Benchmark] public DateTime? ReturnNullableDateTime() => new DateTime(); + [Benchmark] public int? ReturnNullableInt() => 0; + + public struct StructWithReferencesOnly { public object _ref; } + [Benchmark] public StructWithReferencesOnly ReturnStructWithReferencesOnly() => new StructWithReferencesOnly(); + + public struct EmptyStruct { } + [Benchmark] public EmptyStruct ReturnEmptyStruct() => new EmptyStruct(); + + [Benchmark] public ValueTuple ReturnGenericStructOfValueType() => new ValueTuple(0); + [Benchmark] public ValueTuple ReturnGenericStructOfReferenceType() => new ValueTuple(null); + + [Benchmark] public ValueTask ReturnValueTaskOfValueType() => new ValueTask(0); + [Benchmark] public ValueTask ReturnValueTaskOfReferenceType() => new ValueTask(result: null); + + [Benchmark] public byte ReturnByte() => 0; + public struct Byte1 { public byte _1; } + [Benchmark] public Byte1 ReturnByte1() => new Byte1(); + public struct Byte2 { public byte _1, _2; } + [Benchmark] public Byte2 ReturnByte2() => new Byte2(); + public struct Byte3 { public byte _1, _2, _3; } + [Benchmark] public Byte3 ReturnByte3() => new Byte3(); + public struct Byte4 { public byte _1, _2, _3, _4; } + [Benchmark] public Byte4 ReturnByte4() => new Byte4(); + + [Benchmark] public short ReturnShort() => 0; + public struct Short1 { public short _1; } + [Benchmark] public Short1 ReturnShort1() => new Short1(); + public struct Short2 { public short _1, _2; } + [Benchmark] public Short2 ReturnShort2() => new Short2(); + public struct Short3 { public short _1, _2, _3; } + [Benchmark] public Short3 ReturnShort3() => new Short3(); + public struct Short4 { public short _1, _2, _3, _4; } + [Benchmark] public Short4 ReturnShort4() => new Short4(); + + [Benchmark] public int ReturnInt() => 0; + public struct Int1 { public int _1; } + [Benchmark] public Int1 ReturnInt1() => new Int1(); + public struct Int2 { public int _1, _2; } + [Benchmark] public Int2 ReturnInt2() => new Int2(); + public struct Int3 { public int _1, _2, _3; } + [Benchmark] public Int3 ReturnInt3() => new Int3(); + public struct Int4 { public int _1, _2, _3, _4; } + [Benchmark] public Int4 ReturnInt4() => new Int4(); + + [Benchmark] public long ReturnLong() => 0; + public struct Long1 { public long _1; } + [Benchmark] public Long1 ReturnLong1() => new Long1(); + public struct Long2 { public long _1, _2; } + [Benchmark] public Long2 ReturnLong2() => new Long2(); + public struct Long3 { public long _1, _2, _3; } + [Benchmark] public Long3 ReturnLong3() => new Long3(); + public struct Long4 { public long _1, _2, _3, _4; } + [Benchmark] public Long4 ReturnLong4() => new Long4(); +} +// ReSharper restore InconsistentNaming \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroVisualStudioDiagnoser.cs b/samples/BenchmarkDotNet.Samples/IntroVisualStudioDiagnoser.cs new file mode 100644 index 0000000000..e513f39320 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroVisualStudioDiagnoser.cs @@ -0,0 +1,23 @@ +using System; +using BenchmarkDotNet.Attributes; +using Microsoft.VSDiagnostics; + +namespace BenchmarkDotNet.Samples +{ + // Enables profiling with the CPU Usage tool + // See: https://learn.microsoft.com/visualstudio/profiling/profiling-with-benchmark-dotnet + [CPUUsageDiagnoser] + public class IntroVisualStudioProfiler + { + private readonly Random rand = new Random(42); + + [Benchmark] + public void BurnCPU() + { + for (int i = 0; i < 100000; ++i) + { + rand.Next(1, 100); + } + } + } +} \ No newline at end of file diff --git a/samples/BenchmarkDotNet.Samples/IntroWakeLock.cs b/samples/BenchmarkDotNet.Samples/IntroWakeLock.cs new file mode 100644 index 0000000000..099b347c7b --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroWakeLock.cs @@ -0,0 +1,30 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using System; +using System.Threading; + +// *** Attribute Style applied to Assembly *** +[assembly: WakeLock(WakeLockType.System)] + +namespace BenchmarkDotNet.Samples; + +// *** Attribute Style *** +[WakeLock(WakeLockType.Display)] +public class IntroWakeLock +{ + [Benchmark] + public void LongRunning() => Thread.Sleep(TimeSpan.FromSeconds(10)); +} + +// *** Object Style *** +[Config(typeof(Config))] +public class IntroWakeLockObjectStyle +{ + private class Config : ManualConfig + { + public Config() => WakeLock = WakeLockType.System; + } + + [Benchmark] + public void LongRunning() => Thread.Sleep(TimeSpan.FromSeconds(10)); +} diff --git a/src/BenchmarkDotNet.Annotations/Attributes/ArgumentsSourceAttribute.cs b/src/BenchmarkDotNet.Annotations/Attributes/ArgumentsSourceAttribute.cs index a7d3fe38bb..f4836e329a 100644 --- a/src/BenchmarkDotNet.Annotations/Attributes/ArgumentsSourceAttribute.cs +++ b/src/BenchmarkDotNet.Annotations/Attributes/ArgumentsSourceAttribute.cs @@ -6,7 +6,18 @@ namespace BenchmarkDotNet.Attributes public class ArgumentsSourceAttribute : PriorityAttribute { public string Name { get; } + public Type? Type { get; } - public ArgumentsSourceAttribute(string name) => Name = name; + public ArgumentsSourceAttribute(string name) + { + Name = name; + Type = null; + } + + public ArgumentsSourceAttribute(Type type, string name) + { + Name = name; + Type = type; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet.Annotations/Attributes/ParamsAttribute.cs b/src/BenchmarkDotNet.Annotations/Attributes/ParamsAttribute.cs index 22148ffa64..aa47ebdf96 100644 --- a/src/BenchmarkDotNet.Annotations/Attributes/ParamsAttribute.cs +++ b/src/BenchmarkDotNet.Annotations/Attributes/ParamsAttribute.cs @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Attributes [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)] public class ParamsAttribute : PriorityAttribute { - public object?[] Values { get; } + public object?[] Values { get; protected set; } // CLS-Compliant Code requires a constructor without an array in the argument list public ParamsAttribute() => Values = new object[0]; diff --git a/src/BenchmarkDotNet.Annotations/Attributes/ParamsSourceAttribute.cs b/src/BenchmarkDotNet.Annotations/Attributes/ParamsSourceAttribute.cs index 570be32179..3587907d64 100644 --- a/src/BenchmarkDotNet.Annotations/Attributes/ParamsSourceAttribute.cs +++ b/src/BenchmarkDotNet.Annotations/Attributes/ParamsSourceAttribute.cs @@ -6,7 +6,18 @@ namespace BenchmarkDotNet.Attributes public class ParamsSourceAttribute : PriorityAttribute { public string Name { get; } + public Type? Type { get; } - public ParamsSourceAttribute(string name) => Name = name; + public ParamsSourceAttribute(string name) + { + Name = name; + Type = null; + } + + public ParamsSourceAttribute(Type type, string name) + { + Name = name; + Type = type; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj b/src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj index 2e7361e163..ccc51bcd9a 100644 --- a/src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj +++ b/src/BenchmarkDotNet.Annotations/BenchmarkDotNet.Annotations.csproj @@ -2,7 +2,7 @@ Basic BenchmarkDotNet attributes that can be used to annotate your benchmarks - netstandard1.0;netstandard2.0 + netstandard2.0 $(NoWarn);1701;1702;1705;1591;3005;NU1702;CA1825 BenchmarkDotNet.Annotations BenchmarkDotNet.Annotations diff --git a/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs b/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs index 4fc2db7388..16b242d2dc 100644 --- a/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs +++ b/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs @@ -110,6 +110,11 @@ public enum RuntimeMoniker /// Net90, + /// + /// .NET 10.0 + /// + Net10_0, + /// /// NativeAOT compiled as net6.0 /// @@ -130,6 +135,11 @@ public enum RuntimeMoniker /// NativeAot90, + /// + /// NativeAOT compiled as net10.0 + /// + NativeAot10_0, + /// /// WebAssembly with default .Net version /// @@ -160,6 +170,11 @@ public enum RuntimeMoniker /// WasmNet90, + /// + /// WebAssembly with net10.0 + /// + WasmNet10_0, + /// /// Mono with the Ahead of Time LLVM Compiler backend /// @@ -185,6 +200,11 @@ public enum RuntimeMoniker /// MonoAOTLLVMNet90, + /// + /// Mono with the Ahead of Time LLVM Compiler backend and net10.0 + /// + MonoAOTLLVMNet10_0, + /// /// .NET 6 using MonoVM (not CLR which is the default) /// @@ -204,5 +224,10 @@ public enum RuntimeMoniker /// .NET 9 using MonoVM (not CLR which is the default) /// Mono90, + + /// + /// .NET 10 using MonoVM (not CLR which is the default) + /// + Mono10_0, } } diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/BenchmarkDotNet.Diagnostics.Windows.csproj b/src/BenchmarkDotNet.Diagnostics.Windows/BenchmarkDotNet.Diagnostics.Windows.csproj index 6c9e43a3f4..e2eb3d2b68 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/BenchmarkDotNet.Diagnostics.Windows.csproj +++ b/src/BenchmarkDotNet.Diagnostics.Windows/BenchmarkDotNet.Diagnostics.Windows.csproj @@ -12,6 +12,6 @@ - + diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs index 80423d72cf..4205195dc5 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs @@ -8,6 +8,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; using Microsoft.Diagnostics.Tracing; using Microsoft.Diagnostics.Tracing.Parsers; @@ -15,7 +16,7 @@ namespace BenchmarkDotNet.Diagnostics.Windows { - public abstract class EtwDiagnoser where TStats : new() + public abstract class EtwDiagnoser : DisposeAtProcessTermination where TStats : new() { internal readonly LogCapture Logger = new LogCapture(); protected readonly Dictionary BenchmarkToProcess = new Dictionary(); @@ -39,11 +40,6 @@ protected void Start(DiagnoserActionParameters parameters) BenchmarkToProcess.Add(parameters.BenchmarkCase, parameters.Process.Id); StatsPerProcess.TryAdd(parameters.Process.Id, GetInitializedStats(parameters)); - // Important: Must wire-up clean-up events prior to acquiring IDisposable instance (Session property) - // This is in effect the inverted sequence of actions in the Stop() method. - Console.CancelKeyPress += OnConsoleCancelKeyPress; - AppDomain.CurrentDomain.ProcessExit += OnProcessExit; - Session = CreateSession(parameters.BenchmarkCase); EnableProvider(); @@ -80,11 +76,13 @@ protected virtual void EnableProvider() protected void Stop() { WaitForDelayedEvents(); + Dispose(); + } - Session.Dispose(); - - Console.CancelKeyPress -= OnConsoleCancelKeyPress; - AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; + public override void Dispose() + { + Session?.Dispose(); + base.Dispose(); } private void Clear() @@ -93,10 +91,6 @@ private void Clear() StatsPerProcess.Clear(); } - private void OnConsoleCancelKeyPress(object sender, ConsoleCancelEventArgs e) => Session?.Dispose(); - - private void OnProcessExit(object sender, EventArgs e) => Session?.Dispose(); - private static string GetSessionName(string prefix, BenchmarkCase benchmarkCase, ParameterInstances? parameters = null) { if (parameters != null && parameters.Items.Count > 0) diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs b/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs index 6641dd5a45..57f57d9e1a 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs @@ -120,21 +120,10 @@ private void Start(DiagnoserActionParameters parameters) private void Stop(DiagnoserActionParameters parameters) { WaitForDelayedEvents(); - string userSessionFile; - try - { - kernelSession.Stop(); - heapSession?.Stop(); - userSession.Stop(); - - userSessionFile = userSession.FilePath; - } - finally - { - kernelSession.Dispose(); - heapSession?.Dispose(); - userSession.Dispose(); - } + string userSessionFile = userSession.FilePath; + kernelSession.Dispose(); + heapSession?.Dispose(); + userSession.Dispose(); // Merge the 'primary' etl file X.etl (userSession) with any files that match .clr*.etl .user*.etl. and .kernel.etl. TraceEventSession.MergeInPlace(userSessionFile, TextWriter.Null); diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/HardwareCounters.cs b/src/BenchmarkDotNet.Diagnostics.Windows/HardwareCounters.cs index 8b3c46b4d5..d159e72a65 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/HardwareCounters.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/HardwareCounters.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Toolchains.InProcess.Emit; @@ -31,7 +32,7 @@ private static readonly Dictionary EtwTranslations public static IEnumerable Validate(ValidationParameters validationParameters, bool mandatory) { - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { yield return new ValidationError(true, "Hardware Counters and EtwProfiler are supported only on Windows"); yield break; diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs index b189af0acb..0d06faf75b 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/JitDiagnoser.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Loggers; @@ -33,7 +34,7 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) public IEnumerable Validate(ValidationParameters validationParameters) { - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { yield return new ValidationError(true, $"{GetType().Name} is supported only on Windows"); } diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs index a101888a0e..b5d8a9c359 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs @@ -60,7 +60,7 @@ protected override void AttachToEvents(TraceEventSession session, BenchmarkCase private sealed class MethodsJittedDescriptor : IMetricDescriptor { - internal static readonly MethodsJittedDescriptor Instance = new (); + internal static readonly MethodsJittedDescriptor Instance = new(); public string Id => nameof(MethodsJittedDescriptor); public string DisplayName => "Methods JITted"; @@ -75,7 +75,7 @@ private sealed class MethodsJittedDescriptor : IMetricDescriptor private sealed class MethodsTieredDescriptor : IMetricDescriptor { - internal static readonly MethodsTieredDescriptor Instance = new (); + internal static readonly MethodsTieredDescriptor Instance = new(); public string Id => nameof(MethodsTieredDescriptor); public string DisplayName => "Methods Tiered"; @@ -90,7 +90,7 @@ private sealed class MethodsTieredDescriptor : IMetricDescriptor private sealed class JitAllocatedMemoryDescriptor : IMetricDescriptor { - internal static readonly JitAllocatedMemoryDescriptor Instance = new (); + internal static readonly JitAllocatedMemoryDescriptor Instance = new(); public string Id => nameof(JitAllocatedMemoryDescriptor); public string DisplayName => "JIT allocated memory"; diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet.Diagnostics.Windows/Properties/AssemblyInfo.cs index a710492849..bf31db808e 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Properties/AssemblyInfo.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/Properties/AssemblyInfo.cs @@ -7,8 +7,4 @@ [assembly: CLSCompliant(true)] -#if RELEASE [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] -#else -[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests")] -#endif \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Sessions.cs b/src/BenchmarkDotNet.Diagnostics.Windows/Sessions.cs index cf0b6fa88f..f0f5dcc475 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Sessions.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/Sessions.cs @@ -5,13 +5,13 @@ using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Running; using Microsoft.Diagnostics.Tracing; using Microsoft.Diagnostics.Tracing.Parsers; using Microsoft.Diagnostics.Tracing.Session; -using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Diagnostics.Windows { @@ -90,7 +90,7 @@ internal override Session EnableProviders() } } - internal abstract class Session : IDisposable + internal abstract class Session : DisposeAtProcessTermination { private const int MaxSessionNameLength = 128; @@ -114,27 +114,16 @@ protected Session(string sessionName, DiagnoserActionParameters details, EtwProf BufferSizeMB = config.BufferSizeInMb, CpuSampleIntervalMSec = config.CpuSampleIntervalInMilliseconds, }; - - Console.CancelKeyPress += OnConsoleCancelKeyPress; - AppDomain.CurrentDomain.ProcessExit += OnProcessExit; } - public void Dispose() => TraceEventSession.Dispose(); - - internal void Stop() + public override void Dispose() { - TraceEventSession.Stop(); - - Console.CancelKeyPress -= OnConsoleCancelKeyPress; - AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; + TraceEventSession.Dispose(); + base.Dispose(); } internal abstract Session EnableProviders(); - private void OnConsoleCancelKeyPress(object sender, ConsoleCancelEventArgs e) => Stop(); - - private void OnProcessExit(object sender, EventArgs e) => Stop(); - protected static string GetSessionName(BenchmarkCase benchmarkCase) { string benchmarkName = FullNameProvider.GetBenchmarkName(benchmarkCase); diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs index 4d8b79c3d1..295ea1d46b 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs @@ -38,7 +38,7 @@ public NativeMemoryLogParser(string etlFilePath, BenchmarkCase benchmarkCase, IL this.benchmarkCase = benchmarkCase; this.logger = logger; - moduleName = programName; + moduleName = programName.ToLowerInvariant(); functionNames = new[] { nameof(EngineParameters.WorkloadActionUnroll), diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/BenchmarkDotNet.Diagnostics.dotMemory.csproj b/src/BenchmarkDotNet.Diagnostics.dotMemory/BenchmarkDotNet.Diagnostics.dotMemory.csproj index 414d927c0d..cc209f4258 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotMemory/BenchmarkDotNet.Diagnostics.dotMemory.csproj +++ b/src/BenchmarkDotNet.Diagnostics.dotMemory/BenchmarkDotNet.Diagnostics.dotMemory.csproj @@ -14,7 +14,7 @@ - + diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoser.cs index 24e62feb31..7616ca468c 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoser.cs @@ -1,148 +1,126 @@ using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using BenchmarkDotNet.Analysers; +using System.Reflection; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Validators; -using RunMode = BenchmarkDotNet.Diagnosers.RunMode; +using JetBrains.Profiler.SelfApi; -namespace BenchmarkDotNet.Diagnostics.dotMemory +namespace BenchmarkDotNet.Diagnostics.dotMemory; + +public class DotMemoryDiagnoser(Uri? nugetUrl = null, string? downloadTo = null) : SnapshotProfilerBase { - public class DotMemoryDiagnoser(Uri? nugetUrl = null, string? toolsDownloadFolder = null) : IProfiler + public override string ShortName => "dotMemory"; + + protected override void InitTool(Progress progress) { - private DotMemoryTool? tool; + DotMemory.InitAsync(progress, nugetUrl, NuGetApi.V3, downloadTo).Wait(); + } - public IEnumerable Ids => new[] { "DotMemory" }; - public string ShortName => "dotMemory"; + protected override void AttachToCurrentProcess(string snapshotFile) + { + DotMemory.Attach(new DotMemory.Config().SaveToFile(snapshotFile)); + } - public RunMode GetRunMode(BenchmarkCase benchmarkCase) - { - return IsSupported(benchmarkCase.Job.Environment.GetRuntime().RuntimeMoniker) ? RunMode.ExtraRun : RunMode.None; - } + protected override void AttachToProcessByPid(int pid, string snapshotFile) + { + DotMemory.Attach(new DotMemory.Config().ProfileExternalProcess(pid).SaveToFile(snapshotFile)); + } - private readonly List snapshotFilePaths = new (); + protected override void TakeSnapshot() + { + DotMemory.GetSnapshot(); + } - public void Handle(HostSignal signal, DiagnoserActionParameters parameters) - { - var logger = parameters.Config.GetCompositeLogger(); - var job = parameters.BenchmarkCase.Job; + protected override void Detach() + { + DotMemory.Detach(); + } - var runtimeMoniker = job.Environment.GetRuntime().RuntimeMoniker; - if (!IsSupported(runtimeMoniker)) - { - logger.WriteLineError($"Runtime '{runtimeMoniker}' is not supported by dotMemory"); - return; - } + protected override string CreateSnapshotFilePath(DiagnoserActionParameters parameters) + { + return ArtifactFileNameHelper.GetFilePath(parameters, "snapshots", DateTime.Now, "dmw", ".0000".Length); + } - switch (signal) - { - case HostSignal.BeforeAnythingElse: - if (tool is null) - { - tool = new DotMemoryTool(logger, nugetUrl, downloadTo: toolsDownloadFolder); - tool.Init(); - } - break; - case HostSignal.BeforeActualRun: - if (tool is null) - throw new InvalidOperationException("DotMemory tool is not initialized"); - snapshotFilePaths.Add(tool.Start(parameters)); - break; - case HostSignal.AfterActualRun: - if (tool is null) - throw new InvalidOperationException("DotMemory tool is not initialized"); - tool.Stop(); - tool = null; - break; - } - } + protected override string GetRunnerPath() + { + var consoleRunnerPackageField = typeof(DotMemory).GetField("ConsoleRunnerPackage", BindingFlags.NonPublic | BindingFlags.Static); + if (consoleRunnerPackageField == null) + throw new InvalidOperationException("Field 'ConsoleRunnerPackage' not found."); - public IEnumerable Exporters => Enumerable.Empty(); - public IEnumerable Analysers => Enumerable.Empty(); + object? consoleRunnerPackage = consoleRunnerPackageField.GetValue(null); + if (consoleRunnerPackage == null) + throw new InvalidOperationException("Unable to get value of 'ConsoleRunnerPackage'."); - public IEnumerable Validate(ValidationParameters validationParameters) - { - var runtimeMonikers = validationParameters.Benchmarks.Select(b => b.Job.Environment.GetRuntime().RuntimeMoniker).Distinct(); - foreach (var runtimeMoniker in runtimeMonikers) - { - if (!IsSupported(runtimeMoniker)) - yield return new ValidationError(true, $"Runtime '{runtimeMoniker}' is not supported by dotMemory"); - } - } + var consoleRunnerPackageType = consoleRunnerPackage.GetType(); + var getRunnerPathMethod = consoleRunnerPackageType.GetMethod("GetRunnerPath"); + if (getRunnerPathMethod == null) + throw new InvalidOperationException("Method 'GetRunnerPath' not found."); - internal static bool IsSupported(RuntimeMoniker runtimeMoniker) - { - switch (runtimeMoniker) - { - case RuntimeMoniker.HostProcess: - case RuntimeMoniker.Net461: - case RuntimeMoniker.Net462: - case RuntimeMoniker.Net47: - case RuntimeMoniker.Net471: - case RuntimeMoniker.Net472: - case RuntimeMoniker.Net48: - case RuntimeMoniker.Net481: - case RuntimeMoniker.Net50: - case RuntimeMoniker.Net60: - case RuntimeMoniker.Net70: - case RuntimeMoniker.Net80: - case RuntimeMoniker.Net90: - return true; - case RuntimeMoniker.NotRecognized: - case RuntimeMoniker.Mono: - case RuntimeMoniker.NativeAot60: - case RuntimeMoniker.NativeAot70: - case RuntimeMoniker.NativeAot80: - case RuntimeMoniker.NativeAot90: - case RuntimeMoniker.Wasm: - case RuntimeMoniker.WasmNet50: - case RuntimeMoniker.WasmNet60: - case RuntimeMoniker.WasmNet70: - case RuntimeMoniker.WasmNet80: - case RuntimeMoniker.WasmNet90: - case RuntimeMoniker.MonoAOTLLVM: - case RuntimeMoniker.MonoAOTLLVMNet60: - case RuntimeMoniker.MonoAOTLLVMNet70: - case RuntimeMoniker.MonoAOTLLVMNet80: - case RuntimeMoniker.MonoAOTLLVMNet90: - case RuntimeMoniker.Mono60: - case RuntimeMoniker.Mono70: - case RuntimeMoniker.Mono80: - case RuntimeMoniker.Mono90: -#pragma warning disable CS0618 // Type or member is obsolete - case RuntimeMoniker.NetCoreApp50: -#pragma warning restore CS0618 // Type or member is obsolete - return false; - case RuntimeMoniker.NetCoreApp20: - case RuntimeMoniker.NetCoreApp21: - case RuntimeMoniker.NetCoreApp22: - return RuntimeInformation.IsWindows(); - case RuntimeMoniker.NetCoreApp30: - case RuntimeMoniker.NetCoreApp31: - return RuntimeInformation.IsWindows() || RuntimeInformation.IsLinux(); - default: - throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, $"Runtime moniker {runtimeMoniker} is not supported"); - } - } + string? runnerPath = getRunnerPathMethod.Invoke(consoleRunnerPackage, null) as string; + if (runnerPath == null) + throw new InvalidOperationException("Unable to invoke 'GetRunnerPath'."); - public IEnumerable ProcessResults(DiagnoserResults results) => ImmutableArray.Empty; + return runnerPath; + } - public void DisplayResults(ILogger logger) + internal override bool IsSupported(RuntimeMoniker runtimeMoniker) + { + switch (runtimeMoniker) { - if (snapshotFilePaths.Any()) - { - logger.WriteLineInfo("The following dotMemory snapshots were generated:"); - foreach (string snapshotFilePath in snapshotFilePaths) - logger.WriteLineInfo($"* {snapshotFilePath}"); - } + case RuntimeMoniker.HostProcess: + case RuntimeMoniker.Net461: + case RuntimeMoniker.Net462: + case RuntimeMoniker.Net47: + case RuntimeMoniker.Net471: + case RuntimeMoniker.Net472: + case RuntimeMoniker.Net48: + case RuntimeMoniker.Net481: + case RuntimeMoniker.Net50: + case RuntimeMoniker.Net60: + case RuntimeMoniker.Net70: + case RuntimeMoniker.Net80: + case RuntimeMoniker.Net90: + case RuntimeMoniker.Net10_0: + return true; + case RuntimeMoniker.NotRecognized: + case RuntimeMoniker.Mono: + case RuntimeMoniker.NativeAot60: + case RuntimeMoniker.NativeAot70: + case RuntimeMoniker.NativeAot80: + case RuntimeMoniker.NativeAot90: + case RuntimeMoniker.NativeAot10_0: + case RuntimeMoniker.Wasm: + case RuntimeMoniker.WasmNet50: + case RuntimeMoniker.WasmNet60: + case RuntimeMoniker.WasmNet70: + case RuntimeMoniker.WasmNet80: + case RuntimeMoniker.WasmNet90: + case RuntimeMoniker.WasmNet10_0: + case RuntimeMoniker.MonoAOTLLVM: + case RuntimeMoniker.MonoAOTLLVMNet60: + case RuntimeMoniker.MonoAOTLLVMNet70: + case RuntimeMoniker.MonoAOTLLVMNet80: + case RuntimeMoniker.MonoAOTLLVMNet90: + case RuntimeMoniker.MonoAOTLLVMNet10_0: + case RuntimeMoniker.Mono60: + case RuntimeMoniker.Mono70: + case RuntimeMoniker.Mono80: + case RuntimeMoniker.Mono90: + case RuntimeMoniker.Mono10_0: +#pragma warning disable CS0618 // Type or member is obsolete + case RuntimeMoniker.NetCoreApp50: +#pragma warning restore CS0618 // Type or member is obsolete + return false; + case RuntimeMoniker.NetCoreApp20: + case RuntimeMoniker.NetCoreApp21: + case RuntimeMoniker.NetCoreApp22: + return OsDetector.IsWindows(); + case RuntimeMoniker.NetCoreApp30: + case RuntimeMoniker.NetCoreApp31: + return OsDetector.IsWindows() || OsDetector.IsLinux(); + default: + throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, $"Runtime moniker {runtimeMoniker} is not supported"); } } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoserAttribute.cs b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoserAttribute.cs index f9a3612471..c0fb55d4f1 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoserAttribute.cs @@ -1,22 +1,22 @@ using System; using BenchmarkDotNet.Configs; -namespace BenchmarkDotNet.Diagnostics.dotMemory +namespace BenchmarkDotNet.Diagnostics.dotMemory; + +[AttributeUsage(AttributeTargets.Class)] +public class DotMemoryDiagnoserAttribute : Attribute, IConfigSource { - [AttributeUsage(AttributeTargets.Class)] - public class DotMemoryDiagnoserAttribute : Attribute, IConfigSource - { - public IConfig Config { get; } + public IConfig Config { get; } - public DotMemoryDiagnoserAttribute() - { - Config = ManualConfig.CreateEmpty().AddDiagnoser(new DotMemoryDiagnoser()); - } + public DotMemoryDiagnoserAttribute() + { + var diagnoser = new DotMemoryDiagnoser(); + Config = ManualConfig.CreateEmpty().AddDiagnoser(diagnoser); + } - public DotMemoryDiagnoserAttribute(string? nugetUrl = null, string? toolsDownloadFolder = null) - { - var nugetUri = nugetUrl == null ? null : new Uri(nugetUrl); - Config = ManualConfig.CreateEmpty().AddDiagnoser(new DotMemoryDiagnoser(nugetUri, toolsDownloadFolder)); - } + public DotMemoryDiagnoserAttribute(Uri? nugetUrl, string? downloadTo = null) + { + var diagnoser = new DotMemoryDiagnoser(nugetUrl, downloadTo); + Config = ManualConfig.CreateEmpty().AddDiagnoser(diagnoser); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryTool.cs b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryTool.cs deleted file mode 100644 index c28e08fdb2..0000000000 --- a/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryTool.cs +++ /dev/null @@ -1,140 +0,0 @@ -using System; -using System.Diagnostics; -using System.IO; -using System.Reflection; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Loggers; -using JetBrains.Profiler.SelfApi; - -namespace BenchmarkDotNet.Diagnostics.dotMemory -{ - internal sealed class DotMemoryTool - { - private readonly ILogger logger; - private readonly Uri? nugetUrl; - private readonly NuGetApi nugetApi; - private readonly string? downloadTo; - - public DotMemoryTool(ILogger logger, Uri? nugetUrl = null, NuGetApi nugetApi = NuGetApi.V3, string? downloadTo = null) - { - this.logger = logger; - this.nugetUrl = nugetUrl; - this.nugetApi = nugetApi; - this.downloadTo = downloadTo; - } - - public void Init() - { - try - { - logger.WriteLineInfo("Ensuring that dotMemory prerequisite is installed..."); - var progress = new Progress(logger, "Installing DotMemory"); - DotMemory.InitAsync(progress, nugetUrl, nugetApi, downloadTo).Wait(); - logger.WriteLineInfo("dotMemory prerequisite is installed"); - logger.WriteLineInfo($"dotMemory runner path: {GetRunnerPath()}"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - } - - public string Start(DiagnoserActionParameters parameters) - { - string snapshotFile = ArtifactFileNameHelper.GetFilePath(parameters, "snapshots", DateTime.Now, "dmw", ".0000".Length); - string? snapshotDirectory = Path.GetDirectoryName(snapshotFile); - logger.WriteLineInfo($"Target snapshot file: {snapshotFile}"); - if (!Directory.Exists(snapshotDirectory) && snapshotDirectory != null) - { - try - { - Directory.CreateDirectory(snapshotDirectory); - } - catch (Exception e) - { - logger.WriteLineError($"Failed to create directory: {snapshotDirectory}"); - logger.WriteLineError(e.ToString()); - } - } - - try - { - logger.WriteLineInfo("Attaching dotMemory to the process..."); - Attach(parameters, snapshotFile); - logger.WriteLineInfo("dotMemory is successfully attached"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - return snapshotFile; - } - - return snapshotFile; - } - - public void Stop() - { - try - { - logger.WriteLineInfo("Taking dotMemory snapshot..."); - Snapshot(); - logger.WriteLineInfo("dotMemory snapshot is successfully taken"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - - try - { - logger.WriteLineInfo("Detaching dotMemory from the process..."); - Detach(); - logger.WriteLineInfo("dotMemory is successfully detached"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - } - - private void Attach(DiagnoserActionParameters parameters, string snapshotFile) - { - var config = new DotMemory.Config(); - - var pid = parameters.Process.Id; - var currentPid = Process.GetCurrentProcess().Id; - if (pid != currentPid) - config = config.ProfileExternalProcess(pid); - - config = config.SaveToFile(snapshotFile); - DotMemory.Attach(config); - } - - private void Snapshot() => DotMemory.GetSnapshot(); - - private void Detach() => DotMemory.Detach(); - - private string GetRunnerPath() - { - var consoleRunnerPackageField = typeof(DotMemory).GetField("ConsoleRunnerPackage", BindingFlags.NonPublic | BindingFlags.Static); - if (consoleRunnerPackageField == null) - throw new InvalidOperationException("Field 'ConsoleRunnerPackage' not found."); - - object? consoleRunnerPackage = consoleRunnerPackageField.GetValue(null); - if (consoleRunnerPackage == null) - throw new InvalidOperationException("Unable to get value of 'ConsoleRunnerPackage'."); - - var consoleRunnerPackageType = consoleRunnerPackage.GetType(); - var getRunnerPathMethod = consoleRunnerPackageType.GetMethod("GetRunnerPath"); - if (getRunnerPathMethod == null) - throw new InvalidOperationException("Method 'GetRunnerPath' not found."); - - string? runnerPath = getRunnerPathMethod.Invoke(consoleRunnerPackage, null) as string; - if (runnerPath == null) - throw new InvalidOperationException("Unable to invoke 'GetRunnerPath'."); - - return runnerPath; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/Progress.cs b/src/BenchmarkDotNet.Diagnostics.dotMemory/Progress.cs deleted file mode 100644 index 738997bb6d..0000000000 --- a/src/BenchmarkDotNet.Diagnostics.dotMemory/Progress.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System; -using System.Diagnostics; -using BenchmarkDotNet.Loggers; - -namespace BenchmarkDotNet.Diagnostics.dotMemory -{ - public class Progress : IProgress - { - private static readonly TimeSpan ReportInterval = TimeSpan.FromSeconds(0.1); - - private readonly ILogger logger; - private readonly string title; - - public Progress(ILogger logger, string title) - { - this.logger = logger; - this.title = title; - } - - private int lastProgress; - private Stopwatch? stopwatch; - - public void Report(double value) - { - int progress = (int)Math.Floor(value); - bool needToReport = stopwatch == null || - (stopwatch != null && stopwatch?.Elapsed > ReportInterval) || - progress == 100; - - if (lastProgress != progress && needToReport) - { - logger.WriteLineInfo($"{title}: {progress}%"); - lastProgress = progress; - stopwatch = Stopwatch.StartNew(); - } - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet.Diagnostics.dotMemory/Properties/AssemblyInfo.cs index 270fdc2c9c..4cc109a0a4 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotMemory/Properties/AssemblyInfo.cs +++ b/src/BenchmarkDotNet.Diagnostics.dotMemory/Properties/AssemblyInfo.cs @@ -4,8 +4,4 @@ [assembly: CLSCompliant(true)] -#if RELEASE [assembly: InternalsVisibleTo("BenchmarkDotNet.Tests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] -#else -[assembly: InternalsVisibleTo("BenchmarkDotNet.Tests")] -#endif \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/BenchmarkDotNet.Diagnostics.dotTrace.csproj b/src/BenchmarkDotNet.Diagnostics.dotTrace/BenchmarkDotNet.Diagnostics.dotTrace.csproj index 4ea24eeb04..8206182d6a 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/BenchmarkDotNet.Diagnostics.dotTrace.csproj +++ b/src/BenchmarkDotNet.Diagnostics.dotTrace/BenchmarkDotNet.Diagnostics.dotTrace.csproj @@ -14,7 +14,7 @@ - + diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs index 2aada424b7..8b5d3a858f 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs @@ -1,142 +1,129 @@ using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using BenchmarkDotNet.Analysers; +using System.Reflection; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains; -using BenchmarkDotNet.Validators; -using RunMode = BenchmarkDotNet.Diagnosers.RunMode; +using JetBrains.Profiler.SelfApi; -namespace BenchmarkDotNet.Diagnostics.dotTrace +namespace BenchmarkDotNet.Diagnostics.dotTrace; + +public class DotTraceDiagnoser(Uri? nugetUrl = null, string? downloadTo = null) : SnapshotProfilerBase { - public class DotTraceDiagnoser(Uri? nugetUrl = null, string? toolsDownloadFolder = null) : IProfiler + public override string ShortName => "dotTrace"; + + protected override void InitTool(Progress progress) { - public IEnumerable Ids => new[] { "DotTrace" }; - public string ShortName => "dotTrace"; + DotTrace.InitAsync(progress, nugetUrl, NuGetApi.V3, downloadTo).Wait(); + } - public RunMode GetRunMode(BenchmarkCase benchmarkCase) - { - return IsSupported(benchmarkCase.Job.Environment.GetRuntime().RuntimeMoniker) ? RunMode.ExtraRun : RunMode.None; - } + protected override void AttachToCurrentProcess(string snapshotFile) + { + DotTrace.Attach(new DotTrace.Config().SaveToFile(snapshotFile)); + DotTrace.StartCollectingData(); + } - private readonly List snapshotFilePaths = new (); + protected override void AttachToProcessByPid(int pid, string snapshotFile) + { + DotTrace.Attach(new DotTrace.Config().ProfileExternalProcess(pid).SaveToFile(snapshotFile)); + DotTrace.StartCollectingData(); + } - public void Handle(HostSignal signal, DiagnoserActionParameters parameters) - { - var job = parameters.BenchmarkCase.Job; - bool isInProcess = job.GetToolchain().IsInProcess; - var logger = parameters.Config.GetCompositeLogger(); - DotTraceToolBase tool = isInProcess - ? new InProcessDotTraceTool(logger, nugetUrl, downloadTo: toolsDownloadFolder) - : new ExternalDotTraceTool(logger, nugetUrl, downloadTo: toolsDownloadFolder); + protected override void TakeSnapshot() + { + DotTrace.StopCollectingData(); + DotTrace.SaveData(); + } - var runtimeMoniker = job.Environment.GetRuntime().RuntimeMoniker; - if (!IsSupported(runtimeMoniker)) - { - logger.WriteLineError($"Runtime '{runtimeMoniker}' is not supported by dotTrace"); - return; - } + protected override void Detach() + { + DotTrace.Detach(); + } - switch (signal) - { - case HostSignal.BeforeAnythingElse: - tool.Init(parameters); - break; - case HostSignal.BeforeActualRun: - snapshotFilePaths.Add(tool.Start(parameters)); - break; - case HostSignal.AfterActualRun: - tool.Stop(parameters); - break; - } - } + protected override string CreateSnapshotFilePath(DiagnoserActionParameters parameters) + { + return ArtifactFileNameHelper.GetFilePath(parameters, "snapshots", DateTime.Now, "dtp", ".0000".Length); + } - public IEnumerable Exporters => Enumerable.Empty(); - public IEnumerable Analysers => Enumerable.Empty(); + protected override string GetRunnerPath() + { + var consoleRunnerPackageField = typeof(DotTrace).GetField("ConsoleRunnerPackage", BindingFlags.NonPublic | BindingFlags.Static); + if (consoleRunnerPackageField == null) + throw new InvalidOperationException("Field 'ConsoleRunnerPackage' not found."); - public IEnumerable Validate(ValidationParameters validationParameters) - { - var runtimeMonikers = validationParameters.Benchmarks.Select(b => b.Job.Environment.GetRuntime().RuntimeMoniker).Distinct(); - foreach (var runtimeMoniker in runtimeMonikers) - { - if (!IsSupported(runtimeMoniker)) - yield return new ValidationError(true, $"Runtime '{runtimeMoniker}' is not supported by dotTrace"); - } - } + object? consoleRunnerPackage = consoleRunnerPackageField.GetValue(null); + if (consoleRunnerPackage == null) + throw new InvalidOperationException("Unable to get value of 'ConsoleRunnerPackage'."); - internal static bool IsSupported(RuntimeMoniker runtimeMoniker) - { - switch (runtimeMoniker) - { - case RuntimeMoniker.HostProcess: - case RuntimeMoniker.Net461: - case RuntimeMoniker.Net462: - case RuntimeMoniker.Net47: - case RuntimeMoniker.Net471: - case RuntimeMoniker.Net472: - case RuntimeMoniker.Net48: - case RuntimeMoniker.Net481: - case RuntimeMoniker.Net50: - case RuntimeMoniker.Net60: - case RuntimeMoniker.Net70: - case RuntimeMoniker.Net80: - case RuntimeMoniker.Net90: - return true; - case RuntimeMoniker.NotRecognized: - case RuntimeMoniker.Mono: - case RuntimeMoniker.NativeAot60: - case RuntimeMoniker.NativeAot70: - case RuntimeMoniker.NativeAot80: - case RuntimeMoniker.NativeAot90: - case RuntimeMoniker.Wasm: - case RuntimeMoniker.WasmNet50: - case RuntimeMoniker.WasmNet60: - case RuntimeMoniker.WasmNet70: - case RuntimeMoniker.WasmNet80: - case RuntimeMoniker.WasmNet90: - case RuntimeMoniker.MonoAOTLLVM: - case RuntimeMoniker.MonoAOTLLVMNet60: - case RuntimeMoniker.MonoAOTLLVMNet70: - case RuntimeMoniker.MonoAOTLLVMNet80: - case RuntimeMoniker.MonoAOTLLVMNet90: - case RuntimeMoniker.Mono60: - case RuntimeMoniker.Mono70: - case RuntimeMoniker.Mono80: - case RuntimeMoniker.Mono90: -#pragma warning disable CS0618 // Type or member is obsolete - case RuntimeMoniker.NetCoreApp50: -#pragma warning restore CS0618 // Type or member is obsolete - return false; - case RuntimeMoniker.NetCoreApp20: - case RuntimeMoniker.NetCoreApp21: - case RuntimeMoniker.NetCoreApp22: - return RuntimeInformation.IsWindows(); - case RuntimeMoniker.NetCoreApp30: - case RuntimeMoniker.NetCoreApp31: - return RuntimeInformation.IsWindows() || RuntimeInformation.IsLinux(); - default: - throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, $"Runtime moniker {runtimeMoniker} is not supported"); - } - } + var consoleRunnerPackageType = consoleRunnerPackage.GetType(); + var getRunnerPathMethod = consoleRunnerPackageType.GetMethod("GetRunnerPath"); + if (getRunnerPathMethod == null) + throw new InvalidOperationException("Method 'GetRunnerPath' not found."); - public IEnumerable ProcessResults(DiagnoserResults results) => ImmutableArray.Empty; + string? runnerPath = getRunnerPathMethod.Invoke(consoleRunnerPackage, null) as string; + if (runnerPath == null) + throw new InvalidOperationException("Unable to invoke 'GetRunnerPath'."); + + return runnerPath; + } - public void DisplayResults(ILogger logger) + internal override bool IsSupported(RuntimeMoniker runtimeMoniker) + { + switch (runtimeMoniker) { - if (snapshotFilePaths.Any()) - { - logger.WriteLineInfo("The following dotTrace snapshots were generated:"); - foreach (string snapshotFilePath in snapshotFilePaths) - logger.WriteLineInfo($"* {snapshotFilePath}"); - } + case RuntimeMoniker.HostProcess: + case RuntimeMoniker.Net461: + case RuntimeMoniker.Net462: + case RuntimeMoniker.Net47: + case RuntimeMoniker.Net471: + case RuntimeMoniker.Net472: + case RuntimeMoniker.Net48: + case RuntimeMoniker.Net481: + case RuntimeMoniker.Net50: + case RuntimeMoniker.Net60: + case RuntimeMoniker.Net70: + case RuntimeMoniker.Net80: + case RuntimeMoniker.Net90: + case RuntimeMoniker.Net10_0: + return true; + case RuntimeMoniker.NotRecognized: + case RuntimeMoniker.Mono: + case RuntimeMoniker.NativeAot60: + case RuntimeMoniker.NativeAot70: + case RuntimeMoniker.NativeAot80: + case RuntimeMoniker.NativeAot90: + case RuntimeMoniker.NativeAot10_0: + case RuntimeMoniker.Wasm: + case RuntimeMoniker.WasmNet50: + case RuntimeMoniker.WasmNet60: + case RuntimeMoniker.WasmNet70: + case RuntimeMoniker.WasmNet80: + case RuntimeMoniker.WasmNet90: + case RuntimeMoniker.WasmNet10_0: + case RuntimeMoniker.MonoAOTLLVM: + case RuntimeMoniker.MonoAOTLLVMNet60: + case RuntimeMoniker.MonoAOTLLVMNet70: + case RuntimeMoniker.MonoAOTLLVMNet80: + case RuntimeMoniker.MonoAOTLLVMNet90: + case RuntimeMoniker.MonoAOTLLVMNet10_0: + case RuntimeMoniker.Mono60: + case RuntimeMoniker.Mono70: + case RuntimeMoniker.Mono80: + case RuntimeMoniker.Mono90: + case RuntimeMoniker.Mono10_0: +#pragma warning disable CS0618 // Type or member is obsolete + case RuntimeMoniker.NetCoreApp50: +#pragma warning restore CS0618 // Type or member is obsolete + return false; + case RuntimeMoniker.NetCoreApp20: + case RuntimeMoniker.NetCoreApp21: + case RuntimeMoniker.NetCoreApp22: + return OsDetector.IsWindows(); + case RuntimeMoniker.NetCoreApp30: + case RuntimeMoniker.NetCoreApp31: + return OsDetector.IsWindows() || OsDetector.IsLinux(); + default: + throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, $"Runtime moniker {runtimeMoniker} is not supported"); } } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoserAttribute.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoserAttribute.cs index 90d4173016..f056a98cbd 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoserAttribute.cs @@ -1,22 +1,22 @@ using System; using BenchmarkDotNet.Configs; -namespace BenchmarkDotNet.Diagnostics.dotTrace +namespace BenchmarkDotNet.Diagnostics.dotTrace; + +[AttributeUsage(AttributeTargets.Class)] +public class DotTraceDiagnoserAttribute : Attribute, IConfigSource { - [AttributeUsage(AttributeTargets.Class)] - public class DotTraceDiagnoserAttribute : Attribute, IConfigSource - { - public IConfig Config { get; } + public IConfig Config { get; } - public DotTraceDiagnoserAttribute() - { - Config = ManualConfig.CreateEmpty().AddDiagnoser(new DotTraceDiagnoser()); - } + public DotTraceDiagnoserAttribute() + { + var diagnoser = new DotTraceDiagnoser(); + Config = ManualConfig.CreateEmpty().AddDiagnoser(diagnoser); + } - public DotTraceDiagnoserAttribute(string? nugetUrl = null, string? toolsDownloadFolder = null) - { - var nugetUri = nugetUrl == null ? null : new Uri(nugetUrl); - Config = ManualConfig.CreateEmpty().AddDiagnoser(new DotTraceDiagnoser(nugetUri, toolsDownloadFolder)); - } + public DotTraceDiagnoserAttribute(Uri? nugetUrl, string? downloadTo = null) + { + var diagnoser = new DotTraceDiagnoser(nugetUrl, downloadTo); + Config = ManualConfig.CreateEmpty().AddDiagnoser(diagnoser); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceToolBase.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceToolBase.cs deleted file mode 100644 index 2fb36e9c66..0000000000 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceToolBase.cs +++ /dev/null @@ -1,145 +0,0 @@ -using System; -using System.IO; -using System.Reflection; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Loggers; -using JetBrains.Profiler.SelfApi; - -namespace BenchmarkDotNet.Diagnostics.dotTrace -{ - internal abstract class DotTraceToolBase - { - private readonly ILogger logger; - private readonly Uri? nugetUrl; - private readonly NuGetApi nugetApi; - private readonly string? downloadTo; - - protected DotTraceToolBase(ILogger logger, Uri? nugetUrl = null, NuGetApi nugetApi = NuGetApi.V3, string? downloadTo = null) - { - this.logger = logger; - this.nugetUrl = nugetUrl; - this.nugetApi = nugetApi; - this.downloadTo = downloadTo; - } - - public void Init(DiagnoserActionParameters parameters) - { - try - { - logger.WriteLineInfo("Ensuring that dotTrace prerequisite is installed..."); - var progress = new Progress(logger, "Installing DotTrace"); - DotTrace.InitAsync(progress, nugetUrl, nugetApi, downloadTo).Wait(); - logger.WriteLineInfo("dotTrace prerequisite is installed"); - logger.WriteLineInfo($"dotTrace runner path: {GetRunnerPath()}"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - } - - protected abstract bool AttachOnly { get; } - protected abstract void Attach(DiagnoserActionParameters parameters, string snapshotFile); - protected abstract void StartCollectingData(); - protected abstract void SaveData(); - protected abstract void Detach(); - - public string Start(DiagnoserActionParameters parameters) - { - string snapshotFile = ArtifactFileNameHelper.GetFilePath(parameters, "snapshots", DateTime.Now, "dtp", ".0000".Length); - string? snapshotDirectory = Path.GetDirectoryName(snapshotFile); - logger.WriteLineInfo($"Target snapshot file: {snapshotFile}"); - if (!Directory.Exists(snapshotDirectory) && snapshotDirectory != null) - { - try - { - Directory.CreateDirectory(snapshotDirectory); - } - catch (Exception e) - { - logger.WriteLineError($"Failed to create directory: {snapshotDirectory}"); - logger.WriteLineError(e.ToString()); - } - } - - try - { - logger.WriteLineInfo("Attaching dotTrace to the process..."); - Attach(parameters, snapshotFile); - logger.WriteLineInfo("dotTrace is successfully attached"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - return snapshotFile; - } - - if (!AttachOnly) - { - try - { - logger.WriteLineInfo("Start collecting data using dataTrace..."); - StartCollectingData(); - logger.WriteLineInfo("Data collecting is successfully started"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - } - - return snapshotFile; - } - - public void Stop(DiagnoserActionParameters parameters) - { - if (!AttachOnly) - { - try - { - logger.WriteLineInfo("Saving dotTrace snapshot..."); - SaveData(); - logger.WriteLineInfo("dotTrace snapshot is successfully saved to the artifact folder"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - - try - { - logger.WriteLineInfo("Detaching dotTrace from the process..."); - Detach(); - logger.WriteLineInfo("dotTrace is successfully detached"); - } - catch (Exception e) - { - logger.WriteLineError(e.ToString()); - } - } - } - - protected string GetRunnerPath() - { - var consoleRunnerPackageField = typeof(DotTrace).GetField("ConsoleRunnerPackage", BindingFlags.NonPublic | BindingFlags.Static); - if (consoleRunnerPackageField == null) - throw new InvalidOperationException("Field 'ConsoleRunnerPackage' not found."); - - object? consoleRunnerPackage = consoleRunnerPackageField.GetValue(null); - if (consoleRunnerPackage == null) - throw new InvalidOperationException("Unable to get value of 'ConsoleRunnerPackage'."); - - var consoleRunnerPackageType = consoleRunnerPackage.GetType(); - var getRunnerPathMethod = consoleRunnerPackageType.GetMethod("GetRunnerPath"); - if (getRunnerPathMethod == null) - throw new InvalidOperationException("Method 'GetRunnerPath' not found."); - - string? runnerPath = getRunnerPathMethod.Invoke(consoleRunnerPackage, null) as string; - if (runnerPath == null) - throw new InvalidOperationException("Unable to invoke 'GetRunnerPath'."); - - return runnerPath; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/ExternalDotTraceTool.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/ExternalDotTraceTool.cs deleted file mode 100644 index c7f1cf18c8..0000000000 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/ExternalDotTraceTool.cs +++ /dev/null @@ -1,84 +0,0 @@ -using System; -using System.Diagnostics; -using System.Threading.Tasks; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Loggers; -using JetBrains.Profiler.SelfApi; -using ILogger = BenchmarkDotNet.Loggers.ILogger; - -namespace BenchmarkDotNet.Diagnostics.dotTrace -{ - internal class ExternalDotTraceTool : DotTraceToolBase - { - private static readonly TimeSpan AttachTimeout = TimeSpan.FromMinutes(5); - - public ExternalDotTraceTool(ILogger logger, Uri? nugetUrl = null, NuGetApi nugetApi = NuGetApi.V3, string? downloadTo = null) : - base(logger, nugetUrl, nugetApi, downloadTo) { } - - protected override bool AttachOnly => true; - - protected override void Attach(DiagnoserActionParameters parameters, string snapshotFile) - { - var logger = parameters.Config.GetCompositeLogger(); - - string runnerPath = GetRunnerPath(); - int pid = parameters.Process.Id; - string arguments = $"attach {pid} --save-to=\"{snapshotFile}\" --service-output=on"; - - logger.WriteLineInfo($"Starting process: '{runnerPath} {arguments}'"); - - var processStartInfo = new ProcessStartInfo - { - FileName = runnerPath, - WorkingDirectory = "", - Arguments = arguments, - UseShellExecute = false, - CreateNoWindow = true, - RedirectStandardOutput = true, - RedirectStandardError = true - }; - - var attachWaitingTask = new TaskCompletionSource(); - var process = new Process { StartInfo = processStartInfo }; - try - { - process.OutputDataReceived += (_, args) => - { - string? content = args.Data; - if (content != null) - { - logger.WriteLineInfo("[dotTrace] " + content); - if (content.Contains("##dotTrace[\"started\"")) - attachWaitingTask.TrySetResult(true); - } - }; - process.ErrorDataReceived += (_, args) => - { - string? content = args.Data; - if (content != null) - logger.WriteLineError("[dotTrace] " + args.Data); - }; - process.Exited += (_, _) => { attachWaitingTask.TrySetResult(false); }; - process.Start(); - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - } - catch (Exception e) - { - attachWaitingTask.TrySetResult(false); - logger.WriteLineError(e.ToString()); - } - - if (!attachWaitingTask.Task.Wait(AttachTimeout)) - throw new Exception($"Failed to attach dotTrace to the target process (timeout: {AttachTimeout.TotalSeconds} sec)"); - if (!attachWaitingTask.Task.Result) - throw new Exception($"Failed to attach dotTrace to the target process (ExitCode={process.ExitCode})"); - } - - protected override void StartCollectingData() { } - - protected override void SaveData() { } - - protected override void Detach() { } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/InProcessDotTraceTool.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/InProcessDotTraceTool.cs deleted file mode 100644 index a02c9c1995..0000000000 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/InProcessDotTraceTool.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Loggers; -using JetBrains.Profiler.SelfApi; - -namespace BenchmarkDotNet.Diagnostics.dotTrace -{ - internal class InProcessDotTraceTool : DotTraceToolBase - { - public InProcessDotTraceTool(ILogger logger, Uri? nugetUrl = null, NuGetApi nugetApi = NuGetApi.V3, string? downloadTo = null) : - base(logger, nugetUrl, nugetApi, downloadTo) { } - - protected override bool AttachOnly => false; - - protected override void Attach(DiagnoserActionParameters parameters, string snapshotFile) - { - var config = new DotTrace.Config(); - config.SaveToFile(snapshotFile); - DotTrace.Attach(config); - } - - protected override void StartCollectingData() => DotTrace.StartCollectingData(); - - protected override void SaveData() => DotTrace.SaveData(); - - protected override void Detach() => DotTrace.Detach(); - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/Progress.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/Progress.cs deleted file mode 100644 index c353939f1f..0000000000 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/Progress.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System; -using System.Diagnostics; -using BenchmarkDotNet.Loggers; - -namespace BenchmarkDotNet.Diagnostics.dotTrace -{ - public class Progress : IProgress - { - private static readonly TimeSpan ReportInterval = TimeSpan.FromSeconds(0.1); - - private readonly ILogger logger; - private readonly string title; - - public Progress(ILogger logger, string title) - { - this.logger = logger; - this.title = title; - } - - private int lastProgress; - private Stopwatch? stopwatch; - - public void Report(double value) - { - int progress = (int)Math.Floor(value); - bool needToReport = stopwatch == null || - (stopwatch != null && stopwatch?.Elapsed > ReportInterval) || - progress == 100; - - if (lastProgress != progress && needToReport) - { - logger.WriteLineInfo($"{title}: {progress}%"); - lastProgress = progress; - stopwatch = Stopwatch.StartNew(); - } - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/Properties/AssemblyInfo.cs index 270fdc2c9c..4cc109a0a4 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/Properties/AssemblyInfo.cs +++ b/src/BenchmarkDotNet.Diagnostics.dotTrace/Properties/AssemblyInfo.cs @@ -4,8 +4,4 @@ [assembly: CLSCompliant(true)] -#if RELEASE [assembly: InternalsVisibleTo("BenchmarkDotNet.Tests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] -#else -[assembly: InternalsVisibleTo("BenchmarkDotNet.Tests")] -#endif \ No newline at end of file diff --git a/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj b/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj index 3162c74e5d..2f15efcc13 100644 --- a/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj +++ b/src/BenchmarkDotNet.Disassembler.x64/BenchmarkDotNet.Disassembler.x64.csproj @@ -15,7 +15,7 @@ BenchmarkDotNet.Disassembler - - + + diff --git a/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj b/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj index 21051d6104..5410f6d77b 100644 --- a/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj +++ b/src/BenchmarkDotNet.Disassembler.x86/BenchmarkDotNet.Disassembler.x86.csproj @@ -21,7 +21,7 @@ - - + + diff --git a/src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj b/src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj index 799a055885..cefa91a7ca 100644 --- a/src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj +++ b/src/BenchmarkDotNet.Exporters.Plotting/BenchmarkDotNet.Exporters.Plotting.csproj @@ -1,19 +1,19 @@ - - - - BenchmarkDotNet plotting export support. - netstandard2.0 - BenchmarkDotNet.Exporters.Plotting - BenchmarkDotNet.Exporters.Plotting - BenchmarkDotNet.Exporters.Plotting - - True - enable - - - - - - - - + + + + BenchmarkDotNet plotting export support. + netstandard2.0 + BenchmarkDotNet.Exporters.Plotting + BenchmarkDotNet.Exporters.Plotting + BenchmarkDotNet.Exporters.Plotting + + True + enable + + + + + + + + diff --git a/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs b/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs index cfb7a735b7..df43599aec 100644 --- a/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs +++ b/src/BenchmarkDotNet.Exporters.Plotting/ScottPlotExporter.cs @@ -1,256 +1,438 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Properties; -using BenchmarkDotNet.Reports; -using ScottPlot; -using ScottPlot.Plottables; - -namespace BenchmarkDotNet.Exporters.Plotting -{ - /// - /// Provides plot exports as .png files. - /// - public class ScottPlotExporter : IExporter - { - /// - /// Default instance of the exporter with default configuration. - /// - public static readonly IExporter Default = new ScottPlotExporter(); - - /// - /// Gets the name of the Exporter type. - /// - public string Name => nameof(ScottPlotExporter); - - /// - /// Initializes a new instance of ScottPlotExporter. - /// - /// The width of all plots in pixels (optional). Defaults to 1920. - /// The height of all plots in pixels (optional). Defaults to 1080. - public ScottPlotExporter(int width = 1920, int height = 1080) - { - this.Width = width; - this.Height = height; - this.IncludeBarPlot = true; - this.RotateLabels = true; - } - - /// - /// Gets or sets the width of all plots in pixels. - /// - public int Width { get; set; } - - /// - /// Gets or sets the height of all plots in pixels. - /// - public int Height { get; set; } - - /// - /// Gets or sets a value indicating whether labels for Plot X-axis should be rotated. - /// This allows for longer labels at the expense of chart height. - /// - public bool RotateLabels { get; set; } - - /// - /// Gets or sets a value indicating whether a bar plot for time-per-op - /// measurement values should be exported. - /// - public bool IncludeBarPlot { get; set; } - - /// - /// Not supported. - /// - /// This parameter is not used. - /// This parameter is not used. - /// - public void ExportToLog(Summary summary, ILogger logger) - { - throw new NotSupportedException(); - } - - /// - /// Exports plots to .png file. - /// - /// The summary to be exported. - /// Logger to output to. - /// The file paths of every plot exported. - public IEnumerable ExportToFiles(Summary summary, ILogger consoleLogger) - { - var title = summary.Title; - var version = BenchmarkDotNetInfo.Instance.BrandTitle; - var annotations = GetAnnotations(version); - - var (timeUnit, timeScale) = GetTimeUnit(summary.Reports.SelectMany(m => m.AllMeasurements)); - - foreach (var benchmark in summary.Reports.GroupBy(r => r.BenchmarkCase.Descriptor.Type.Name)) - { - var benchmarkName = benchmark.Key; - - // Get the measurement nanoseconds per op, divided by time scale, grouped by target and Job [param]. - var timeStats = from report in benchmark - let jobId = report.BenchmarkCase.DisplayInfo.Replace(report.BenchmarkCase.Descriptor.DisplayInfo + ": ", string.Empty) - from measurement in report.AllMeasurements - let measurementValue = measurement.Nanoseconds / measurement.Operations - group measurementValue / timeScale by (Target: report.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo, JobId: jobId) into g - select (g.Key.Target, g.Key.JobId, Mean: g.Average(), StdError: StandardError(g.ToList())); - - if (this.IncludeBarPlot) - { - // -barplot.png - yield return CreateBarPlot( - $"{title} - {benchmarkName}", - Path.Combine(summary.ResultsDirectoryPath, $"{title}-{benchmarkName}-barplot.png"), - $"Time ({timeUnit})", - "Target", - timeStats, - annotations); - } - - /* TODO: Rest of the RPlotExporter plots. - -boxplot.png - --density.png - --facetTimeline.png - --facetTimelineSmooth.png - ---timelineSmooth.png - ---timelineSmooth.png*/ - } - } - - /// - /// Calculate Standard Deviation. - /// - /// Values to calculate from. - /// Standard deviation of values. - private static double StandardError(IReadOnlyList values) - { - double average = values.Average(); - double sumOfSquaresOfDifferences = values.Select(val => (val - average) * (val - average)).Sum(); - double standardDeviation = Math.Sqrt(sumOfSquaresOfDifferences / values.Count); - return standardDeviation / Math.Sqrt(values.Count); - } - - /// - /// Gets the lowest appropriate time scale across all measurements. - /// - /// All measurements - /// A unit and scaling factor to convert from nanoseconds. - private (string Unit, double ScaleFactor) GetTimeUnit(IEnumerable values) - { - var minValue = values.Select(m => m.Nanoseconds / m.Operations).DefaultIfEmpty(0d).Min(); - if (minValue > 1000000000d) - { - return ("sec", 1000000000d); - } - - if (minValue > 1000000d) - { - return ("ms", 1000000d); - } - - if (minValue > 1000d) - { - return ("us", 1000d); - } - - return ("ns", 1d); - } - - private string CreateBarPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable<(string Target, string JobId, double Mean, double StdError)> data, IReadOnlyList annotations) - { - Plot plt = new Plot(); - plt.Title(title, 28); - plt.YLabel(yLabel); - plt.XLabel(xLabel); - - var palette = new ScottPlot.Palettes.Category10(); - - var legendPalette = data.Select(d => d.JobId) - .Distinct() - .Select((jobId, index) => (jobId, index)) - .ToDictionary(t => t.jobId, t => palette.GetColor(t.index)); - - plt.Legend.IsVisible = true; - plt.Legend.Location = Alignment.UpperRight; - var legend = data.Select(d => d.JobId) - .Distinct() - .Select((label, index) => new LegendItem() - { - Label = label, - FillColor = legendPalette[label] - }) - .ToList(); - - plt.Legend.ManualItems.AddRange(legend); - - var jobCount = plt.Legend.ManualItems.Count; - var ticks = data - .Select((d, index) => new Tick(index, d.Target)) - .ToArray(); - plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks); - plt.Axes.Bottom.MajorTickStyle.Length = 0; - - if (this.RotateLabels) - { - plt.Axes.Bottom.TickLabelStyle.Rotation = 45; - plt.Axes.Bottom.TickLabelStyle.Alignment = Alignment.MiddleLeft; - - // determine the width of the largest tick label - float largestLabelWidth = 0; - foreach (Tick tick in ticks) - { - PixelSize size = plt.Axes.Bottom.TickLabelStyle.Measure(tick.Label); - largestLabelWidth = Math.Max(largestLabelWidth, size.Width); - } - - // ensure axis panels do not get smaller than the largest label - plt.Axes.Bottom.MinimumSize = largestLabelWidth; - plt.Axes.Right.MinimumSize = largestLabelWidth; - } - - var bars = data - .Select((d, index) => new Bar() - { - Position = ticks[index].Position, - Value = d.Mean, - Error = d.StdError, - FillColor = legendPalette[d.JobId] - }); - plt.Add.Bars(bars); - - // Tell the plot to autoscale with no padding beneath the bars - plt.Axes.Margins(bottom: 0, right: .2); - - plt.PlottableList.AddRange(annotations); - - plt.SavePng(fileName, this.Width, this.Height); - return Path.GetFullPath(fileName); - } - - /// - /// Provides a list of annotations to put over the data area. - /// - /// The version to be displayed. - /// A list of annotations for every plot. - private IReadOnlyList GetAnnotations(string version) - { - var versionAnnotation = new Annotation() - { - Label = - { - Text = version, - FontSize = 14, - ForeColor = new Color(0, 0, 0, 100) - }, - OffsetY = 10, - OffsetX = 20, - Alignment = Alignment.LowerRight - }; - - - return new[] { versionAnnotation }; - } - } +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Properties; +using BenchmarkDotNet.Reports; +using ScottPlot; +using ScottPlot.Plottables; + +namespace BenchmarkDotNet.Exporters.Plotting +{ + /// + /// Provides plot exports as .png files. + /// + public class ScottPlotExporter : IExporter + { + /// + /// Default instance of the exporter with default configuration. + /// + public static readonly IExporter Default = new ScottPlotExporter(); + + /// + /// Gets the name of the Exporter type. + /// + public string Name => nameof(ScottPlotExporter); + + /// + /// Initializes a new instance of ScottPlotExporter. + /// + /// The width of all plots in pixels (optional). Defaults to 1920. + /// The height of all plots in pixels (optional). Defaults to 1080. + public ScottPlotExporter(int width = 1920, int height = 1080) + { + this.Width = width; + this.Height = height; + this.IncludeBarPlot = true; + this.IncludeBoxPlot = true; + this.RotateLabels = true; + } + + /// + /// Gets or sets the width of all plots in pixels. + /// + public int Width { get; set; } + + /// + /// Gets or sets the height of all plots in pixels. + /// + public int Height { get; set; } + + /// + /// Gets or sets the common font size for ticks, labels etc. (defaults to 14). + /// + public int FontSize { get; set; } = 14; + + /// + /// Gets or sets the font size for the chart title. (defaults to 28). + /// + public int TitleFontSize { get; set; } = 28; + + /// + /// Gets or sets a value indicating whether labels for Plot X-axis should be rotated. + /// This allows for longer labels at the expense of chart height. + /// + public bool RotateLabels { get; set; } + + /// + /// Gets or sets a value indicating whether a bar plot for time-per-op + /// measurement values should be exported. + /// + public bool IncludeBarPlot { get; set; } + + /// + /// Gets or sets a value indicating whether a box plot or whisker plot for time-per-op + /// measurement values should be exported. + /// + public bool IncludeBoxPlot { get; set; } + + /// + /// Not supported. + /// + /// This parameter is not used. + /// This parameter is not used. + /// + public void ExportToLog(Summary summary, ILogger logger) + { + throw new NotSupportedException(); + } + + /// + /// Exports plots to .png file. + /// + /// The summary to be exported. + /// Logger to output to. + /// The file paths of every plot exported. + public IEnumerable ExportToFiles(Summary summary, ILogger consoleLogger) + { + var title = summary.Title; + var version = BenchmarkDotNetInfo.Instance.BrandTitle; + var annotations = GetAnnotations(version); + + var (timeUnit, timeScale) = GetTimeUnit(summary.Reports + .SelectMany(m => m.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Result)))); + + foreach (var benchmark in summary.Reports.GroupBy(r => r.BenchmarkCase.Descriptor.Type.Name)) + { + var benchmarkName = benchmark.Key; + + // Get the measurement nanoseconds per op, divided by time scale, grouped by target and Job [param]. + var timeStats = from report in benchmark + let jobId = report.BenchmarkCase.DisplayInfo.Replace(report.BenchmarkCase.Descriptor.DisplayInfo + ": ", string.Empty) + from measurement in report.AllMeasurements + where measurement.Is(IterationMode.Workload, IterationStage.Result) + let measurementValue = measurement.Nanoseconds / measurement.Operations + group measurementValue / timeScale by (Target: report.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo, JobId: jobId) into g + select new ChartStats(g.Key.Target, g.Key.JobId, g.ToList()); + + if (this.IncludeBarPlot) + { + // -barplot.png + yield return CreateBarPlot( + $"{title} - {benchmarkName}", + Path.Combine(summary.ResultsDirectoryPath, $"{title}-{benchmarkName}-barplot.png"), + $"Time ({timeUnit})", + "Target", + timeStats, + annotations); + } + + if (this.IncludeBoxPlot) + { + // -boxplot.png + yield return CreateBoxPlot( + $"{title} - {benchmarkName}", + Path.Combine(summary.ResultsDirectoryPath, $"{title}-{benchmarkName}-boxplot.png"), + $"Time ({timeUnit})", + "Target", + timeStats, + annotations); + } + + /* TODO: Rest of the RPlotExporter plots. + --density.png + --facetTimeline.png + --facetTimelineSmooth.png + ---timelineSmooth.png + ---timelineSmooth.png*/ + } + } + + /// + /// Calculate Standard Deviation. + /// + /// Values to calculate from. + /// Standard deviation of values. + private static double StandardError(IReadOnlyList values) + { + double average = values.Average(); + double sumOfSquaresOfDifferences = values.Select(val => (val - average) * (val - average)).Sum(); + double standardDeviation = Math.Sqrt(sumOfSquaresOfDifferences / values.Count); + return standardDeviation / Math.Sqrt(values.Count); + } + + /// + /// Gets the lowest appropriate time scale across all measurements. + /// + /// All measurements + /// A unit and scaling factor to convert from nanoseconds. + private (string Unit, double ScaleFactor) GetTimeUnit(IEnumerable values) + { + var minValue = values.Select(m => m.Nanoseconds / m.Operations).DefaultIfEmpty(0d).Min(); + if (minValue > 1000000000d) + { + return ("sec", 1000000000d); + } + + if (minValue > 1000000d) + { + return ("ms", 1000000d); + } + + if (minValue > 1000d) + { + return ("us", 1000d); + } + + return ("ns", 1d); + } + + private string CreateBarPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable data, IReadOnlyList annotations) + { + Plot plt = new Plot(); + plt.Title(title, this.TitleFontSize); + plt.YLabel(yLabel, this.FontSize); + plt.XLabel(xLabel, this.FontSize); + + var palette = new ScottPlot.Palettes.Category10(); + + var legendPalette = data.Select(d => d.JobId) + .Distinct() + .Select((jobId, index) => (jobId, index)) + .ToDictionary(t => t.jobId, t => palette.GetColor(t.index)); + + plt.Legend.IsVisible = true; + plt.Legend.Alignment = Alignment.UpperRight; + plt.Legend.FontSize = this.FontSize; + var legend = data.Select(d => d.JobId) + .Distinct() + .Select((label, index) => new LegendItem() + { + LabelText = label, + FillColor = legendPalette[label] + }) + .ToList(); + + plt.Legend.ManualItems.AddRange(legend); + + var jobCount = plt.Legend.ManualItems.Count; + var ticks = data + .Select((d, index) => new Tick(index, d.Target)) + .ToArray(); + + plt.Axes.Left.TickLabelStyle.FontSize = this.FontSize; + plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks); + plt.Axes.Bottom.MajorTickStyle.Length = 0; + plt.Axes.Bottom.TickLabelStyle.FontSize = this.FontSize; + + if (this.RotateLabels) + { + plt.Axes.Bottom.TickLabelStyle.Rotation = 45; + plt.Axes.Bottom.TickLabelStyle.Alignment = Alignment.MiddleLeft; + + // determine the width of the largest tick label + float largestLabelWidth = 0; + foreach (Tick tick in ticks) + { + PixelSize size = plt.Axes.Bottom.TickLabelStyle.Measure(tick.Label).Size; + largestLabelWidth = Math.Max(largestLabelWidth, size.Width); + } + + // ensure axis panels do not get smaller than the largest label + plt.Axes.Bottom.MinimumSize = largestLabelWidth * 2; + plt.Axes.Right.MinimumSize = largestLabelWidth; + } + + var bars = data + .Select((d, index) => new Bar() + { + Position = ticks[index].Position, + Value = d.Mean, + Error = d.StdError, + FillColor = legendPalette[d.JobId] + }); + plt.Add.Bars(bars.ToList()); + + // Tell the plot to autoscale with no padding beneath the bars + plt.Axes.Margins(bottom: 0, right: .2); + + plt.PlottableList.AddRange(annotations); + + plt.SavePng(fileName, this.Width, this.Height); + return Path.GetFullPath(fileName); + } + + private string CreateBoxPlot(string title, string fileName, string yLabel, string xLabel, IEnumerable data, IReadOnlyList annotations) + { + Plot plt = new Plot(); + plt.Title(title, this.TitleFontSize); + plt.YLabel(yLabel, this.FontSize); + plt.XLabel(xLabel, this.FontSize); + + var palette = new ScottPlot.Palettes.Category10(); + + var legendPalette = data.Select(d => d.JobId) + .Distinct() + .Select((jobId, index) => (jobId, index)) + .ToDictionary(t => t.jobId, t => palette.GetColor(t.index)); + + plt.Legend.IsVisible = true; + plt.Legend.Alignment = Alignment.UpperRight; + plt.Legend.FontSize = this.FontSize; + var legend = data.Select(d => d.JobId) + .Distinct() + .Select((label, index) => new LegendItem() + { + LabelText = label, + FillColor = legendPalette[label] + }) + .ToList(); + + plt.Legend.ManualItems.AddRange(legend); + + var jobCount = plt.Legend.ManualItems.Count; + var ticks = data + .Select((d, index) => new Tick(index, d.Target)) + .ToArray(); + + plt.Axes.Left.TickLabelStyle.FontSize = this.FontSize; + plt.Axes.Bottom.TickGenerator = new ScottPlot.TickGenerators.NumericManual(ticks); + plt.Axes.Bottom.MajorTickStyle.Length = 0; + plt.Axes.Bottom.TickLabelStyle.FontSize = this.FontSize; + + if (this.RotateLabels) + { + plt.Axes.Bottom.TickLabelStyle.Rotation = 45; + plt.Axes.Bottom.TickLabelStyle.Alignment = Alignment.MiddleLeft; + + // determine the width of the largest tick label + float largestLabelWidth = 0; + foreach (Tick tick in ticks) + { + PixelSize size = plt.Axes.Bottom.TickLabelStyle.Measure(tick.Label).Size; + largestLabelWidth = Math.Max(largestLabelWidth, size.Width); + } + + // ensure axis panels do not get smaller than the largest label + plt.Axes.Bottom.MinimumSize = largestLabelWidth * 2; + plt.Axes.Right.MinimumSize = largestLabelWidth; + } + + int globalIndex = 0; + foreach (var (targetGroup, targetGroupIndex) in data.GroupBy(s => s.Target).Select((targetGroup, index) => (targetGroup, index))) + { + var boxes = targetGroup.Select(job => (job.JobId, Stats: job.CalculateBoxPlotStatistics())).Select((j, jobIndex) => new Box() + { + Position = ticks[globalIndex++].Position, + FillStyle = new FillStyle() { Color = legendPalette[j.JobId] }, + LineStyle = new LineStyle() { Color = Colors.Black }, + BoxMin = j.Stats.Q1, + BoxMax = j.Stats.Q3, + WhiskerMin = j.Stats.Min, + WhiskerMax = j.Stats.Max, + BoxMiddle = j.Stats.Median + }) + .ToList(); + plt.Add.Boxes(boxes); + } + + // Tell the plot to autoscale with a small padding below the boxes. + plt.Axes.Margins(bottom: 0.05, right: .2); + + plt.PlottableList.AddRange(annotations); + + plt.SavePng(fileName, this.Width, this.Height); + return Path.GetFullPath(fileName); + } + + /// + /// Provides a list of annotations to put over the data area. + /// + /// The version to be displayed. + /// A list of annotations for every plot. + private IReadOnlyList GetAnnotations(string version) + { + var versionAnnotation = new Annotation() + { + LabelStyle = + { + Text = version, + FontSize = 14, + ForeColor = new Color(0, 0, 0, 100) + }, + OffsetY = 10, + OffsetX = 20, + Alignment = Alignment.LowerRight + }; + + + return new[] { versionAnnotation }; + } + + private class ChartStats + { + public ChartStats(string Target, string JobId, IReadOnlyList Values) + { + this.Target = Target; + this.JobId = JobId; + this.Values = Values; + } + + public string Target { get; } + + public string JobId { get; } + + public IReadOnlyList Values { get; } + + public double Min => this.Values.DefaultIfEmpty(0d).Min(); + + public double Max => this.Values.DefaultIfEmpty(0d).Max(); + + public double Mean => this.Values.DefaultIfEmpty(0d).Average(); + + public double StdError => StandardError(this.Values); + + + private static (int MidPoint, double Median) CalculateMedian(ReadOnlySpan values) + { + int n = values.Length; + var midPoint = n / 2; + + // Check if count is even, if so use average of the two middle values, + // otherwise take the middle value. + var median = n % 2 == 0 ? (values[midPoint - 1] + values[midPoint]) / 2d : values[midPoint]; + return (midPoint, median); + } + + /// + /// Calculate the mid points. + /// + /// + public (double Min, double Q1, double Median, double Q3, double Max, double[] Outliers) CalculateBoxPlotStatistics() + { + var values = this.Values.ToArray(); + Array.Sort(values); + var s = values.AsSpan(); + var (midPoint, median) = CalculateMedian(s); + + var (q1Index, q1) = midPoint > 0 ? CalculateMedian(s.Slice(0, midPoint)) : (midPoint, median); + var (q3Index, q3) = midPoint + 1 < s.Length ? CalculateMedian(s.Slice(midPoint + 1)) : (midPoint, median); + var iqr = q3 - q1; + var lowerFence = q1 - 1.5d * iqr; + var upperFence = q3 + 1.5d * iqr; + var outliers = values.Where(v => v < lowerFence || v > upperFence).ToArray(); + var nonOutliers = values.Where(v => v >= lowerFence && v <= upperFence).ToArray(); + return ( + nonOutliers.FirstOrDefault(), + q1, + median, + q3, + nonOutliers.LastOrDefault(), + outliers + ); + } + } + } } \ No newline at end of file diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj b/src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj index bd6c053bab..5def448d9c 100644 --- a/src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj @@ -9,10 +9,12 @@ enable + + - - - + + + diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs b/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs index daf3e2222e..d49c3d24e1 100644 --- a/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs @@ -3,7 +3,7 @@ using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains; using System; -using System.Collections.Generic; +using System.IO; using System.Linq; using System.Reflection; @@ -21,6 +21,28 @@ internal static class BenchmarkEnumerator /// The benchmarks inside the assembly. public static BenchmarkRunInfo[] GetBenchmarksFromAssemblyPath(string assemblyPath) { +#if NET462 + // Temporary workaround for BenchmarkDotNet assembly loading issue that occurred under the following conditions: + // 1. Run BenchmarkDotNet.Samples project with following command. + // > dotnet test -c Release --list-tests --framework net462 -tl:off + // 2. When using `BenchmarkDotNet.TestAdapter` package and targeting .NET Framework. + AppDomain.CurrentDomain.AssemblyResolve += (sender, eventArgs) => + { + if (eventArgs.Name.StartsWith("BenchmarkDotNet, Version=")) + { + var baseDir = Path.GetDirectoryName(assemblyPath); + var path = Path.Combine(baseDir, "BenchmarkDotNet.dll"); + if (File.Exists(path)) + { + return Assembly.LoadFrom(path); + } + } + + // Fallback to default assembly resolver + return null; + }; +#endif + var assembly = Assembly.LoadFrom(assemblyPath); var isDebugAssembly = assembly.IsJitOptimizationDisabled() ?? false; diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs b/src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs index f1cd64f34e..aad574ed76 100644 --- a/src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkExecutor.cs @@ -1,4 +1,5 @@ using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using BenchmarkDotNet.TestAdapter.Remoting; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -14,7 +15,7 @@ namespace BenchmarkDotNet.TestAdapter /// internal class BenchmarkExecutor { - private readonly CancellationTokenSource cts = new (); + private readonly CancellationTokenSource cts = new(); /// /// Runs all the benchmarks in the given assembly, updating the TestExecutionRecorder as they get run. @@ -67,7 +68,10 @@ public void RunBenchmarks(string assemblyPath, TestExecutionRecorderWrapper reco .Select(b => new BenchmarkRunInfo( b.BenchmarksCases, b.Type, - b.Config.AddEventProcessor(eventProcessor).AddLogger(logger).CreateImmutableConfig())) + b.Config.AddEventProcessor(eventProcessor) + .AddLogger(logger) + .RemoveLoggersOfType() // Console logs are also outputted by VSTestLogger. + .CreateImmutableConfig())) .ToArray(); // Run all the benchmarks, and ensure that any tests that don't have a result yet are sent. diff --git a/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkExecutorWrapper.cs b/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkExecutorWrapper.cs index 646ae2f8be..28444ae8db 100644 --- a/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkExecutorWrapper.cs +++ b/src/BenchmarkDotNet.TestAdapter/Remoting/BenchmarkExecutorWrapper.cs @@ -8,7 +8,7 @@ namespace BenchmarkDotNet.TestAdapter.Remoting /// internal class BenchmarkExecutorWrapper : MarshalByRefObject { - private readonly BenchmarkExecutor benchmarkExecutor = new (); + private readonly BenchmarkExecutor benchmarkExecutor = new(); public void RunBenchmarks(string assemblyPath, TestExecutionRecorderWrapper recorder, HashSet? benchmarkIds = null) { diff --git a/src/BenchmarkDotNet.TestAdapter/Utility/LoggerHelper.cs b/src/BenchmarkDotNet.TestAdapter/Utility/LoggerHelper.cs new file mode 100644 index 0000000000..08238c1927 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Utility/LoggerHelper.cs @@ -0,0 +1,57 @@ +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using System.Diagnostics; +using System.IO; + +namespace BenchmarkDotNet.TestAdapter; + +internal class LoggerHelper +{ + public LoggerHelper(IMessageLogger logger, Stopwatch stopwatch) + { + InnerLogger = logger; + Stopwatch = stopwatch; + } + + public IMessageLogger InnerLogger { get; private set; } + + public Stopwatch Stopwatch { get; private set; } + + public void Log(string format, params object[] args) + { + SendMessage(TestMessageLevel.Informational, null, string.Format(format, args)); + } + + public void LogWithSource(string source, string format, params object[] args) + { + SendMessage(TestMessageLevel.Informational, source, string.Format(format, args)); + } + + public void LogError(string format, params object[] args) + { + SendMessage(TestMessageLevel.Error, null, string.Format(format, args)); + } + + public void LogErrorWithSource(string source, string format, params object[] args) + { + SendMessage(TestMessageLevel.Error, source, string.Format(format, args)); + } + + public void LogWarning(string format, params object[] args) + { + SendMessage(TestMessageLevel.Warning, null, string.Format(format, args)); + } + + public void LogWarningWithSource(string source, string format, params object[] args) + { + SendMessage(TestMessageLevel.Warning, source, string.Format(format, args)); + } + + private void SendMessage(TestMessageLevel level, string? assemblyName, string message) + { + var assemblyText = assemblyName == null + ? "" : + $"{Path.GetFileNameWithoutExtension(assemblyName)}: "; + + InnerLogger.SendMessage(level, $"[BenchmarkDotNet {Stopwatch.Elapsed:hh\\:mm\\:ss\\.ff}] {assemblyText}{message}"); + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/Utility/TestCaseFilter.cs b/src/BenchmarkDotNet.TestAdapter/Utility/TestCaseFilter.cs new file mode 100644 index 0000000000..ffd891bf61 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/Utility/TestCaseFilter.cs @@ -0,0 +1,166 @@ +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; + +namespace BenchmarkDotNet.TestAdapter; + +internal class TestCaseFilter +{ + private const string DisplayNameString = "DisplayName"; + private const string FullyQualifiedNameString = "FullyQualifiedName"; + + private readonly HashSet knownTraits; + private List supportedPropertyNames; + private readonly ITestCaseFilterExpression? filterExpression; + private readonly bool successfullyGotFilter; + private readonly bool isDiscovery; + + public TestCaseFilter(IDiscoveryContext discoveryContext, LoggerHelper logger) + { + // Traits are not known at discovery time because we load them from benchmarks + isDiscovery = true; + knownTraits = []; + supportedPropertyNames = GetSupportedPropertyNames(); + successfullyGotFilter = GetTestCaseFilterExpressionFromDiscoveryContext(discoveryContext, logger, out filterExpression); + } + + public TestCaseFilter(IRunContext runContext, LoggerHelper logger, string assemblyFileName, HashSet knownTraits) + { + this.knownTraits = knownTraits; + supportedPropertyNames = GetSupportedPropertyNames(); + successfullyGotFilter = GetTestCaseFilterExpression(runContext, logger, assemblyFileName, out filterExpression); + } + + public string GetTestCaseFilterValue() + { + return successfullyGotFilter + ? filterExpression?.TestCaseFilterValue ?? "" + : ""; + } + + public bool MatchTestCase(TestCase testCase) + { + if (!successfullyGotFilter) + { + // Had an error while getting filter, match no testcase to ensure discovered test list is empty + return false; + } + else if (filterExpression == null) + { + // No filter specified, keep every testcase + return true; + } + + return filterExpression.MatchTestCase(testCase, p => PropertyProvider(testCase, p)); + } + + public object? PropertyProvider(TestCase testCase, string name) + { + // Traits filtering + if (isDiscovery || knownTraits.Contains(name)) + { + var result = new List(); + + foreach (var trait in GetTraits(testCase)) + if (string.Equals(trait.Key, name, StringComparison.OrdinalIgnoreCase)) + result.Add(trait.Value); + + if (result.Count > 0) + return result.ToArray(); + } + + // Property filtering + switch (name.ToLowerInvariant()) + { + // FullyQualifiedName + case "fullyqualifiedname": + return testCase.FullyQualifiedName; + // DisplayName + case "displayname": + return testCase.DisplayName; + default: + return null; + } + } + + private bool GetTestCaseFilterExpression(IRunContext runContext, LoggerHelper logger, string assemblyFileName, out ITestCaseFilterExpression? filter) + { + filter = null; + + try + { + filter = runContext.GetTestCaseFilter(supportedPropertyNames, null!); + return true; + } + catch (TestPlatformFormatException e) + { + logger.LogWarning("{0}: Exception filtering tests: {1}", Path.GetFileNameWithoutExtension(assemblyFileName), e.Message); + return false; + } + } + + private bool GetTestCaseFilterExpressionFromDiscoveryContext(IDiscoveryContext discoveryContext, LoggerHelper logger, out ITestCaseFilterExpression? filter) + { + filter = null; + + if (discoveryContext is IRunContext runContext) + { + try + { + filter = runContext.GetTestCaseFilter(supportedPropertyNames, null!); + return true; + } + catch (TestPlatformException e) + { + logger.LogWarning("Exception filtering tests: {0}", e.Message); + return false; + } + } + else + { + try + { + // GetTestCaseFilter is present on DiscoveryContext but not in IDiscoveryContext interface + var method = discoveryContext.GetType().GetRuntimeMethod("GetTestCaseFilter", [typeof(IEnumerable), typeof(Func)]); + filter = (ITestCaseFilterExpression)method?.Invoke(discoveryContext, [supportedPropertyNames, null])!; + + return true; + } + catch (TargetInvocationException e) + { + if (e?.InnerException is TestPlatformException ex) + { + logger.LogWarning("Exception filtering tests: {0}", ex.InnerException.Message ?? ""); + return false; + } + + throw e!.InnerException; + } + } + } + + private List GetSupportedPropertyNames() + { + // Returns the set of well-known property names usually used with the Test Plugins (Used Test Traits + DisplayName + FullyQualifiedName) + if (supportedPropertyNames == null) + { + supportedPropertyNames = knownTraits.ToList(); + supportedPropertyNames.Add(DisplayNameString); + supportedPropertyNames.Add(FullyQualifiedNameString); + } + + return supportedPropertyNames; + } + + private static IEnumerable> GetTraits(TestCase testCase) + { + var traitProperty = TestProperty.Find("TestObject.Traits"); + return traitProperty != null + ? testCase.GetPropertyValue(traitProperty, Array.Empty>()) + : []; + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs b/src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs index eb6695de1b..3a3e155bf4 100644 --- a/src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs +++ b/src/BenchmarkDotNet.TestAdapter/VSTestAdapter.cs @@ -4,6 +4,7 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Reflection; @@ -42,11 +43,18 @@ public void DiscoverTests( IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { + var stopwatch = Stopwatch.StartNew(); + var loggerHelper = new LoggerHelper(logger, stopwatch); + var testCaseFilter = new TestCaseFilter(discoveryContext, loggerHelper); + foreach (var source in sources) { ValidateSourceIsAssemblyOrThrow(source); foreach (var testCase in GetVsTestCasesFromAssembly(source, logger)) { + if (!testCaseFilter.MatchTestCase(testCase)) + continue; + discoverySink.SendTestCase(testCase); } } @@ -67,14 +75,19 @@ public void RunTests(IEnumerable? tests, IRunContext? runContext, IFra cts ??= new CancellationTokenSource(); + var stopwatch = Stopwatch.StartNew(); + var logger = new LoggerHelper(frameworkHandle, stopwatch); + foreach (var testsPerAssembly in tests.GroupBy(t => t.Source)) + { RunBenchmarks(testsPerAssembly.Key, frameworkHandle, testsPerAssembly); + } cts = null; } /// - /// Runs all benchmarks in the given set of sources (assemblies). + /// Runs all/filtered benchmarks in the given set of sources (assemblies). /// /// The assemblies to run. /// A context that the run is performed in. @@ -88,8 +101,31 @@ public void RunTests(IEnumerable? sources, IRunContext? runContext, IFra cts ??= new CancellationTokenSource(); + var stopwatch = Stopwatch.StartNew(); + var logger = new LoggerHelper(frameworkHandle, stopwatch); + foreach (var source in sources) - RunBenchmarks(source, frameworkHandle); + { + var filter = new TestCaseFilter(runContext!, logger, source, ["Category"]); + if (filter.GetTestCaseFilterValue() != "") + { + var discoveredBenchmarks = GetVsTestCasesFromAssembly(source, frameworkHandle); + var filteredTestCases = discoveredBenchmarks.Where(x => filter.MatchTestCase(x)) + .ToArray(); + + if (filteredTestCases.Length == 0) + continue; + + // Run filtered tests. + RunBenchmarks(source, frameworkHandle, filteredTestCases); + } + else + { + // Run all benchmarks + RunBenchmarks(source, frameworkHandle); + } + } + cts = null; } diff --git a/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs b/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs index 35438d4d80..f002e7eb29 100644 --- a/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs +++ b/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs @@ -25,9 +25,9 @@ internal class VsTestEventProcessor : EventProcessor private readonly Dictionary cases; private readonly TestExecutionRecorderWrapper recorder; private readonly CancellationToken cancellationToken; - private readonly Stopwatch runTimerStopwatch = new (); - private readonly Dictionary testResults = new (); - private readonly HashSet sentTestResults = new (); + private readonly Stopwatch runTimerStopwatch = new(); + private readonly Dictionary testResults = new(); + private readonly HashSet sentTestResults = new(); public VsTestEventProcessor( List cases, diff --git a/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props b/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props index a9e8340b30..b672c62139 100644 --- a/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props +++ b/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props @@ -1,6 +1,8 @@  $(MSBuildThisFileDirectory)..\entrypoints\ + + false + false - \ No newline at end of file + diff --git a/src/BenchmarkDotNet/Analysers/BaselineCustomAnalyzer.cs b/src/BenchmarkDotNet/Analysers/BaselineCustomAnalyzer.cs index 300d520f15..9f2ff0028c 100644 --- a/src/BenchmarkDotNet/Analysers/BaselineCustomAnalyzer.cs +++ b/src/BenchmarkDotNet/Analysers/BaselineCustomAnalyzer.cs @@ -31,7 +31,8 @@ protected override IEnumerable AnalyseSummary(Summary summary) continue; var message = "A question mark '?' symbol indicates that it was not possible to compute the " + - $"({columnNames}) column(s) because the baseline value is too close to zero."; + $"({columnNames}) column(s) because the baseline or benchmark could not be found, or " + + $"the baseline value is too close to zero."; yield return Conclusion.CreateWarning(Id, message); } diff --git a/src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs b/src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs index 9269b3ebf0..4d83a5d154 100644 --- a/src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs +++ b/src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs @@ -19,7 +19,7 @@ private ZeroMeasurementAnalyser() { } protected override IEnumerable AnalyseReport(BenchmarkReport report, Summary summary) { - var currentFrequency = summary.HostEnvironmentInfo.CpuInfo.Value.MaxFrequency; + var currentFrequency = summary.HostEnvironmentInfo.Cpu.Value.MaxFrequency(); if (!currentFrequency.HasValue || currentFrequency <= 0) currentFrequency = FallbackCpuResolutionValue.ToFrequency(); diff --git a/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs b/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs index bff956e968..21f2903124 100644 --- a/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserAttribute.cs @@ -9,6 +9,10 @@ public class ExceptionDiagnoserAttribute : Attribute, IConfigSource { public IConfig Config { get; } - public ExceptionDiagnoserAttribute() => Config = ManualConfig.CreateEmpty().AddDiagnoser(ExceptionDiagnoser.Default); + /// Display Exceptions column. True by default. + public ExceptionDiagnoserAttribute(bool displayExceptionsIfZeroValue = true) + { + Config = ManualConfig.CreateEmpty().AddDiagnoser(new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue))); + } } } diff --git a/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserConfig.cs b/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserConfig.cs new file mode 100644 index 0000000000..86f8d99be4 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/ExceptionDiagnoserConfig.cs @@ -0,0 +1,20 @@ +using JetBrains.Annotations; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BenchmarkDotNet.Attributes +{ + public class ExceptionDiagnoserConfig + { + /// Determines whether the Exceptions column is displayed when its value is not calculated. True by default. + + [PublicAPI] + public ExceptionDiagnoserConfig(bool displayExceptionsIfZeroValue = true) + { + DisplayExceptionsIfZeroValue = displayExceptionsIfZeroValue; + } + + public bool DisplayExceptionsIfZeroValue { get; } + } +} diff --git a/src/BenchmarkDotNet/Attributes/Exporters/PerfonarExporterAttribute.cs b/src/BenchmarkDotNet/Attributes/Exporters/PerfonarExporterAttribute.cs new file mode 100644 index 0000000000..e7547f3e7a --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/Exporters/PerfonarExporterAttribute.cs @@ -0,0 +1,10 @@ +using System; +using BenchmarkDotNet.Exporters; + +namespace BenchmarkDotNet.Attributes; + +/// +/// IMPORTANT: Not fully implemented yet +/// +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)] +internal class PerfonarExporterAttribute() : ExporterConfigBaseAttribute(new PerfonarJsonExporter(), new PerfonarMdExporter()); \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs b/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs index 83b5051895..b20a9108ca 100644 --- a/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs @@ -9,9 +9,10 @@ public class OrdererAttribute : Attribute, IConfigSource { public OrdererAttribute( SummaryOrderPolicy summaryOrderPolicy = SummaryOrderPolicy.Default, - MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared) + MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared, + JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default) { - Config = ManualConfig.CreateEmpty().WithOrderer(new DefaultOrderer(summaryOrderPolicy, methodOrderPolicy)); + Config = ManualConfig.CreateEmpty().WithOrderer(new DefaultOrderer(summaryOrderPolicy, methodOrderPolicy, jobOrderPolicy)); } public IConfig Config { get; } diff --git a/src/BenchmarkDotNet/Attributes/ThreadingDiagnoserAttribute.cs b/src/BenchmarkDotNet/Attributes/ThreadingDiagnoserAttribute.cs index 7627170b8b..4ad7651bfc 100644 --- a/src/BenchmarkDotNet/Attributes/ThreadingDiagnoserAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/ThreadingDiagnoserAttribute.cs @@ -1,6 +1,7 @@ using System; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; +using JetBrains.Annotations; namespace BenchmarkDotNet.Attributes { @@ -9,6 +10,15 @@ public class ThreadingDiagnoserAttribute : Attribute, IConfigSource { public IConfig Config { get; } - public ThreadingDiagnoserAttribute() => Config = ManualConfig.CreateEmpty().AddDiagnoser(ThreadingDiagnoser.Default); + //public ThreadingDiagnoserAttribute() => Config = ManualConfig.CreateEmpty().AddDiagnoser(ThreadingDiagnoser.Default); + + /// Display configuration for 'LockContentionCount' when it is empty. True (displayed) by default. + /// Display configuration for 'CompletedWorkItemCount' when it is empty. True (displayed) by default. + + [PublicAPI] + public ThreadingDiagnoserAttribute(bool displayLockContentionWhenZero = true, bool displayCompletedWorkItemCountWhenZero = true) + { + Config = ManualConfig.CreateEmpty().AddDiagnoser(new ThreadingDiagnoser(new ThreadingDiagnoserConfig(displayLockContentionWhenZero, displayCompletedWorkItemCountWhenZero))); + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Attributes/WakeLockAttribute.cs b/src/BenchmarkDotNet/Attributes/WakeLockAttribute.cs new file mode 100644 index 0000000000..1243ac8f8f --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/WakeLockAttribute.cs @@ -0,0 +1,18 @@ +using BenchmarkDotNet.Configs; +using System; + +namespace BenchmarkDotNet.Attributes +{ + /// + /// Placing a on your assembly or class controls whether the + /// Windows system enters sleep or turns off the display while benchmarks run. + /// + [AttributeUsage(AttributeTargets.Assembly | AttributeTargets.Class)] + public sealed class WakeLockAttribute : Attribute, IConfigSource + { + public WakeLockAttribute(WakeLockType wakeLockType) => + Config = ManualConfig.CreateEmpty().WithWakeLock(wakeLockType); + + public IConfig Config { get; } + } +} diff --git a/src/BenchmarkDotNet/BenchmarkDotNet.csproj b/src/BenchmarkDotNet/BenchmarkDotNet.csproj index 4bdceb0f8a..7953a036ae 100644 --- a/src/BenchmarkDotNet/BenchmarkDotNet.csproj +++ b/src/BenchmarkDotNet/BenchmarkDotNet.csproj @@ -13,26 +13,25 @@ - - - - - + + + + - - - + + + - + diff --git a/src/BenchmarkDotNet/Code/CodeGenerator.cs b/src/BenchmarkDotNet/Code/CodeGenerator.cs index 005564b77c..9a6228dd88 100644 --- a/src/BenchmarkDotNet/Code/CodeGenerator.cs +++ b/src/BenchmarkDotNet/Code/CodeGenerator.cs @@ -56,6 +56,7 @@ internal static string Generate(BuildPartition buildPartition) .Replace("$OverheadImplementation$", provider.OverheadImplementation) .Replace("$ConsumeField$", provider.ConsumeField) .Replace("$JobSetDefinition$", GetJobsSetDefinition(benchmark)) + .Replace("$ParamsInitializer$", GetParamsInitializer(benchmark)) .Replace("$ParamsContent$", GetParamsContent(benchmark)) .Replace("$ArgumentsDefinition$", GetArgumentsDefinition(benchmark)) .Replace("$DeclareArgumentFields$", GetDeclareArgumentFields(benchmark)) @@ -186,7 +187,15 @@ private static DeclarationsProvider GetDeclarationsProvider(Descriptor descripto return new NonVoidDeclarationsProvider(descriptor); } + private static string GetParamsInitializer(BenchmarkCase benchmarkCase) + => string.Join( + ", ", + benchmarkCase.Parameters.Items + .Where(parameter => !parameter.IsArgument && !parameter.IsStatic) + .Select(parameter => $"{parameter.Name} = default")); + // internal for tests + internal static string GetParamsContent(BenchmarkCase benchmarkCase) => string.Join( string.Empty, diff --git a/src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs b/src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs index 51edbb40f9..52a95bc04e 100644 --- a/src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs +++ b/src/BenchmarkDotNet/Columns/StatisticalTestColumn.cs @@ -15,9 +15,9 @@ public class StatisticalTestColumn(Threshold threshold, SignificanceLevel? signi { private static readonly SignificanceLevel DefaultSignificanceLevel = SignificanceLevel.P1E5; - public static StatisticalTestColumn CreateDefault() => new (new PercentValue(10).ToThreshold()); + public static StatisticalTestColumn CreateDefault() => new(new PercentValue(10).ToThreshold()); - public static StatisticalTestColumn Create(Threshold threshold, SignificanceLevel? significanceLevel = null) => new (threshold, significanceLevel); + public static StatisticalTestColumn Create(Threshold threshold, SignificanceLevel? significanceLevel = null) => new(threshold, significanceLevel); public static StatisticalTestColumn Create(string threshold, SignificanceLevel? significanceLevel = null) { diff --git a/src/BenchmarkDotNet/Configs/ConfigOptions.cs b/src/BenchmarkDotNet/Configs/ConfigOptions.cs index c1ad75d642..06bc7768c7 100644 --- a/src/BenchmarkDotNet/Configs/ConfigOptions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigOptions.cs @@ -48,7 +48,11 @@ public enum ConfigOptions /// /// Continue the execution if the last run was stopped. /// - Resume = 1 << 9 + Resume = 1 << 9, + /// + /// Determines whether parallel build of benchmark projects should be disabled. + /// + DisableParallelBuild = 1 << 10, } internal static class ConfigOptionsExtensions diff --git a/src/BenchmarkDotNet/Configs/DebugConfig.cs b/src/BenchmarkDotNet/Configs/DebugConfig.cs index 0fdda7b08d..551c38fa09 100644 --- a/src/BenchmarkDotNet/Configs/DebugConfig.cs +++ b/src/BenchmarkDotNet/Configs/DebugConfig.cs @@ -71,6 +71,7 @@ public abstract class DebugConfig : IConfig public SummaryStyle SummaryStyle => SummaryStyle.Default; public ConfigUnionRule UnionRule => ConfigUnionRule.Union; public TimeSpan BuildTimeout => DefaultConfig.Instance.BuildTimeout; + public WakeLockType WakeLock => WakeLockType.None; public string ArtifactsPath => null; // DefaultConfig.ArtifactsPath will be used if the user does not specify it in explicit way diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index 2323d4fdc4..d188b67415 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -4,6 +4,7 @@ using System.IO; using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.EventProcessors; using BenchmarkDotNet.Exporters; @@ -71,6 +72,7 @@ public IEnumerable GetValidators() yield return DeferredExecutionValidator.FailOnError; yield return ParamsAllValuesValidator.FailOnError; yield return ParamsValidator.FailOnError; + yield return RuntimeValidator.DontFailOnError; } public IOrderer Orderer => null; @@ -86,11 +88,13 @@ public IEnumerable GetValidators() public TimeSpan BuildTimeout => TimeSpan.FromSeconds(120); + public WakeLockType WakeLock => WakeLockType.System; + public string ArtifactsPath { get { - var root = RuntimeInformation.IsAndroid() ? + var root = OsDetector.IsAndroid() ? Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) : Directory.GetCurrentDirectory(); return Path.Combine(root, "BenchmarkDotNet.Artifacts"); diff --git a/src/BenchmarkDotNet/Configs/IConfig.cs b/src/BenchmarkDotNet/Configs/IConfig.cs index b311c235f5..9e3bf50ce6 100644 --- a/src/BenchmarkDotNet/Configs/IConfig.cs +++ b/src/BenchmarkDotNet/Configs/IConfig.cs @@ -55,6 +55,8 @@ public interface IConfig /// TimeSpan BuildTimeout { get; } + public WakeLockType WakeLock { get; } + /// /// Collect any errors or warnings when composing the configuration /// diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs index b6e03126fd..0043ecbe4b 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs @@ -56,6 +56,7 @@ internal ImmutableConfig( SummaryStyle summaryStyle, ConfigOptions options, TimeSpan buildTimeout, + WakeLockType wakeLock, IReadOnlyList configAnalysisConclusion) { columnProviders = uniqueColumnProviders; @@ -78,6 +79,7 @@ internal ImmutableConfig( SummaryStyle = summaryStyle; Options = options; BuildTimeout = buildTimeout; + WakeLock = wakeLock; ConfigAnalysisConclusion = configAnalysisConclusion; } @@ -89,6 +91,7 @@ internal ImmutableConfig( public ICategoryDiscoverer CategoryDiscoverer { get; } public SummaryStyle SummaryStyle { get; } public TimeSpan BuildTimeout { get; } + public WakeLockType WakeLock { get; } public IEnumerable GetColumnProviders() => columnProviders; public IEnumerable GetExporters() => exporters; diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index d7c0b5eb0f..f93e5590d0 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -76,6 +76,7 @@ public static ImmutableConfig Create(IConfig source) source.SummaryStyle ?? SummaryStyle.Default, source.Options, source.BuildTimeout, + source.WakeLock, configAnalyse.AsReadOnly() ); } diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs index 5ea1be24e9..4ececd28f1 100644 --- a/src/BenchmarkDotNet/Configs/ManualConfig.cs +++ b/src/BenchmarkDotNet/Configs/ManualConfig.cs @@ -58,6 +58,7 @@ public class ManualConfig : IConfig [PublicAPI] public ICategoryDiscoverer CategoryDiscoverer { get; set; } [PublicAPI] public SummaryStyle SummaryStyle { get; set; } [PublicAPI] public TimeSpan BuildTimeout { get; set; } = DefaultConfig.Instance.BuildTimeout; + [PublicAPI] public WakeLockType WakeLock { get; set; } = DefaultConfig.Instance.WakeLock; public IReadOnlyList ConfigAnalysisConclusion => emptyConclusion; @@ -109,6 +110,12 @@ public ManualConfig WithBuildTimeout(TimeSpan buildTimeout) return this; } + public ManualConfig WithWakeLock(WakeLockType wakeLockType) + { + WakeLock = wakeLockType; + return this; + } + [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("This method will soon be removed, please start using .AddColumn() instead.")] public void Add(params IColumn[] newColumns) => AddColumn(newColumns); @@ -273,6 +280,7 @@ public void Add(IConfig config) columnHidingRules.AddRange(config.GetColumnHidingRules()); Options |= config.Options; BuildTimeout = GetBuildTimeout(BuildTimeout, config.BuildTimeout); + WakeLock = GetWakeLock(WakeLock, config.WakeLock); } /// @@ -319,6 +327,12 @@ public static ManualConfig Union(IConfig globalConfig, IConfig localConfig) return manualConfig; } + internal ManualConfig RemoveLoggersOfType() + { + loggers.RemoveAll(logger => logger is T); + return this; + } + internal void RemoveAllJobs() => jobs.Clear(); internal void RemoveAllDiagnosers() => diagnosers.Clear(); @@ -327,5 +341,12 @@ private static TimeSpan GetBuildTimeout(TimeSpan current, TimeSpan other) => current == DefaultConfig.Instance.BuildTimeout ? other : TimeSpan.FromMilliseconds(Math.Max(current.TotalMilliseconds, other.TotalMilliseconds)); + + private static WakeLockType GetWakeLock(WakeLockType current, WakeLockType other) + { + if (current == DefaultConfig.Instance.WakeLock) { return other; } + if (other == DefaultConfig.Instance.WakeLock) { return current; } + return current.CompareTo(other) > 0 ? current : other; + } } } diff --git a/src/BenchmarkDotNet/Configs/WakeLockType.cs b/src/BenchmarkDotNet/Configs/WakeLockType.cs new file mode 100644 index 0000000000..f547e44764 --- /dev/null +++ b/src/BenchmarkDotNet/Configs/WakeLockType.cs @@ -0,0 +1,20 @@ +namespace BenchmarkDotNet.Configs +{ + public enum WakeLockType + { + /// + /// Allows the system to enter sleep and/or turn off the display while benchmarks are running. + /// + None, + + /// + /// Forces the system to be in the working state while benchmarks are running. + /// + System, + + /// + /// Forces the display to be on while benchmarks are running. + /// + Display + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index e3aed1fedd..6c7b48b0e2 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -2,6 +2,7 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; +using BenchmarkDotNet.Configs; using BenchmarkDotNet.ConsoleArguments.ListBenchmarks; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; @@ -179,6 +180,9 @@ public bool UseDisassemblyDiagnoser [Option("buildTimeout", Required = false, HelpText = "Build timeout in seconds.")] public int? TimeOutInSeconds { get; set; } + [Option("wakeLock", Required = false, HelpText = "Prevents the system from entering sleep or turning off the display. None/System/Display.")] + public WakeLockType? WakeLock { get; set; } + [Option("stopOnFirstError", Required = false, Default = false, HelpText = "Stop on first error.")] public bool StopOnFirstError { get; set; } @@ -237,8 +241,8 @@ public static IEnumerable Examples yield return new Example("Use Job.ShortRun for running the benchmarks", shortName, new CommandLineOptions { BaseJob = "short" }); yield return new Example("Run benchmarks in process", shortName, new CommandLineOptions { RunInProcess = true }); - yield return new Example("Run benchmarks for .NET 4.7.2, .NET Core 2.1 and Mono. .NET 4.7.2 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "net472", "netcoreapp2.1", "Mono" } }); - yield return new Example("Run benchmarks for .NET Core 2.0, .NET Core 2.1 and .NET Core 2.2. .NET Core 2.0 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "netcoreapp2.0", "netcoreapp2.1", "netcoreapp2.2" } }); + yield return new Example("Run benchmarks for .NET 4.7.2, .NET 8.0 and Mono. .NET 4.7.2 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "net472", "net8.0", "Mono" } }); + yield return new Example("Run benchmarks for .NET Core 3.1, .NET 6.0 and .NET 8.0. .NET Core 3.1 will be baseline because it was first.", longName, new CommandLineOptions { Runtimes = new[] { "netcoreapp3.1", "net6.0", "net8.0" } }); yield return new Example("Use MemoryDiagnoser to get GC stats", shortName, new CommandLineOptions { UseMemoryDiagnoser = true }); yield return new Example("Use DisassemblyDiagnoser to get disassembly", shortName, new CommandLineOptions { UseDisassemblyDiagnoser = true }); yield return new Example("Use HardwareCountersDiagnoser to get hardware counter info", longName, new CommandLineOptions { HardwareCounters = new[] { nameof(HardwareCounter.CacheMisses), nameof(HardwareCounter.InstructionRetired) } }); @@ -250,8 +254,8 @@ public static IEnumerable Examples yield return new Example("Run selected benchmarks once per iteration", longName, new CommandLineOptions { RunOncePerIteration = true }); yield return new Example("Run selected benchmarks 100 times per iteration. Perform single warmup iteration and 5 actual workload iterations", longName, new CommandLineOptions { InvocationCount = 100, WarmupIterationCount = 1, IterationCount = 5}); yield return new Example("Run selected benchmarks 250ms per iteration. Perform from 9 to 15 iterations", longName, new CommandLineOptions { IterationTimeInMilliseconds = 250, MinIterationCount = 9, MaxIterationCount = 15}); - yield return new Example("Run MannWhitney test with relative ratio of 5% for all benchmarks for .NET Core 2.0 (base) vs .NET Core 2.1 (diff). .NET Core 2.0 will be baseline because it was provided as first.", longName, - new CommandLineOptions { Filters = new[] { "*"}, Runtimes = new[] { "netcoreapp2.0", "netcoreapp2.1" }, StatisticalTestThreshold = "5%" }); + yield return new Example("Run MannWhitney test with relative ratio of 5% for all benchmarks for .NET 6.0 (base) vs .NET 8.0 (diff). .NET Core 6.0 will be baseline because it was provided as first.", longName, + new CommandLineOptions { Filters = new[] { "*"}, Runtimes = new[] { "net6.0", "net8.0" }, StatisticalTestThreshold = "5%" }); yield return new Example("Run benchmarks using environment variables 'ENV_VAR_KEY_1' with value 'value_1' and 'ENV_VAR_KEY_2' with value 'value_2'", longName, new CommandLineOptions { EnvironmentVariables = new[] { "ENV_VAR_KEY_1:value_1", "ENV_VAR_KEY_2:value_2" } }); yield return new Example("Hide Mean and Ratio columns (use double quotes for multi-word columns: \"Alloc Ratio\")", shortName, new CommandLineOptions { HiddenColumns = new[] { "Mean", "Ratio" }, }); diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 355f1a6b75..ac61ddb5f1 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -1,774 +1,797 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Linq; -using System.Text; -using BenchmarkDotNet.Columns; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Exporters; -using BenchmarkDotNet.Exporters.Csv; -using BenchmarkDotNet.Exporters.Json; -using BenchmarkDotNet.Exporters.Xml; -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Filters; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Toolchains.CoreRun; -using BenchmarkDotNet.Toolchains.CsProj; -using BenchmarkDotNet.Toolchains.DotNetCli; -using BenchmarkDotNet.Toolchains.InProcess.Emit; -using BenchmarkDotNet.Toolchains.MonoAotLLVM; -using BenchmarkDotNet.Toolchains.MonoWasm; -using BenchmarkDotNet.Toolchains.NativeAot; -using CommandLine; -using Perfolizer.Horology; -using Perfolizer.Mathematics.OutlierDetection; -using BenchmarkDotNet.Toolchains.Mono; -using Perfolizer.Metrology; - -namespace BenchmarkDotNet.ConsoleArguments -{ - public static class ConfigParser - { - private const int MinimumDisplayWidth = 80; - private const char EnvVarKeyValueSeparator = ':'; - - private static readonly IReadOnlyDictionary AvailableJobs = new Dictionary(StringComparer.InvariantCultureIgnoreCase) - { - { "default", Job.Default }, - { "dry", Job.Dry }, - { "short", Job.ShortRun }, - { "medium", Job.MediumRun }, - { "long", Job.LongRun }, - { "verylong", Job.VeryLongRun } - }; - - [SuppressMessage("ReSharper", "StringLiteralTypo")] - [SuppressMessage("ReSharper", "CoVariantArrayConversion")] - private static readonly IReadOnlyDictionary AvailableExporters = - new Dictionary(StringComparer.InvariantCultureIgnoreCase) - { - { "csv", new[] { CsvExporter.Default } }, - { "csvmeasurements", new[] { CsvMeasurementsExporter.Default } }, - { "html", new[] { HtmlExporter.Default } }, - { "markdown", new[] { MarkdownExporter.Default } }, - { "atlassian", new[] { MarkdownExporter.Atlassian } }, - { "stackoverflow", new[] { MarkdownExporter.StackOverflow } }, - { "github", new[] { MarkdownExporter.GitHub } }, - { "plain", new[] { PlainExporter.Default } }, - { "rplot", new[] { CsvMeasurementsExporter.Default, RPlotExporter.Default } }, // R Plots depends on having the full measurements available - { "json", new[] { JsonExporter.Default } }, - { "briefjson", new[] { JsonExporter.Brief } }, - { "fulljson", new[] { JsonExporter.Full } }, - { "asciidoc", new[] { AsciiDocExporter.Default } }, - { "xml", new[] { XmlExporter.Default } }, - { "briefxml", new[] { XmlExporter.Brief } }, - { "fullxml", new[] { XmlExporter.Full } } - }; - - public static (bool isSuccess, IConfig config, CommandLineOptions options) Parse(string[] args, ILogger logger, IConfig? globalConfig = null) - { - (bool isSuccess, IConfig config, CommandLineOptions options) result = default; - - var (expandSuccess, expandedArgs) = ExpandResponseFile(args, logger); - if (!expandSuccess) - { - return (false, default, default); - } - - args = expandedArgs; - using (var parser = CreateParser(logger)) - { - parser - .ParseArguments(args) - .WithParsed(options => result = Validate(options, logger) ? (true, CreateConfig(options, globalConfig, args), options) : (false, default, default)) - .WithNotParsed(errors => result = (false, default, default)); - } - - return result; - } - - private static (bool Success, string[] ExpandedTokens) ExpandResponseFile(string[] args, ILogger logger) - { - List result = new (); - foreach (var arg in args) - { - if (arg.StartsWith("@")) - { - var fileName = arg.Substring(1); - try - { - if (File.Exists(fileName)) - { - var lines = File.ReadAllLines(fileName); - foreach (var line in lines) - { - result.AddRange(ConsumeTokens(line)); - } - } - else - { - logger.WriteLineError($"Response file {fileName} does not exists."); - return (false, Array.Empty()); - } - } - catch (Exception ex) - { - logger.WriteLineError($"Failed to parse RSP file: {fileName}, {ex.Message}"); - return (false, Array.Empty()); - } - } - else - { - result.Add(arg); - } - } - - return (true, result.ToArray()); - } - - private static IEnumerable ConsumeTokens(string line) - { - bool insideQuotes = false; - var token = new StringBuilder(); - for (int i = 0; i < line.Length; i++) - { - char currentChar = line[i]; - if (currentChar == ' ' && !insideQuotes) - { - if (token.Length > 0) - { - yield return GetToken(); - token = new StringBuilder(); - } - - continue; - } - - if (currentChar == '"') - { - insideQuotes = !insideQuotes; - continue; - } - - if (currentChar == '\\' && insideQuotes) - { - if (line[i + 1] == '"') - { - insideQuotes = false; - i++; - continue; - } - - if (line[i + 1] == '\\') - { - token.Append('\\'); - i++; - continue; - } - } - - token.Append(currentChar); - } - - if (token.Length > 0) - { - yield return GetToken(); - } - - string GetToken() - { - var result = token.ToString(); - if (result.Contains(' ')) - { - // Workaround for CommandLine library issue with parsing these kind of args. - return " " + result; - } - - return result; - } - } - - internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Action updater) - { - (bool isSuccess, CommandLineOptions options) result = default; - - ILogger logger = NullLogger.Instance; - using (var parser = CreateParser(logger)) - { - parser - .ParseArguments(args) - .WithParsed(options => result = Validate(options, logger) ? (true, options) : (false, default)) - .WithNotParsed(errors => result = (false, default)); - - if (!result.isSuccess) - { - updatedArgs = null; - return false; - } - - updater(result.options); - - updatedArgs = parser.FormatCommandLine(result.options, settings => settings.SkipDefault = true).Split(); - return true; - } - } - - private static Parser CreateParser(ILogger logger) - => new Parser(settings => - { - settings.CaseInsensitiveEnumValues = true; - settings.CaseSensitive = false; - settings.EnableDashDash = true; - settings.IgnoreUnknownArguments = false; - settings.HelpWriter = new LoggerWrapper(logger); - settings.MaximumDisplayWidth = Math.Max(MinimumDisplayWidth, GetMaximumDisplayWidth()); - }); - - private static bool Validate(CommandLineOptions options, ILogger logger) - { - if (!AvailableJobs.ContainsKey(options.BaseJob)) - { - logger.WriteLineError($"The provided base job \"{options.BaseJob}\" is invalid. Available options are: {string.Join(", ", AvailableJobs.Keys)}."); - return false; - } - - foreach (string runtime in options.Runtimes) - { - if (!TryParse(runtime, out RuntimeMoniker runtimeMoniker)) - { - logger.WriteLineError($"The provided runtime \"{runtime}\" is invalid. Available options are: {string.Join(", ", Enum.GetNames(typeof(RuntimeMoniker)).Select(name => name.ToLower()))}."); - return false; - } - else if (runtimeMoniker == RuntimeMoniker.MonoAOTLLVM && (options.AOTCompilerPath == null || options.AOTCompilerPath.IsNotNullButDoesNotExist())) - { - logger.WriteLineError($"The provided {nameof(options.AOTCompilerPath)} \"{options.AOTCompilerPath}\" does NOT exist. It MUST be provided."); - } - } - - foreach (string exporter in options.Exporters) - if (!AvailableExporters.ContainsKey(exporter)) - { - logger.WriteLineError($"The provided exporter \"{exporter}\" is invalid. Available options are: {string.Join(", ", AvailableExporters.Keys)}."); - return false; - } - - if (options.CliPath.IsNotNullButDoesNotExist()) - { - logger.WriteLineError($"The provided {nameof(options.CliPath)} \"{options.CliPath}\" does NOT exist."); - return false; - } - - foreach (var coreRunPath in options.CoreRunPaths) - if (coreRunPath.IsNotNullButDoesNotExist()) - { - if (Directory.Exists(coreRunPath.FullName)) - { - logger.WriteLineError($"The provided path to CoreRun: \"{coreRunPath}\" exists but it's a directory, not an executable. You need to include CoreRun.exe (corerun on Unix) in the path."); - } - else - { - logger.WriteLineError($"The provided path to CoreRun: \"{coreRunPath}\" does NOT exist."); - } - - return false; - } - - if (options.MonoPath.IsNotNullButDoesNotExist()) - { - logger.WriteLineError($"The provided {nameof(options.MonoPath)} \"{options.MonoPath}\" does NOT exist."); - return false; - } - - if (options.WasmJavascriptEngine.IsNotNullButDoesNotExist()) - { - logger.WriteLineError($"The provided {nameof(options.WasmJavascriptEngine)} \"{options.WasmJavascriptEngine}\" does NOT exist."); - return false; - } - - if (options.IlcPackages.IsNotNullButDoesNotExist()) - { - logger.WriteLineError($"The provided {nameof(options.IlcPackages)} \"{options.IlcPackages}\" does NOT exist."); - return false; - } - - if (options.HardwareCounters.Count() > 3) - { - logger.WriteLineError("You can't use more than 3 HardwareCounters at the same time."); - return false; - } - - foreach (var counterName in options.HardwareCounters) - if (!Enum.TryParse(counterName, ignoreCase: true, out HardwareCounter _)) - { - logger.WriteLineError($"The provided hardware counter \"{counterName}\" is invalid. Available options are: {string.Join("+", Enum.GetNames(typeof(HardwareCounter)))}."); - return false; - } - - if (!string.IsNullOrEmpty(options.StatisticalTestThreshold) && !Threshold.TryParse(options.StatisticalTestThreshold, out _)) - { - logger.WriteLineError("Invalid Threshold for Statistical Test. Use --help to see examples."); - return false; - } - - if (options.EnvironmentVariables.Any(envVar => envVar.IndexOf(EnvVarKeyValueSeparator) <= 0)) - { - logger.WriteLineError($"Environment variable value must be separated from the key using '{EnvVarKeyValueSeparator}'. Use --help to see examples."); - return false; - } - - return true; - } - - private static IConfig CreateConfig(CommandLineOptions options, IConfig globalConfig, string[] args) - { - var config = new ManualConfig(); - - var baseJob = GetBaseJob(options, globalConfig); - var expanded = Expand(baseJob.UnfreezeCopy(), options, args).ToArray(); // UnfreezeCopy ensures that each of the expanded jobs will have it's own ID - if (expanded.Length > 1) - expanded[0] = expanded[0].AsBaseline(); // if the user provides multiple jobs, then the first one should be a baseline - config.AddJob(expanded); - if (config.GetJobs().IsEmpty() && baseJob != Job.Default) - config.AddJob(baseJob); - - config.AddExporter(options.Exporters.SelectMany(exporter => AvailableExporters[exporter]).ToArray()); - - config.AddHardwareCounters(options.HardwareCounters - .Select(counterName => (HardwareCounter)Enum.Parse(typeof(HardwareCounter), counterName, ignoreCase: true)) - .ToArray()); - - if (options.UseMemoryDiagnoser) - config.AddDiagnoser(MemoryDiagnoser.Default); - if (options.UseThreadingDiagnoser) - config.AddDiagnoser(ThreadingDiagnoser.Default); - if (options.UseExceptionDiagnoser) - config.AddDiagnoser(ExceptionDiagnoser.Default); - if (options.UseDisassemblyDiagnoser) - config.AddDiagnoser(new DisassemblyDiagnoser(new DisassemblyDiagnoserConfig( - maxDepth: options.DisassemblerRecursiveDepth, - filters: options.DisassemblerFilters.ToArray(), - exportDiff: options.DisassemblerDiff))); - if (!string.IsNullOrEmpty(options.Profiler)) - config.AddDiagnoser(DiagnosersLoader.GetImplementation(profiler => profiler.ShortName.EqualsWithIgnoreCase(options.Profiler))); - - if (options.DisplayAllStatistics) - config.AddColumn(StatisticColumn.AllStatistics); - if (!string.IsNullOrEmpty(options.StatisticalTestThreshold) && Threshold.TryParse(options.StatisticalTestThreshold, out var threshold)) - config.AddColumn(new StatisticalTestColumn(threshold)); - - if (options.ArtifactsDirectory != null) - config.ArtifactsPath = options.ArtifactsDirectory.FullName; - - var filters = GetFilters(options).ToArray(); - if (filters.Length > 1) - config.AddFilter(new UnionFilter(filters)); - else - config.AddFilter(filters); - - config.HideColumns(options.HiddenColumns.ToArray()); - - config.WithOption(ConfigOptions.JoinSummary, options.Join); - config.WithOption(ConfigOptions.KeepBenchmarkFiles, options.KeepBenchmarkFiles); - config.WithOption(ConfigOptions.DontOverwriteResults, options.DontOverwriteResults); - config.WithOption(ConfigOptions.StopOnFirstError, options.StopOnFirstError); - config.WithOption(ConfigOptions.DisableLogFile, options.DisableLogFile); - config.WithOption(ConfigOptions.LogBuildOutput, options.LogBuildOutput); - config.WithOption(ConfigOptions.GenerateMSBuildBinLog, options.GenerateMSBuildBinLog); - config.WithOption(ConfigOptions.ApplesToApples, options.ApplesToApples); - config.WithOption(ConfigOptions.Resume, options.Resume); - - if (config.Options.IsSet(ConfigOptions.GenerateMSBuildBinLog)) - config.Options |= ConfigOptions.KeepBenchmarkFiles; - - if (options.MaxParameterColumnWidth.HasValue) - config.WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(options.MaxParameterColumnWidth.Value)); - - if (options.TimeOutInSeconds.HasValue) - config.WithBuildTimeout(TimeSpan.FromSeconds(options.TimeOutInSeconds.Value)); - - return config; - } - - private static Job GetBaseJob(CommandLineOptions options, IConfig globalConfig) - { - var baseJob = - globalConfig?.GetJobs().SingleOrDefault(job => job.Meta.IsDefault) // global config might define single custom Default job - ?? AvailableJobs[options.BaseJob.ToLowerInvariant()]; - - if (baseJob != Job.Dry && options.Outliers != OutlierMode.RemoveUpper) - baseJob = baseJob.WithOutlierMode(options.Outliers); - - if (options.Affinity.HasValue) - baseJob = baseJob.WithAffinity((IntPtr)options.Affinity.Value); - - if (options.LaunchCount.HasValue) - baseJob = baseJob.WithLaunchCount(options.LaunchCount.Value); - if (options.WarmupIterationCount.HasValue) - baseJob = baseJob.WithWarmupCount(options.WarmupIterationCount.Value); - if (options.MinWarmupIterationCount.HasValue) - baseJob = baseJob.WithMinWarmupCount(options.MinWarmupIterationCount.Value); - if (options.MaxWarmupIterationCount.HasValue) - baseJob = baseJob.WithMaxWarmupCount(options.MaxWarmupIterationCount.Value); - if (options.IterationTimeInMilliseconds.HasValue) - baseJob = baseJob.WithIterationTime(TimeInterval.FromMilliseconds(options.IterationTimeInMilliseconds.Value)); - if (options.IterationCount.HasValue) - baseJob = baseJob.WithIterationCount(options.IterationCount.Value); - if (options.MinIterationCount.HasValue) - baseJob = baseJob.WithMinIterationCount(options.MinIterationCount.Value); - if (options.MaxIterationCount.HasValue) - baseJob = baseJob.WithMaxIterationCount(options.MaxIterationCount.Value); - if (options.InvocationCount.HasValue) - baseJob = baseJob.WithInvocationCount(options.InvocationCount.Value); - if (options.UnrollFactor.HasValue) - baseJob = baseJob.WithUnrollFactor(options.UnrollFactor.Value); - if (options.RunStrategy.HasValue) - baseJob = baseJob.WithStrategy(options.RunStrategy.Value); - if (options.Platform.HasValue) - baseJob = baseJob.WithPlatform(options.Platform.Value); - if (options.RunOncePerIteration) - baseJob = baseJob.RunOncePerIteration(); - if (options.MemoryRandomization) - baseJob = baseJob.WithMemoryRandomization(); - if (options.NoForcedGCs) - baseJob = baseJob.WithGcForce(false); - if (options.NoEvaluationOverhead) - baseJob = baseJob.WithEvaluateOverhead(false); - - if (options.EnvironmentVariables.Any()) - { - baseJob = baseJob.WithEnvironmentVariables(options.EnvironmentVariables.Select(text => - { - var separated = text.Split(new[] { EnvVarKeyValueSeparator }, 2); - return new EnvironmentVariable(separated[0], separated[1]); - }).ToArray()); - } - - if (AvailableJobs.Values.Contains(baseJob)) // no custom settings - return baseJob; - - return baseJob - .AsDefault(false) // after applying all settings from console args the base job is not default anymore - .AsMutator(); // we mark it as mutator so it will be applied to other jobs defined via attributes and merged later in GetRunnableJobs method - } - - private static IEnumerable Expand(Job baseJob, CommandLineOptions options, string[] args) - { - if (options.RunInProcess) - { - yield return baseJob.WithToolchain(InProcessEmitToolchain.Instance); - } - else if (!string.IsNullOrEmpty(options.ClrVersion)) - { - yield return baseJob.WithRuntime(ClrRuntime.CreateForLocalFullNetFrameworkBuild(options.ClrVersion)); // local builds of .NET Runtime - } - else if (options.CliPath != null && options.Runtimes.IsEmpty() && options.CoreRunPaths.IsEmpty()) - { - yield return CreateCoreJobWithCli(baseJob, options); - } - else - { - // in case both --runtimes and --corerun are specified, the first one is returned first and becomes a baseline job - string first = args.FirstOrDefault(arg => - arg.Equals("--runtimes", StringComparison.OrdinalIgnoreCase) - || arg.Equals("-r", StringComparison.OrdinalIgnoreCase) - - || arg.Equals("--corerun", StringComparison.OrdinalIgnoreCase)); - - if (first is null || first.Equals("--corerun", StringComparison.OrdinalIgnoreCase)) - { - foreach (var coreRunPath in options.CoreRunPaths) - yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local dotnet/runtime builds - - foreach (string runtime in options.Runtimes) // known runtimes - yield return CreateJobForGivenRuntime(baseJob, runtime, options); - } - else - { - foreach (string runtime in options.Runtimes) // known runtimes - yield return CreateJobForGivenRuntime(baseJob, runtime, options); - - foreach (var coreRunPath in options.CoreRunPaths) - yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local dotnet/runtime builds - } - } - } - - private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, CommandLineOptions options) - { - if (!TryParse(runtimeId, out RuntimeMoniker runtimeMoniker)) - { - throw new InvalidOperationException("Impossible, already validated by the Validate method"); - } - - switch (runtimeMoniker) - { - case RuntimeMoniker.Net461: - case RuntimeMoniker.Net462: - case RuntimeMoniker.Net47: - case RuntimeMoniker.Net471: - case RuntimeMoniker.Net472: - case RuntimeMoniker.Net48: - case RuntimeMoniker.Net481: - return baseJob - .WithRuntime(runtimeMoniker.GetRuntime()) - .WithToolchain(CsProjClassicNetToolchain.From(runtimeId, options.RestorePath?.FullName, options.CliPath?.FullName)); - - case RuntimeMoniker.NetCoreApp20: - case RuntimeMoniker.NetCoreApp21: - case RuntimeMoniker.NetCoreApp22: - case RuntimeMoniker.NetCoreApp30: - case RuntimeMoniker.NetCoreApp31: -#pragma warning disable CS0618 // Type or member is obsolete - case RuntimeMoniker.NetCoreApp50: -#pragma warning restore CS0618 // Type or member is obsolete - case RuntimeMoniker.Net50: - case RuntimeMoniker.Net60: - case RuntimeMoniker.Net70: - case RuntimeMoniker.Net80: - case RuntimeMoniker.Net90: - return baseJob - .WithRuntime(runtimeMoniker.GetRuntime()) - .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings(runtimeId, null, runtimeId, options.CliPath?.FullName, options.RestorePath?.FullName))); - - case RuntimeMoniker.Mono: - return baseJob.WithRuntime(new MonoRuntime("Mono", options.MonoPath?.FullName)); - - case RuntimeMoniker.NativeAot60: - return CreateAotJob(baseJob, options, runtimeMoniker, "6.0.0-*", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json"); - - case RuntimeMoniker.NativeAot70: - return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://api.nuget.org/v3/index.json"); - - case RuntimeMoniker.NativeAot80: - return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://api.nuget.org/v3/index.json"); - - case RuntimeMoniker.NativeAot90: - return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json"); - - case RuntimeMoniker.Wasm: - return MakeWasmJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net5.0", runtimeMoniker); - - case RuntimeMoniker.WasmNet50: - return MakeWasmJob(baseJob, options, "net5.0", runtimeMoniker); - - case RuntimeMoniker.WasmNet60: - return MakeWasmJob(baseJob, options, "net6.0", runtimeMoniker); - - case RuntimeMoniker.WasmNet70: - return MakeWasmJob(baseJob, options, "net7.0", runtimeMoniker); - - case RuntimeMoniker.WasmNet80: - return MakeWasmJob(baseJob, options, "net8.0", runtimeMoniker); - - case RuntimeMoniker.WasmNet90: - return MakeWasmJob(baseJob, options, "net9.0", runtimeMoniker); - - case RuntimeMoniker.MonoAOTLLVM: - return MakeMonoAOTLLVMJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net6.0", runtimeMoniker); - - case RuntimeMoniker.MonoAOTLLVMNet60: - return MakeMonoAOTLLVMJob(baseJob, options, "net6.0", runtimeMoniker); - - case RuntimeMoniker.MonoAOTLLVMNet70: - return MakeMonoAOTLLVMJob(baseJob, options, "net7.0", runtimeMoniker); - - case RuntimeMoniker.MonoAOTLLVMNet80: - return MakeMonoAOTLLVMJob(baseJob, options, "net8.0", runtimeMoniker); - - case RuntimeMoniker.MonoAOTLLVMNet90: - return MakeMonoAOTLLVMJob(baseJob, options, "net9.0", runtimeMoniker); - - case RuntimeMoniker.Mono60: - return MakeMonoJob(baseJob, options, MonoRuntime.Mono60); - - case RuntimeMoniker.Mono70: - return MakeMonoJob(baseJob, options, MonoRuntime.Mono70); - - case RuntimeMoniker.Mono80: - return MakeMonoJob(baseJob, options, MonoRuntime.Mono80); - - case RuntimeMoniker.Mono90: - return MakeMonoJob(baseJob, options, MonoRuntime.Mono90); - - default: - throw new NotSupportedException($"Runtime {runtimeId} is not supported"); - } - } - - private static Job CreateAotJob(Job baseJob, CommandLineOptions options, RuntimeMoniker runtimeMoniker, string ilCompilerVersion, string nuGetFeedUrl) - { - var builder = NativeAotToolchain.CreateBuilder(); - - if (options.CliPath != null) - builder.DotNetCli(options.CliPath.FullName); - if (options.RestorePath != null) - builder.PackagesRestorePath(options.RestorePath.FullName); - - if (options.IlcPackages != null) - builder.UseLocalBuild(options.IlcPackages); - else if (!string.IsNullOrEmpty(options.ILCompilerVersion)) - builder.UseNuGet(options.ILCompilerVersion, nuGetFeedUrl); - else - builder.UseNuGet(ilCompilerVersion, nuGetFeedUrl); - - var runtime = runtimeMoniker.GetRuntime(); - builder.TargetFrameworkMoniker(runtime.MsBuildMoniker); - - return baseJob.WithRuntime(runtime).WithToolchain(builder.ToToolchain()); - } - - private static Job MakeMonoJob(Job baseJob, CommandLineOptions options, MonoRuntime runtime) - { - return baseJob - .WithRuntime(runtime) - .WithToolchain(MonoToolchain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: runtime.MsBuildMoniker, - runtimeFrameworkVersion: null, - name: runtime.Name, - customDotNetCliPath: options.CliPath?.FullName, - packagesPath: options.RestorePath?.FullName))); - } - - private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, string msBuildMoniker, RuntimeMoniker moniker) - { - var monoAotLLVMRuntime = new MonoAotLLVMRuntime(aotCompilerPath: options.AOTCompilerPath, aotCompilerMode: options.AOTCompilerMode, msBuildMoniker: msBuildMoniker, moniker: moniker); - - var toolChain = MonoAotLLVMToolChain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: monoAotLLVMRuntime.MsBuildMoniker, - runtimeFrameworkVersion: null, - name: monoAotLLVMRuntime.Name, - customDotNetCliPath: options.CliPath?.FullName, - packagesPath: options.RestorePath?.FullName, - customRuntimePack: options.CustomRuntimePack, - aotCompilerPath: options.AOTCompilerPath.ToString(), - aotCompilerMode: options.AOTCompilerMode)); - - return baseJob.WithRuntime(monoAotLLVMRuntime).WithToolchain(toolChain); - } - - private static Job MakeWasmJob(Job baseJob, CommandLineOptions options, string msBuildMoniker, RuntimeMoniker moniker) - { - bool wasmAot = options.AOTCompilerMode == MonoAotCompilerMode.wasm; - - var wasmRuntime = new WasmRuntime( - msBuildMoniker: msBuildMoniker, - javaScriptEngine: options.WasmJavascriptEngine?.FullName ?? "v8", - javaScriptEngineArguments: options.WasmJavaScriptEngineArguments, - aot: wasmAot, - wasmDataDir: options.WasmDataDirectory?.FullName, - moniker: moniker); - - var toolChain = WasmToolchain.From(new NetCoreAppSettings( - targetFrameworkMoniker: wasmRuntime.MsBuildMoniker, - runtimeFrameworkVersion: null, - name: wasmRuntime.Name, - customDotNetCliPath: options.CliPath?.FullName, - packagesPath: options.RestorePath?.FullName, - customRuntimePack: options.CustomRuntimePack, - aotCompilerMode: options.AOTCompilerMode)); - - return baseJob.WithRuntime(wasmRuntime).WithToolchain(toolChain); - } - - private static IEnumerable GetFilters(CommandLineOptions options) - { - if (options.Filters.Any()) - yield return new GlobFilter(options.Filters.ToArray()); - if (options.AllCategories.Any()) - yield return new AllCategoriesFilter(options.AllCategories.ToArray()); - if (options.AnyCategories.Any()) - yield return new AnyCategoriesFilter(options.AnyCategories.ToArray()); - if (options.AttributeNames.Any()) - yield return new AttributesFilter(options.AttributeNames.ToArray()); - } - - private static int GetMaximumDisplayWidth() - { - try - { - return Console.WindowWidth; - } - catch (IOException) - { - return MinimumDisplayWidth; - } - } - - private static Job CreateCoreRunJob(Job baseJob, CommandLineOptions options, FileInfo coreRunPath) - => baseJob - .WithToolchain(new CoreRunToolchain( - coreRunPath, - createCopy: true, - targetFrameworkMoniker: - RuntimeInformation.IsNetCore - ? RuntimeInformation.GetCurrentRuntime().MsBuildMoniker - : CoreRuntime.Latest.MsBuildMoniker, // use most recent tfm, as the toolchain is being used only by dotnet/runtime contributors - customDotNetCliPath: options.CliPath, - restorePath: options.RestorePath, - displayName: GetCoreRunToolchainDisplayName(options.CoreRunPaths, coreRunPath))); - - private static Job CreateCoreJobWithCli(Job baseJob, CommandLineOptions options) - => baseJob - .WithToolchain(CsProjCoreToolchain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: RuntimeInformation.GetCurrentRuntime().MsBuildMoniker, - customDotNetCliPath: options.CliPath?.FullName, - runtimeFrameworkVersion: null, - name: RuntimeInformation.GetCurrentRuntime().Name, - packagesPath: options.RestorePath?.FullName))); - - /// - /// we have a limited amount of space when printing the output to the console, so we try to keep things small and simple - /// - /// for following paths: - /// C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root\CoreRun.exe - /// C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root_beforeMyChanges\CoreRun.exe - /// - /// we get: - /// - /// \Core_Root\CoreRun.exe - /// \Core_Root_beforeMyChanges\CoreRun.exe - /// - private static string GetCoreRunToolchainDisplayName(IReadOnlyList paths, FileInfo coreRunPath) - { - if (paths.Count <= 1) - return "CoreRun"; - - int commonLongestPrefixIndex = paths[0].FullName.Length; - for (int i = 1; i < paths.Count; i++) - { - commonLongestPrefixIndex = Math.Min(commonLongestPrefixIndex, paths[i].FullName.Length); - for (int j = 0; j < commonLongestPrefixIndex; j++) - if (paths[i].FullName[j] != paths[0].FullName[j]) - { - commonLongestPrefixIndex = j; - break; - } - } - - if (commonLongestPrefixIndex <= 1) - return coreRunPath.FullName; - - var lastCommonDirectorySeparatorIndex = coreRunPath.FullName.LastIndexOf(Path.DirectorySeparatorChar, commonLongestPrefixIndex - 1); - - return coreRunPath.FullName.Substring(lastCommonDirectorySeparatorIndex); - } - - internal static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker) - { - int index = runtime.IndexOf('-'); - - return index < 0 - ? Enum.TryParse(runtime.Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker) - : Enum.TryParse(runtime.Substring(0, index).Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker); - } - } +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Text; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Exporters.Csv; +using BenchmarkDotNet.Exporters.Json; +using BenchmarkDotNet.Exporters.Xml; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Filters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Toolchains.CoreRun; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using BenchmarkDotNet.Toolchains.MonoAotLLVM; +using BenchmarkDotNet.Toolchains.MonoWasm; +using BenchmarkDotNet.Toolchains.NativeAot; +using CommandLine; +using Perfolizer.Horology; +using Perfolizer.Mathematics.OutlierDetection; +using BenchmarkDotNet.Toolchains.Mono; +using Perfolizer.Metrology; + +namespace BenchmarkDotNet.ConsoleArguments +{ + public static class ConfigParser + { + private const int MinimumDisplayWidth = 80; + private const char EnvVarKeyValueSeparator = ':'; + + private static readonly IReadOnlyDictionary AvailableJobs = new Dictionary(StringComparer.InvariantCultureIgnoreCase) + { + { "default", Job.Default }, + { "dry", Job.Dry }, + { "short", Job.ShortRun }, + { "medium", Job.MediumRun }, + { "long", Job.LongRun }, + { "verylong", Job.VeryLongRun } + }; + + [SuppressMessage("ReSharper", "StringLiteralTypo")] + [SuppressMessage("ReSharper", "CoVariantArrayConversion")] + private static readonly IReadOnlyDictionary AvailableExporters = + new Dictionary(StringComparer.InvariantCultureIgnoreCase) + { + { "csv", new[] { CsvExporter.Default } }, + { "csvmeasurements", new[] { CsvMeasurementsExporter.Default } }, + { "html", new[] { HtmlExporter.Default } }, + { "markdown", new[] { MarkdownExporter.Default } }, + { "atlassian", new[] { MarkdownExporter.Atlassian } }, + { "stackoverflow", new[] { MarkdownExporter.StackOverflow } }, + { "github", new[] { MarkdownExporter.GitHub } }, + { "plain", new[] { PlainExporter.Default } }, + { "rplot", new[] { CsvMeasurementsExporter.Default, RPlotExporter.Default } }, // R Plots depends on having the full measurements available + { "json", new[] { JsonExporter.Default } }, + { "briefjson", new[] { JsonExporter.Brief } }, + { "fulljson", new[] { JsonExporter.Full } }, + { "asciidoc", new[] { AsciiDocExporter.Default } }, + { "xml", new[] { XmlExporter.Default } }, + { "briefxml", new[] { XmlExporter.Brief } }, + { "fullxml", new[] { XmlExporter.Full } } + }; + + public static (bool isSuccess, IConfig config, CommandLineOptions options) Parse(string[] args, ILogger logger, IConfig? globalConfig = null) + { + (bool isSuccess, IConfig config, CommandLineOptions options) result = default; + + var (expandSuccess, expandedArgs) = ExpandResponseFile(args, logger); + if (!expandSuccess) + { + return (false, default, default); + } + + args = expandedArgs; + using (var parser = CreateParser(logger)) + { + parser + .ParseArguments(args) + .WithParsed(options => result = Validate(options, logger) ? (true, CreateConfig(options, globalConfig, args), options) : (false, default, default)) + .WithNotParsed(errors => result = (false, default, default)); + } + + return result; + } + + private static (bool Success, string[] ExpandedTokens) ExpandResponseFile(string[] args, ILogger logger) + { + List result = new(); + foreach (var arg in args) + { + if (arg.StartsWith("@")) + { + var fileName = arg.Substring(1); + try + { + if (File.Exists(fileName)) + { + var lines = File.ReadAllLines(fileName); + foreach (var line in lines) + { + result.AddRange(ConsumeTokens(line)); + } + } + else + { + logger.WriteLineError($"Response file {fileName} does not exists."); + return (false, Array.Empty()); + } + } + catch (Exception ex) + { + logger.WriteLineError($"Failed to parse RSP file: {fileName}, {ex.Message}"); + return (false, Array.Empty()); + } + } + else + { + result.Add(arg); + } + } + + return (true, result.ToArray()); + } + + private static IEnumerable ConsumeTokens(string line) + { + bool insideQuotes = false; + var token = new StringBuilder(); + for (int i = 0; i < line.Length; i++) + { + char currentChar = line[i]; + if (currentChar == ' ' && !insideQuotes) + { + if (token.Length > 0) + { + yield return GetToken(); + token = new StringBuilder(); + } + + continue; + } + + if (currentChar == '"') + { + insideQuotes = !insideQuotes; + continue; + } + + if (currentChar == '\\' && insideQuotes) + { + if (line[i + 1] == '"') + { + insideQuotes = false; + i++; + continue; + } + + if (line[i + 1] == '\\') + { + token.Append('\\'); + i++; + continue; + } + } + + token.Append(currentChar); + } + + if (token.Length > 0) + { + yield return GetToken(); + } + + string GetToken() + { + var result = token.ToString(); + if (result.Contains(' ')) + { + // Workaround for CommandLine library issue with parsing these kind of args. + return " " + result; + } + + return result; + } + } + + internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Action updater) + { + (bool isSuccess, CommandLineOptions options) result = default; + + ILogger logger = NullLogger.Instance; + using (var parser = CreateParser(logger)) + { + parser + .ParseArguments(args) + .WithParsed(options => result = Validate(options, logger) ? (true, options) : (false, default)) + .WithNotParsed(errors => result = (false, default)); + + if (!result.isSuccess) + { + updatedArgs = null; + return false; + } + + updater(result.options); + + updatedArgs = parser.FormatCommandLine(result.options, settings => settings.SkipDefault = true).Split(); + return true; + } + } + + private static Parser CreateParser(ILogger logger) + => new Parser(settings => + { + settings.CaseInsensitiveEnumValues = true; + settings.CaseSensitive = false; + settings.EnableDashDash = true; + settings.IgnoreUnknownArguments = false; + settings.HelpWriter = new LoggerWrapper(logger); + settings.MaximumDisplayWidth = Math.Max(MinimumDisplayWidth, GetMaximumDisplayWidth()); + }); + + private static bool Validate(CommandLineOptions options, ILogger logger) + { + if (!AvailableJobs.ContainsKey(options.BaseJob)) + { + logger.WriteLineError($"The provided base job \"{options.BaseJob}\" is invalid. Available options are: {string.Join(", ", AvailableJobs.Keys)}."); + return false; + } + + foreach (string runtime in options.Runtimes) + { + if (!TryParse(runtime, out RuntimeMoniker runtimeMoniker)) + { + logger.WriteLineError($"The provided runtime \"{runtime}\" is invalid. Available options are: {string.Join(", ", Enum.GetNames(typeof(RuntimeMoniker)).Select(name => name.ToLower()))}."); + return false; + } + else if (runtimeMoniker == RuntimeMoniker.MonoAOTLLVM && (options.AOTCompilerPath == null || options.AOTCompilerPath.IsNotNullButDoesNotExist())) + { + logger.WriteLineError($"The provided {nameof(options.AOTCompilerPath)} \"{options.AOTCompilerPath}\" does NOT exist. It MUST be provided."); + } + } + + foreach (string exporter in options.Exporters) + if (!AvailableExporters.ContainsKey(exporter)) + { + logger.WriteLineError($"The provided exporter \"{exporter}\" is invalid. Available options are: {string.Join(", ", AvailableExporters.Keys)}."); + return false; + } + + if (options.CliPath.IsNotNullButDoesNotExist()) + { + logger.WriteLineError($"The provided {nameof(options.CliPath)} \"{options.CliPath}\" does NOT exist."); + return false; + } + + foreach (var coreRunPath in options.CoreRunPaths) + if (coreRunPath.IsNotNullButDoesNotExist()) + { + if (Directory.Exists(coreRunPath.FullName)) + { + logger.WriteLineError($"The provided path to CoreRun: \"{coreRunPath}\" exists but it's a directory, not an executable. You need to include CoreRun.exe (corerun on Unix) in the path."); + } + else + { + logger.WriteLineError($"The provided path to CoreRun: \"{coreRunPath}\" does NOT exist."); + } + + return false; + } + + if (options.MonoPath.IsNotNullButDoesNotExist()) + { + logger.WriteLineError($"The provided {nameof(options.MonoPath)} \"{options.MonoPath}\" does NOT exist."); + return false; + } + + if (options.WasmJavascriptEngine.IsNotNullButDoesNotExist()) + { + logger.WriteLineError($"The provided {nameof(options.WasmJavascriptEngine)} \"{options.WasmJavascriptEngine}\" does NOT exist."); + return false; + } + + if (options.IlcPackages.IsNotNullButDoesNotExist()) + { + logger.WriteLineError($"The provided {nameof(options.IlcPackages)} \"{options.IlcPackages}\" does NOT exist."); + return false; + } + + if (options.HardwareCounters.Count() > 3) + { + logger.WriteLineError("You can't use more than 3 HardwareCounters at the same time."); + return false; + } + + foreach (var counterName in options.HardwareCounters) + if (!Enum.TryParse(counterName, ignoreCase: true, out HardwareCounter _)) + { + logger.WriteLineError($"The provided hardware counter \"{counterName}\" is invalid. Available options are: {string.Join("+", Enum.GetNames(typeof(HardwareCounter)))}."); + return false; + } + + if (!string.IsNullOrEmpty(options.StatisticalTestThreshold) && !Threshold.TryParse(options.StatisticalTestThreshold, out _)) + { + logger.WriteLineError("Invalid Threshold for Statistical Test. Use --help to see examples."); + return false; + } + + if (options.EnvironmentVariables.Any(envVar => envVar.IndexOf(EnvVarKeyValueSeparator) <= 0)) + { + logger.WriteLineError($"Environment variable value must be separated from the key using '{EnvVarKeyValueSeparator}'. Use --help to see examples."); + return false; + } + + return true; + } + + private static IConfig CreateConfig(CommandLineOptions options, IConfig globalConfig, string[] args) + { + var config = new ManualConfig(); + + var baseJob = GetBaseJob(options, globalConfig); + var expanded = Expand(baseJob.UnfreezeCopy(), options, args).ToArray(); // UnfreezeCopy ensures that each of the expanded jobs will have it's own ID + if (expanded.Length > 1) + expanded[0] = expanded[0].AsBaseline(); // if the user provides multiple jobs, then the first one should be a baseline + config.AddJob(expanded); + if (config.GetJobs().IsEmpty() && baseJob != Job.Default) + config.AddJob(baseJob); + + config.AddExporter(options.Exporters.SelectMany(exporter => AvailableExporters[exporter]).ToArray()); + + config.AddHardwareCounters(options.HardwareCounters + .Select(counterName => (HardwareCounter)Enum.Parse(typeof(HardwareCounter), counterName, ignoreCase: true)) + .ToArray()); + + if (options.UseMemoryDiagnoser) + config.AddDiagnoser(MemoryDiagnoser.Default); + if (options.UseThreadingDiagnoser) + config.AddDiagnoser(ThreadingDiagnoser.Default); + if (options.UseExceptionDiagnoser) + config.AddDiagnoser(ExceptionDiagnoser.Default); + if (options.UseDisassemblyDiagnoser) + config.AddDiagnoser(new DisassemblyDiagnoser(new DisassemblyDiagnoserConfig( + maxDepth: options.DisassemblerRecursiveDepth, + filters: options.DisassemblerFilters.ToArray(), + exportDiff: options.DisassemblerDiff))); + if (!string.IsNullOrEmpty(options.Profiler)) + config.AddDiagnoser(DiagnosersLoader.GetImplementation(profiler => profiler.ShortName.EqualsWithIgnoreCase(options.Profiler))); + + if (options.DisplayAllStatistics) + config.AddColumn(StatisticColumn.AllStatistics); + if (!string.IsNullOrEmpty(options.StatisticalTestThreshold) && Threshold.TryParse(options.StatisticalTestThreshold, out var threshold)) + config.AddColumn(new StatisticalTestColumn(threshold)); + + if (options.ArtifactsDirectory != null) + config.ArtifactsPath = options.ArtifactsDirectory.FullName; + + var filters = GetFilters(options).ToArray(); + if (filters.Length > 1) + config.AddFilter(new UnionFilter(filters)); + else + config.AddFilter(filters); + + config.HideColumns(options.HiddenColumns.ToArray()); + + config.WithOption(ConfigOptions.JoinSummary, options.Join); + config.WithOption(ConfigOptions.KeepBenchmarkFiles, options.KeepBenchmarkFiles); + config.WithOption(ConfigOptions.DontOverwriteResults, options.DontOverwriteResults); + config.WithOption(ConfigOptions.StopOnFirstError, options.StopOnFirstError); + config.WithOption(ConfigOptions.DisableLogFile, options.DisableLogFile); + config.WithOption(ConfigOptions.LogBuildOutput, options.LogBuildOutput); + config.WithOption(ConfigOptions.GenerateMSBuildBinLog, options.GenerateMSBuildBinLog); + config.WithOption(ConfigOptions.ApplesToApples, options.ApplesToApples); + config.WithOption(ConfigOptions.Resume, options.Resume); + + if (config.Options.IsSet(ConfigOptions.GenerateMSBuildBinLog)) + config.Options |= ConfigOptions.KeepBenchmarkFiles; + + if (options.MaxParameterColumnWidth.HasValue) + config.WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(options.MaxParameterColumnWidth.Value)); + + if (options.TimeOutInSeconds.HasValue) + config.WithBuildTimeout(TimeSpan.FromSeconds(options.TimeOutInSeconds.Value)); + + if (options.WakeLock.HasValue) + config.WithWakeLock(options.WakeLock.Value); + + return config; + } + + private static Job GetBaseJob(CommandLineOptions options, IConfig globalConfig) + { + var baseJob = + globalConfig?.GetJobs().SingleOrDefault(job => job.Meta.IsDefault) // global config might define single custom Default job + ?? AvailableJobs[options.BaseJob.ToLowerInvariant()]; + + if (baseJob != Job.Dry && options.Outliers != OutlierMode.RemoveUpper) + baseJob = baseJob.WithOutlierMode(options.Outliers); + + if (options.Affinity.HasValue) + baseJob = baseJob.WithAffinity((IntPtr)options.Affinity.Value); + + if (options.LaunchCount.HasValue) + baseJob = baseJob.WithLaunchCount(options.LaunchCount.Value); + if (options.WarmupIterationCount.HasValue) + baseJob = baseJob.WithWarmupCount(options.WarmupIterationCount.Value); + if (options.MinWarmupIterationCount.HasValue) + baseJob = baseJob.WithMinWarmupCount(options.MinWarmupIterationCount.Value); + if (options.MaxWarmupIterationCount.HasValue) + baseJob = baseJob.WithMaxWarmupCount(options.MaxWarmupIterationCount.Value); + if (options.IterationTimeInMilliseconds.HasValue) + baseJob = baseJob.WithIterationTime(TimeInterval.FromMilliseconds(options.IterationTimeInMilliseconds.Value)); + if (options.IterationCount.HasValue) + baseJob = baseJob.WithIterationCount(options.IterationCount.Value); + if (options.MinIterationCount.HasValue) + baseJob = baseJob.WithMinIterationCount(options.MinIterationCount.Value); + if (options.MaxIterationCount.HasValue) + baseJob = baseJob.WithMaxIterationCount(options.MaxIterationCount.Value); + if (options.InvocationCount.HasValue) + baseJob = baseJob.WithInvocationCount(options.InvocationCount.Value); + if (options.UnrollFactor.HasValue) + baseJob = baseJob.WithUnrollFactor(options.UnrollFactor.Value); + if (options.RunStrategy.HasValue) + baseJob = baseJob.WithStrategy(options.RunStrategy.Value); + if (options.Platform.HasValue) + baseJob = baseJob.WithPlatform(options.Platform.Value); + if (options.RunOncePerIteration) + baseJob = baseJob.RunOncePerIteration(); + if (options.MemoryRandomization) + baseJob = baseJob.WithMemoryRandomization(); + if (options.NoForcedGCs) + baseJob = baseJob.WithGcForce(false); + if (options.NoEvaluationOverhead) + baseJob = baseJob.WithEvaluateOverhead(false); + + if (options.EnvironmentVariables.Any()) + { + baseJob = baseJob.WithEnvironmentVariables(options.EnvironmentVariables.Select(text => + { + var separated = text.Split(new[] { EnvVarKeyValueSeparator }, 2); + return new EnvironmentVariable(separated[0], separated[1]); + }).ToArray()); + } + + if (AvailableJobs.Values.Contains(baseJob)) // no custom settings + return baseJob; + + return baseJob + .AsDefault(false) // after applying all settings from console args the base job is not default anymore + .AsMutator(); // we mark it as mutator so it will be applied to other jobs defined via attributes and merged later in GetRunnableJobs method + } + + private static IEnumerable Expand(Job baseJob, CommandLineOptions options, string[] args) + { + if (options.RunInProcess) + { + yield return baseJob.WithToolchain(InProcessEmitToolchain.Instance); + } + else if (!string.IsNullOrEmpty(options.ClrVersion)) + { + yield return baseJob.WithRuntime(ClrRuntime.CreateForLocalFullNetFrameworkBuild(options.ClrVersion)); // local builds of .NET Runtime + } + else if (options.CliPath != null && options.Runtimes.IsEmpty() && options.CoreRunPaths.IsEmpty()) + { + yield return CreateCoreJobWithCli(baseJob, options); + } + else + { + // in case both --runtimes and --corerun are specified, the first one is returned first and becomes a baseline job + string first = args.FirstOrDefault(arg => + arg.Equals("--runtimes", StringComparison.OrdinalIgnoreCase) + || arg.Equals("-r", StringComparison.OrdinalIgnoreCase) + + || arg.Equals("--corerun", StringComparison.OrdinalIgnoreCase)); + + if (first is null || first.Equals("--corerun", StringComparison.OrdinalIgnoreCase)) + { + foreach (var coreRunPath in options.CoreRunPaths) + yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local dotnet/runtime builds + + foreach (string runtime in options.Runtimes) // known runtimes + yield return CreateJobForGivenRuntime(baseJob, runtime, options); + } + else + { + foreach (string runtime in options.Runtimes) // known runtimes + yield return CreateJobForGivenRuntime(baseJob, runtime, options); + + foreach (var coreRunPath in options.CoreRunPaths) + yield return CreateCoreRunJob(baseJob, options, coreRunPath); // local dotnet/runtime builds + } + } + } + + private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, CommandLineOptions options) + { + if (!TryParse(runtimeId, out RuntimeMoniker runtimeMoniker)) + { + throw new InvalidOperationException("Impossible, already validated by the Validate method"); + } + + switch (runtimeMoniker) + { + case RuntimeMoniker.Net461: + case RuntimeMoniker.Net462: + case RuntimeMoniker.Net47: + case RuntimeMoniker.Net471: + case RuntimeMoniker.Net472: + case RuntimeMoniker.Net48: + case RuntimeMoniker.Net481: + return baseJob + .WithRuntime(runtimeMoniker.GetRuntime()) + .WithToolchain(CsProjClassicNetToolchain.From(runtimeId, options.RestorePath?.FullName, options.CliPath?.FullName)); + + case RuntimeMoniker.NetCoreApp20: + case RuntimeMoniker.NetCoreApp21: + case RuntimeMoniker.NetCoreApp22: + case RuntimeMoniker.NetCoreApp30: + case RuntimeMoniker.NetCoreApp31: +#pragma warning disable CS0618 // Type or member is obsolete + case RuntimeMoniker.NetCoreApp50: +#pragma warning restore CS0618 // Type or member is obsolete + case RuntimeMoniker.Net50: + case RuntimeMoniker.Net60: + case RuntimeMoniker.Net70: + case RuntimeMoniker.Net80: + case RuntimeMoniker.Net90: + case RuntimeMoniker.Net10_0: + return baseJob + .WithRuntime(runtimeMoniker.GetRuntime()) + .WithToolchain(CsProjCoreToolchain.From(new NetCoreAppSettings(runtimeId, null, runtimeId, options.CliPath?.FullName, options.RestorePath?.FullName))); + + case RuntimeMoniker.Mono: + return baseJob.WithRuntime(new MonoRuntime("Mono", options.MonoPath?.FullName)); + + case RuntimeMoniker.NativeAot60: + return CreateAotJob(baseJob, options, runtimeMoniker, "6.0.0-*", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json"); + + case RuntimeMoniker.NativeAot70: + return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://api.nuget.org/v3/index.json"); + + case RuntimeMoniker.NativeAot80: + return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://api.nuget.org/v3/index.json"); + + case RuntimeMoniker.NativeAot90: + return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json"); + + case RuntimeMoniker.NativeAot10_0: + return CreateAotJob(baseJob, options, runtimeMoniker, "", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json"); + + case RuntimeMoniker.Wasm: + return MakeWasmJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net5.0", runtimeMoniker); + + case RuntimeMoniker.WasmNet50: + return MakeWasmJob(baseJob, options, "net5.0", runtimeMoniker); + + case RuntimeMoniker.WasmNet60: + return MakeWasmJob(baseJob, options, "net6.0", runtimeMoniker); + + case RuntimeMoniker.WasmNet70: + return MakeWasmJob(baseJob, options, "net7.0", runtimeMoniker); + + case RuntimeMoniker.WasmNet80: + return MakeWasmJob(baseJob, options, "net8.0", runtimeMoniker); + + case RuntimeMoniker.WasmNet90: + return MakeWasmJob(baseJob, options, "net9.0", runtimeMoniker); + + case RuntimeMoniker.WasmNet10_0: + return MakeWasmJob(baseJob, options, "net10.0", runtimeMoniker); + + case RuntimeMoniker.MonoAOTLLVM: + return MakeMonoAOTLLVMJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net6.0", runtimeMoniker); + + case RuntimeMoniker.MonoAOTLLVMNet60: + return MakeMonoAOTLLVMJob(baseJob, options, "net6.0", runtimeMoniker); + + case RuntimeMoniker.MonoAOTLLVMNet70: + return MakeMonoAOTLLVMJob(baseJob, options, "net7.0", runtimeMoniker); + + case RuntimeMoniker.MonoAOTLLVMNet80: + return MakeMonoAOTLLVMJob(baseJob, options, "net8.0", runtimeMoniker); + + case RuntimeMoniker.MonoAOTLLVMNet90: + return MakeMonoAOTLLVMJob(baseJob, options, "net9.0", runtimeMoniker); + + case RuntimeMoniker.MonoAOTLLVMNet10_0: + return MakeMonoAOTLLVMJob(baseJob, options, "net10.0", runtimeMoniker); + + case RuntimeMoniker.Mono60: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono60); + + case RuntimeMoniker.Mono70: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono70); + + case RuntimeMoniker.Mono80: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono80); + + case RuntimeMoniker.Mono90: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono90); + + case RuntimeMoniker.Mono10_0: + return MakeMonoJob(baseJob, options, MonoRuntime.Mono10_0); + + default: + throw new NotSupportedException($"Runtime {runtimeId} is not supported"); + } + } + + private static Job CreateAotJob(Job baseJob, CommandLineOptions options, RuntimeMoniker runtimeMoniker, string ilCompilerVersion, string nuGetFeedUrl) + { + var builder = NativeAotToolchain.CreateBuilder(); + + if (options.CliPath != null) + builder.DotNetCli(options.CliPath.FullName); + if (options.RestorePath != null) + builder.PackagesRestorePath(options.RestorePath.FullName); + + if (options.IlcPackages != null) + builder.UseLocalBuild(options.IlcPackages); + else if (!string.IsNullOrEmpty(options.ILCompilerVersion)) + builder.UseNuGet(options.ILCompilerVersion, nuGetFeedUrl); + else + builder.UseNuGet(ilCompilerVersion, nuGetFeedUrl); + + var runtime = runtimeMoniker.GetRuntime(); + builder.TargetFrameworkMoniker(runtime.MsBuildMoniker); + + return baseJob.WithRuntime(runtime).WithToolchain(builder.ToToolchain()); + } + + private static Job MakeMonoJob(Job baseJob, CommandLineOptions options, MonoRuntime runtime) + { + return baseJob + .WithRuntime(runtime) + .WithToolchain(MonoToolchain.From( + new NetCoreAppSettings( + targetFrameworkMoniker: runtime.MsBuildMoniker, + runtimeFrameworkVersion: null, + name: runtime.Name, + customDotNetCliPath: options.CliPath?.FullName, + packagesPath: options.RestorePath?.FullName))); + } + + private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, string msBuildMoniker, RuntimeMoniker moniker) + { + var monoAotLLVMRuntime = new MonoAotLLVMRuntime(aotCompilerPath: options.AOTCompilerPath, aotCompilerMode: options.AOTCompilerMode, msBuildMoniker: msBuildMoniker, moniker: moniker); + + var toolChain = MonoAotLLVMToolChain.From( + new NetCoreAppSettings( + targetFrameworkMoniker: monoAotLLVMRuntime.MsBuildMoniker, + runtimeFrameworkVersion: null, + name: monoAotLLVMRuntime.Name, + customDotNetCliPath: options.CliPath?.FullName, + packagesPath: options.RestorePath?.FullName, + customRuntimePack: options.CustomRuntimePack, + aotCompilerPath: options.AOTCompilerPath.ToString(), + aotCompilerMode: options.AOTCompilerMode)); + + return baseJob.WithRuntime(monoAotLLVMRuntime).WithToolchain(toolChain); + } + + private static Job MakeWasmJob(Job baseJob, CommandLineOptions options, string msBuildMoniker, RuntimeMoniker moniker) + { + bool wasmAot = options.AOTCompilerMode == MonoAotCompilerMode.wasm; + + var wasmRuntime = new WasmRuntime( + msBuildMoniker: msBuildMoniker, + javaScriptEngine: options.WasmJavascriptEngine?.FullName ?? "v8", + javaScriptEngineArguments: options.WasmJavaScriptEngineArguments, + aot: wasmAot, + wasmDataDir: options.WasmDataDirectory?.FullName, + moniker: moniker); + + var toolChain = WasmToolchain.From(new NetCoreAppSettings( + targetFrameworkMoniker: wasmRuntime.MsBuildMoniker, + runtimeFrameworkVersion: null, + name: wasmRuntime.Name, + customDotNetCliPath: options.CliPath?.FullName, + packagesPath: options.RestorePath?.FullName, + customRuntimePack: options.CustomRuntimePack, + aotCompilerMode: options.AOTCompilerMode)); + + return baseJob.WithRuntime(wasmRuntime).WithToolchain(toolChain); + } + + private static IEnumerable GetFilters(CommandLineOptions options) + { + if (options.Filters.Any()) + yield return new GlobFilter(options.Filters.ToArray()); + if (options.AllCategories.Any()) + yield return new AllCategoriesFilter(options.AllCategories.ToArray()); + if (options.AnyCategories.Any()) + yield return new AnyCategoriesFilter(options.AnyCategories.ToArray()); + if (options.AttributeNames.Any()) + yield return new AttributesFilter(options.AttributeNames.ToArray()); + } + + private static int GetMaximumDisplayWidth() + { + try + { + return Console.WindowWidth; + } + catch (IOException) + { + return MinimumDisplayWidth; + } + } + + private static Job CreateCoreRunJob(Job baseJob, CommandLineOptions options, FileInfo coreRunPath) + => baseJob + .WithToolchain(new CoreRunToolchain( + coreRunPath, + createCopy: true, + targetFrameworkMoniker: + RuntimeInformation.IsNetCore + ? RuntimeInformation.GetCurrentRuntime().MsBuildMoniker + : CoreRuntime.Latest.MsBuildMoniker, // use most recent tfm, as the toolchain is being used only by dotnet/runtime contributors + customDotNetCliPath: options.CliPath, + restorePath: options.RestorePath, + displayName: GetCoreRunToolchainDisplayName(options.CoreRunPaths, coreRunPath))); + + private static Job CreateCoreJobWithCli(Job baseJob, CommandLineOptions options) + => baseJob + .WithToolchain(CsProjCoreToolchain.From( + new NetCoreAppSettings( + targetFrameworkMoniker: RuntimeInformation.GetCurrentRuntime().MsBuildMoniker, + customDotNetCliPath: options.CliPath?.FullName, + runtimeFrameworkVersion: null, + name: RuntimeInformation.GetCurrentRuntime().Name, + packagesPath: options.RestorePath?.FullName))); + + /// + /// we have a limited amount of space when printing the output to the console, so we try to keep things small and simple + /// + /// for following paths: + /// C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root\CoreRun.exe + /// C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root_beforeMyChanges\CoreRun.exe + /// + /// we get: + /// + /// \Core_Root\CoreRun.exe + /// \Core_Root_beforeMyChanges\CoreRun.exe + /// + private static string GetCoreRunToolchainDisplayName(IReadOnlyList paths, FileInfo coreRunPath) + { + if (paths.Count <= 1) + return "CoreRun"; + + int commonLongestPrefixIndex = paths[0].FullName.Length; + for (int i = 1; i < paths.Count; i++) + { + commonLongestPrefixIndex = Math.Min(commonLongestPrefixIndex, paths[i].FullName.Length); + for (int j = 0; j < commonLongestPrefixIndex; j++) + if (paths[i].FullName[j] != paths[0].FullName[j]) + { + commonLongestPrefixIndex = j; + break; + } + } + + if (commonLongestPrefixIndex <= 1) + return coreRunPath.FullName; + + var lastCommonDirectorySeparatorIndex = coreRunPath.FullName.LastIndexOf(Path.DirectorySeparatorChar, commonLongestPrefixIndex - 1); + + return coreRunPath.FullName.Substring(lastCommonDirectorySeparatorIndex); + } + + internal static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker) + { + int index = runtime.IndexOf('-'); + if (index >= 0) + { + runtime = runtime.Substring(0, index); + } + + // Monikers older than Net 10 don't use any version delimiter, newer monikers use _ delimiter. + if (Enum.TryParse(runtime.Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker)) + { + return true; + } + return Enum.TryParse(runtime.Replace('.', '_'), ignoreCase: true, out runtimeMoniker); + } + } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/HardwareIntrinsics.cs b/src/BenchmarkDotNet/Detectors/Cpu/HardwareIntrinsics.cs similarity index 53% rename from src/BenchmarkDotNet/Portability/Cpu/HardwareIntrinsics.cs rename to src/BenchmarkDotNet/Detectors/Cpu/HardwareIntrinsics.cs index be1d40c9ec..5b2815cc43 100644 --- a/src/BenchmarkDotNet/Portability/Cpu/HardwareIntrinsics.cs +++ b/src/BenchmarkDotNet/Detectors/Cpu/HardwareIntrinsics.cs @@ -2,48 +2,49 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Numerics; -using BenchmarkDotNet.Environments; using System.Text; - +using BenchmarkDotNet.Environments; #if NET6_0_OR_GREATER using System.Runtime.Intrinsics.X86; using System.Runtime.Intrinsics.Arm; #endif -namespace BenchmarkDotNet.Portability.Cpu +namespace BenchmarkDotNet.Detectors.Cpu { + // based on https://github.com/dotnet/runtime/tree/v10.0.0-rc.1.25451.107/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt internal static class HardwareIntrinsics { internal static string GetVectorSize() => Vector.IsHardwareAccelerated ? $"VectorSize={Vector.Count * 8}" : string.Empty; internal static string GetShortInfo() { - if (IsX86Avx512FSupported) - return GetShortAvx512Representation(); - if (IsX86Avx2Supported) - return "AVX2"; - else if (IsX86AvxSupported) - return "AVX"; - else if (IsX86Sse42Supported) - return "SSE4.2"; - else if (IsX86Sse41Supported) - return "SSE4.1"; - else if (IsX86Ssse3Supported) - return "SSSE3"; - else if (IsX86Sse3Supported) - return "SSE3"; - else if (IsX86Sse2Supported) - return "SSE2"; - else if (IsX86SseSupported) - return "SSE"; - else if (IsX86BaseSupported) - return "X86Base"; - else if (IsArmAdvSimdSupported) - return "AdvSIMD"; + if (IsX86BaseSupported) + { + if (IsX86Avx512Supported) + { + return "x86-64-v4"; + } + else if (IsX86Avx2Supported) + { + return "x86-64-v3"; + } + else if (IsX86Sse42Supported) + { + return "x86-64-v2"; + } + else + { + return "x86-64-v1"; + } + } else if (IsArmBaseSupported) - return "ArmBase"; + { + return "armv8.0-a"; + } else + { return GetVectorSize(); // Runtimes prior to .NET Core 3.0 (APIs did not exist so we print non-exact Vector info) + } } internal static string GetFullInfo(Platform platform) @@ -56,32 +57,31 @@ static IEnumerable GetCurrentProcessInstructionSets(Platform platform) { case Platform.X86: case Platform.X64: - - if (IsX86Avx512FSupported) yield return GetShortAvx512Representation(); - else if (IsX86Avx2Supported) yield return "AVX2"; - else if (IsX86AvxSupported) yield return "AVX"; - else if (IsX86Sse42Supported) yield return "SSE4.2"; - else if (IsX86Sse41Supported) yield return "SSE4.1"; - else if (IsX86Ssse3Supported) yield return "SSSE3"; - else if (IsX86Sse3Supported) yield return "SSE3"; - else if (IsX86Sse2Supported) yield return "SSE2"; - else if (IsX86SseSupported) yield return "SSE"; - else if (IsX86BaseSupported) yield return "X86Base"; - - if (IsX86AesSupported) yield return "AES"; - if (IsX86Bmi1Supported) yield return "BMI1"; - if (IsX86Bmi2Supported) yield return "BMI2"; - if (IsX86FmaSupported) yield return "FMA"; - if (IsX86LzcntSupported) yield return "LZCNT"; - if (IsX86PclmulqdqSupported) yield return "PCLMUL"; - if (IsX86PopcntSupported) yield return "POPCNT"; + { + if (IsX86Avx10v2Supported) yield return "AVX10v2"; + if (IsX86Avx10v1Supported) + { + yield return "AVX10v1"; + yield return "AVX512 BF16+FP16"; + } + if (IsX86Avx512v3Supported) yield return "AVX512 BITALG+VBMI2+VNNI+VPOPCNTDQ"; + if (IsX86Avx512v2Supported) yield return "AVX512 IFMA+VBMI"; + if (IsX86Avx512Supported) yield return "AVX512 F+BW+CD+DQ+VL"; + if (IsX86Avx2Supported) yield return "AVX2+BMI1+BMI2+F16C+FMA+LZCNT+MOVBE"; + if (IsX86AvxSupported) yield return "AVX"; + if (IsX86Sse42Supported) yield return "SSE3+SSSE3+SSE4.1+SSE4.2+POPCNT"; + if (IsX86BaseSupported) yield return "X86Base+SSE+SSE2"; + if (IsX86AesSupported) yield return "AES+PCLMUL"; if (IsX86AvxVnniSupported) yield return "AvxVnni"; if (IsX86SerializeSupported) yield return "SERIALIZE"; - // TODO: Add MOVBE when API is added. break; + } case Platform.Arm64: - if (IsArmAdvSimdSupported) yield return "AdvSIMD"; - else if (IsArmBaseSupported) yield return "ArmBase"; + { + if (IsArmBaseSupported) + { + yield return "ArmBase+AdvSimd"; + } if (IsArmAesSupported) yield return "AES"; if (IsArmCrc32Supported) yield return "CRC32"; @@ -90,71 +90,39 @@ static IEnumerable GetCurrentProcessInstructionSets(Platform platform) if (IsArmSha1Supported) yield return "SHA1"; if (IsArmSha256Supported) yield return "SHA256"; break; + } + default: yield break; } } } - private static string GetShortAvx512Representation() - { - StringBuilder avx512 = new ("AVX-512F"); - if (IsX86Avx512CDSupported) avx512.Append("+CD"); - if (IsX86Avx512BWSupported) avx512.Append("+BW"); - if (IsX86Avx512DQSupported) avx512.Append("+DQ"); - if (IsX86Avx512FVLSupported) avx512.Append("+VL"); - if (IsX86Avx512VbmiSupported) avx512.Append("+VBMI"); - - return avx512.ToString(); - } - +#pragma warning disable CA2252 // Some APIs require opting into preview features internal static bool IsX86BaseSupported => #if NET6_0_OR_GREATER - X86Base.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.X86Base"); -#endif - - internal static bool IsX86SseSupported => -#if NET6_0_OR_GREATER - Sse.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Sse"); -#endif - - internal static bool IsX86Sse2Supported => -#if NET6_0_OR_GREATER + X86Base.IsSupported && + Sse.IsSupported && Sse2.IsSupported; #elif NETSTANDARD + GetIsSupported("System.Runtime.Intrinsics.X86.X86Base") && + GetIsSupported("System.Runtime.Intrinsics.X86.Sse") && GetIsSupported("System.Runtime.Intrinsics.X86.Sse2"); #endif - internal static bool IsX86Sse3Supported => -#if NET6_0_OR_GREATER - Sse3.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Sse3"); -#endif - - internal static bool IsX86Ssse3Supported => -#if NET6_0_OR_GREATER - Ssse3.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Ssse3"); -#endif - - internal static bool IsX86Sse41Supported => -#if NET6_0_OR_GREATER - Sse41.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Sse41"); -#endif - internal static bool IsX86Sse42Supported => #if NET6_0_OR_GREATER - Sse42.IsSupported; + Sse3.IsSupported && + Ssse3.IsSupported && + Sse41.IsSupported && + Sse42.IsSupported && + Popcnt.IsSupported; #elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Sse42"); + GetIsSupported("System.Runtime.Intrinsics.X86.Sse3") && + GetIsSupported("System.Runtime.Intrinsics.X86.Ssse3") && + GetIsSupported("System.Runtime.Intrinsics.X86.Sse41") && + GetIsSupported("System.Runtime.Intrinsics.X86.Sse42") && + GetIsSupported("System.Runtime.Intrinsics.X86.Popcnt"); #endif internal static bool IsX86AvxSupported => @@ -166,107 +134,88 @@ private static string GetShortAvx512Representation() internal static bool IsX86Avx2Supported => #if NET6_0_OR_GREATER - Avx2.IsSupported; + Avx2.IsSupported && + Bmi1.IsSupported && + Bmi2.IsSupported && + Fma.IsSupported && + Lzcnt.IsSupported; #elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Avx2"); -#endif - - internal static bool IsX86Avx512FSupported => -#if NET8_0_OR_GREATER - Avx512F.IsSupported; -#else - GetIsSupported("System.Runtime.Intrinsics.X86.Avx512F"); + GetIsSupported("System.Runtime.Intrinsics.X86.Avx2") && + GetIsSupported("System.Runtime.Intrinsics.X86.Bmi1") && + GetIsSupported("System.Runtime.Intrinsics.X86.Bmi2") && + GetIsSupported("System.Runtime.Intrinsics.X86.Fma") && + GetIsSupported("System.Runtime.Intrinsics.X86.Lzcnt"); #endif - internal static bool IsX86Avx512FVLSupported => + internal static bool IsX86Avx512Supported => #if NET8_0_OR_GREATER - Avx512F.VL.IsSupported; + Avx512F.IsSupported && + Avx512F.VL.IsSupported && + Avx512BW.IsSupported && + Avx512BW.VL.IsSupported && + Avx512CD.IsSupported && + Avx512CD.VL.IsSupported && + Avx512DQ.IsSupported && + Avx512DQ.VL.IsSupported; #else - GetIsSupported("System.Runtime.Intrinsics.X86.Avx512F+VL"); + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512F") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512F+VL") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512BW") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512BW+VL") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512CD") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512CD+VL") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512DQ") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512DQ+VL"); #endif - internal static bool IsX86Avx512BWSupported => + internal static bool IsX86Avx512v2Supported => #if NET8_0_OR_GREATER - Avx512BW.IsSupported; + Avx512Vbmi.IsSupported && + Avx512Vbmi.VL.IsSupported; #else - GetIsSupported("System.Runtime.Intrinsics.X86.Avx512BW"); + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512Vbmi") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512Vbmi+VL"); #endif - internal static bool IsX86Avx512CDSupported => -#if NET8_0_OR_GREATER - Avx512CD.IsSupported; + internal static bool IsX86Avx512v3Supported => +#if NET10_0_OR_GREATER + Avx512Vbmi2.IsSupported && + Avx512Vbmi2.VL.IsSupported; #else - GetIsSupported("System.Runtime.Intrinsics.X86.Avx512CD"); + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512Vbmi2") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx512Vbmi2+VL"); #endif - internal static bool IsX86Avx512DQSupported => -#if NET8_0_OR_GREATER - Avx512DQ.IsSupported; + internal static bool IsX86Avx10v1Supported => +#if NET9_0_OR_GREATER + Avx10v1.IsSupported && + Avx10v1.V512.IsSupported; #else - GetIsSupported("System.Runtime.Intrinsics.X86.Avx512DQ"); + GetIsSupported("System.Runtime.Intrinsics.X86.Avx10v1") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx10v1+V512"); #endif - internal static bool IsX86Avx512VbmiSupported => -#if NET8_0_OR_GREATER - Avx512Vbmi.IsSupported; + internal static bool IsX86Avx10v2Supported => +#if NET10_0_OR_GREATER + Avx10v2.IsSupported && + Avx10v2.V512.IsSupported; #else - GetIsSupported("System.Runtime.Intrinsics.X86.Avx512Vbmi"); + GetIsSupported("System.Runtime.Intrinsics.X86.Avx10v2") && + GetIsSupported("System.Runtime.Intrinsics.X86.Avx10v2+V512"); #endif internal static bool IsX86AesSupported => #if NET6_0_OR_GREATER - System.Runtime.Intrinsics.X86.Aes.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Aes"); -#endif - - internal static bool IsX86Bmi1Supported => -#if NET6_0_OR_GREATER - Bmi1.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Bmi1"); -#endif - - internal static bool IsX86Bmi2Supported => -#if NET6_0_OR_GREATER - Bmi2.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Bmi2"); -#endif - - internal static bool IsX86FmaSupported => -#if NET6_0_OR_GREATER - Fma.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Fma"); -#endif - - internal static bool IsX86LzcntSupported => -#if NET6_0_OR_GREATER - Lzcnt.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Lzcnt"); -#endif - - internal static bool IsX86PclmulqdqSupported => -#if NET6_0_OR_GREATER + System.Runtime.Intrinsics.X86.Aes.IsSupported && Pclmulqdq.IsSupported; #elif NETSTANDARD + GetIsSupported("System.Runtime.Intrinsics.X86.Aes") && GetIsSupported("System.Runtime.Intrinsics.X86.Pclmulqdq"); #endif - internal static bool IsX86PopcntSupported => -#if NET6_0_OR_GREATER - Popcnt.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.X86.Popcnt"); -#endif - internal static bool IsX86AvxVnniSupported => #if NET6_0_OR_GREATER -#pragma warning disable CA2252 // This API requires opting into preview features AvxVnni.IsSupported; -#pragma warning restore CA2252 // This API requires opting into preview features #elif NETSTANDARD GetIsSupported("System.Runtime.Intrinsics.X86.AvxVnni"); #endif @@ -280,15 +229,10 @@ private static string GetShortAvx512Representation() internal static bool IsArmBaseSupported => #if NET6_0_OR_GREATER - ArmBase.IsSupported; -#elif NETSTANDARD - GetIsSupported("System.Runtime.Intrinsics.Arm.ArmBase"); -#endif - - internal static bool IsArmAdvSimdSupported => -#if NET6_0_OR_GREATER + ArmBase.IsSupported && AdvSimd.IsSupported; #elif NETSTANDARD + GetIsSupported("System.Runtime.Intrinsics.Arm.ArmBase") && GetIsSupported("System.Runtime.Intrinsics.Arm.AdvSimd"); #endif @@ -333,6 +277,7 @@ private static string GetShortAvx512Representation() #elif NETSTANDARD GetIsSupported("System.Runtime.Intrinsics.Arm.Sha256"); #endif +#pragma warning restore CA2252 // Some APIs require opting into preview features private static bool GetIsSupported([DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)] string typeName) { diff --git a/src/BenchmarkDotNet/Detectors/Cpu/ICpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/ICpuDetector.cs new file mode 100644 index 0000000000..72355b60e5 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/ICpuDetector.cs @@ -0,0 +1,12 @@ +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu; + +/// +/// Loads the for the current hardware +/// +public interface ICpuDetector +{ + bool IsApplicable(); + CpuInfo? Detect(); +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuDetector.cs new file mode 100644 index 0000000000..44973b27e9 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuDetector.cs @@ -0,0 +1,35 @@ +using System.Collections.Generic; +using BenchmarkDotNet.Helpers; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.Linux; + +/// +/// CPU information from output of the `cat /proc/cpuinfo` and `lscpu` command. +/// Linux only. +/// +internal class LinuxCpuDetector : ICpuDetector +{ + public bool IsApplicable() => OsDetector.IsLinux(); + + public CpuInfo? Detect() + { + if (!IsApplicable()) return null; + + // lscpu output respects the system locale, so we should force language invariant environment for correct parsing + var languageInvariantEnvironment = new Dictionary + { + ["LC_ALL"] = "C", + ["LANG"] = "C", + ["LANGUAGE"] = "C" + }; + + string? cpuInfo = ProcessHelper.RunAndReadOutput("cat", "/proc/cpuinfo") ?? string.Empty; + string? lscpu = ProcessHelper.RunAndReadOutput("/bin/bash", "-c \"lscpu\"", environmentVariables: languageInvariantEnvironment) ?? string.Empty; + + if (cpuInfo == string.Empty && lscpu == string.Empty) + return null; + + return LinuxCpuInfoParser.Parse(cpuInfo, lscpu); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuInfoParser.cs b/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuInfoParser.cs new file mode 100644 index 0000000000..7d79c3262f --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Linux/LinuxCpuInfoParser.cs @@ -0,0 +1,160 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Portability; +using Perfolizer.Horology; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.Linux; + +internal static class LinuxCpuInfoParser +{ + private static class ProcCpu + { + internal const string PhysicalId = "physical id"; + internal const string CpuCores = "cpu cores"; + internal const string ModelName = "model name"; + internal const string MaxFrequency = "max freq"; + internal const string NominalFrequencyBackup = "nominal freq"; + internal const string NominalFrequency = "cpu MHz"; + } + + private static class Lscpu + { + internal const string MaxFrequency = "CPU max MHz"; + internal const string ModelName = "Model name"; + internal const string CoresPerSocket = "Core(s) per socket"; + } + + /// Output of `cat /proc/cpuinfo` + /// Output of `lscpu` + internal static CpuInfo Parse(string cpuInfo, string lscpu) + { + var processorModelNames = new HashSet(); + var processorsToPhysicalCoreCount = new Dictionary(); + int logicalCoreCount = 0; + double maxFrequency = 0.0; + double nominalFrequency = 0.0; + + var logicalCores = SectionsHelper.ParseSections(cpuInfo, ':'); + foreach (var logicalCore in logicalCores) + { + if (logicalCore.TryGetValue(ProcCpu.PhysicalId, out string physicalId) && + logicalCore.TryGetValue(ProcCpu.CpuCores, out string cpuCoresValue) && + int.TryParse(cpuCoresValue, out int cpuCoreCount) && + cpuCoreCount > 0) + processorsToPhysicalCoreCount[physicalId] = cpuCoreCount; + + if (logicalCore.TryGetValue(ProcCpu.ModelName, out string modelName)) + { + processorModelNames.Add(modelName); + logicalCoreCount++; + } + + if (logicalCore.TryGetValue(ProcCpu.MaxFrequency, out string maxCpuFreqValue) && + Frequency.TryParseMHz(maxCpuFreqValue.Replace(',', '.'), out Frequency maxCpuFreq) + && maxCpuFreq > 0) + { + maxFrequency = Math.Max(maxFrequency, maxCpuFreq.ToMHz()); + } + + bool nominalFrequencyHasValue = logicalCore.TryGetValue(ProcCpu.NominalFrequency, out string nominalFreqValue); + bool nominalFrequencyBackupHasValue = logicalCore.TryGetValue(ProcCpu.NominalFrequencyBackup, out string nominalFreqBackupValue); + + double nominalCpuFreq = 0.0; + double nominalCpuBackupFreq = 0.0; + + if (nominalFrequencyHasValue && + double.TryParse(nominalFreqValue, out nominalCpuFreq) + && nominalCpuFreq > 0) + { + nominalCpuFreq = nominalFrequency == 0 ? nominalCpuFreq : Math.Min(nominalFrequency, nominalCpuFreq); + } + if (nominalFrequencyBackupHasValue && + double.TryParse(nominalFreqBackupValue, out nominalCpuBackupFreq) + && nominalCpuBackupFreq > 0) + { + nominalCpuBackupFreq = nominalFrequency == 0 ? nominalCpuBackupFreq : Math.Min(nominalFrequency, nominalCpuBackupFreq); + } + + if (nominalFrequencyHasValue && nominalFrequencyBackupHasValue) + { + nominalFrequency = Math.Min(nominalCpuFreq, nominalCpuBackupFreq); + } + else + { + nominalFrequency = nominalCpuFreq == 0.0 ? nominalCpuBackupFreq : nominalCpuFreq; + } + } + + int? coresPerSocket = null; + if (string.IsNullOrEmpty(lscpu) == false) + { + var lscpuParts = lscpu.Split('\n') + .Where(line => line.Contains(':')) + .SelectMany(line => line.Split([':'], 2)) + .ToList(); + for (int i = 0; i + 1 < lscpuParts.Count; i += 2) + { + string name = lscpuParts[i].Trim(); + string value = lscpuParts[i + 1].Trim(); + + if (name.EqualsWithIgnoreCase(Lscpu.MaxFrequency) && + Frequency.TryParseMHz(value.Replace(',', '.'), out Frequency maxFrequencyParsed)) // Example: `CPU max MHz: 3200,0000` + maxFrequency = Math.Max(maxFrequency, maxFrequencyParsed.ToMHz()); + + if (name.EqualsWithIgnoreCase(Lscpu.ModelName)) + processorModelNames.Add(value); + + if (name.EqualsWithIgnoreCase(Lscpu.CoresPerSocket) && + int.TryParse(value, out int coreCount)) + coresPerSocket = coreCount; + } + } + + string processorName = processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null; + int? physicalProcessorCount = processorsToPhysicalCoreCount.Count > 0 ? processorsToPhysicalCoreCount.Count : null; + int? physicalCoreCount = processorsToPhysicalCoreCount.Count > 0 ? processorsToPhysicalCoreCount.Values.Sum() : coresPerSocket; + + Frequency? maxFrequencyActual = maxFrequency > 0 && physicalProcessorCount > 0 + ? Frequency.FromMHz(maxFrequency) : null; + + Frequency? nominalFrequencyActual = nominalFrequency > 0 && physicalProcessorCount > 0 + ? Frequency.FromMHz(nominalFrequency) : null; + + if (nominalFrequencyActual is null) + { + bool nominalFrequencyInBrandString = processorModelNames.Any(x => ParseFrequencyFromBrandString(x) is not null); + + if (nominalFrequencyInBrandString) + nominalFrequencyActual = processorModelNames.Select(x => ParseFrequencyFromBrandString(x)) + .First(x => x is not null); + } + + return new CpuInfo + { + ProcessorName = processorName, + PhysicalProcessorCount = physicalProcessorCount, + PhysicalCoreCount = physicalCoreCount, + LogicalCoreCount = logicalCoreCount > 0 ? logicalCoreCount : null, + NominalFrequencyHz = nominalFrequencyActual?.Hertz.RoundToLong(), + MaxFrequencyHz = maxFrequencyActual?.Hertz.RoundToLong() + }; + } + + internal static Frequency? ParseFrequencyFromBrandString(string brandString) + { + const string pattern = "(\\d.\\d+)GHz"; + var matches = Regex.Matches(brandString, pattern, RegexOptions.IgnoreCase); + if (matches.Count > 0 && matches[0].Groups.Count > 1) + { + string match = Regex.Matches(brandString, pattern, RegexOptions.IgnoreCase)[0].Groups[1].ToString(); + return Frequency.TryParseGHz(match, out Frequency result) ? result : null; + } + + return null; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/MosCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/MosCpuDetector.cs new file mode 100644 index 0000000000..edc48ba25d --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/MosCpuDetector.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Management; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Portability; +using Perfolizer.Horology; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +internal class MosCpuDetector : ICpuDetector +{ +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatform("windows")] +#endif + public bool IsApplicable() => OsDetector.IsWindows() && + RuntimeInformation.IsFullFramework && + !RuntimeInformation.IsMono; + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatform("windows")] +#endif + public CpuInfo? Detect() + { + if (!IsApplicable()) return null; + + var processorModelNames = new HashSet(); + int physicalCoreCount = 0; + int logicalCoreCount = 0; + int processorsCount = 0; + double maxFrequency = 0; + double nominalFrequency = 0; + + using (var mosProcessor = new ManagementObjectSearcher("SELECT * FROM Win32_Processor")) + { + foreach (var moProcessor in mosProcessor.Get().Cast()) + { + string name = moProcessor[WmicCpuInfoKeyNames.Name]?.ToString(); + if (!string.IsNullOrEmpty(name)) + { + processorModelNames.Add(name); + processorsCount++; + physicalCoreCount += (int)(uint)moProcessor[WmicCpuInfoKeyNames.NumberOfCores]; + logicalCoreCount += (int)(uint)moProcessor[WmicCpuInfoKeyNames.NumberOfLogicalProcessors]; + double tempMaxFrequency = (uint)moProcessor[WmicCpuInfoKeyNames.MaxClockSpeed]; + + if (tempMaxFrequency > 0) + { + nominalFrequency = nominalFrequency == 0 ? tempMaxFrequency : Math.Min(nominalFrequency, tempMaxFrequency); + } + maxFrequency = Math.Max(maxFrequency, tempMaxFrequency); + } + } + } + + string processorName = processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null; + Frequency? maxFrequencyActual = maxFrequency > 0 && processorsCount > 0 + ? Frequency.FromMHz(maxFrequency) + : null; + Frequency? nominalFrequencyActual = nominalFrequency > 0 && processorsCount > 0 + ? Frequency.FromMHz(nominalFrequency) + : null; + + return new CpuInfo + { + ProcessorName = processorName, + PhysicalProcessorCount = processorsCount > 0 ? processorsCount : null, + PhysicalCoreCount = physicalCoreCount > 0 ? physicalCoreCount : null, + LogicalCoreCount = logicalCoreCount > 0 ? logicalCoreCount : null, + NominalFrequencyHz = nominalFrequencyActual?.Hertz.RoundToLong(), + MaxFrequencyHz = maxFrequencyActual?.Hertz.RoundToLong() + }; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/PowershellWmiCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/PowershellWmiCpuDetector.cs new file mode 100644 index 0000000000..7f61c1b8fc --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/PowershellWmiCpuDetector.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.Versioning; +using System.Text.RegularExpressions; +using BenchmarkDotNet.Helpers; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +/// +/// CPU information from output of the `wmic cpu get Name, NumberOfCores, NumberOfLogicalProcessors /Format:List` command. +/// Windows only. +/// +internal class PowershellWmiCpuDetector : ICpuDetector +{ + private readonly string windowsPowershellPath = + $"{Environment.SystemDirectory}{Path.DirectorySeparatorChar}WindowsPowerShell{Path.DirectorySeparatorChar}" + + $"v1.0{Path.DirectorySeparatorChar}powershell.exe"; + + public bool IsApplicable() => OsDetector.IsWindows(); + + #if NET6_0_OR_GREATER + [SupportedOSPlatform("windows")] + #endif + public CpuInfo? Detect() + { + if (!IsApplicable()) return null; + + const string argList = $"{WmicCpuInfoKeyNames.Name}, " + + $"{WmicCpuInfoKeyNames.NumberOfCores}, " + + $"{WmicCpuInfoKeyNames.NumberOfLogicalProcessors}, " + + $"{WmicCpuInfoKeyNames.MaxClockSpeed}"; + + string output = ProcessHelper.RunAndReadOutput(PowerShellLocator.LocateOnWindows() ?? "PowerShell", + "Get-CimInstance Win32_Processor -Property " + argList); + + if (string.IsNullOrEmpty(output)) + return null; + + return PowershellWmiCpuInfoParser.Parse(output); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/PowershellWmiCpuInfoParser.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/PowershellWmiCpuInfoParser.cs new file mode 100644 index 0000000000..15ad4d6c77 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/PowershellWmiCpuInfoParser.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using Perfolizer.Horology; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +internal static class PowershellWmiCpuInfoParser +{ + internal static CpuInfo Parse(string powershellWmiOutput) + { + HashSet processorModelNames = new HashSet(); + + int physicalCoreCount = 0; + int logicalCoreCount = 0; + int processorCount = 0; + double maxFrequency = 0.0; + double nominalFrequency = 0.0; + + List> processors = SectionsHelper.ParseSectionsForPowershellWmi(powershellWmiOutput, ':'); + foreach (Dictionary processor in processors) + { + if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfCores, out string numberOfCoresValue) && + int.TryParse(numberOfCoresValue, out int numberOfCores) && + numberOfCores > 0) + physicalCoreCount += numberOfCores; + + if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfLogicalProcessors, out string numberOfLogicalValue) && + int.TryParse(numberOfLogicalValue, out int numberOfLogical) && + numberOfLogical > 0) + logicalCoreCount += numberOfLogical; + + if (processor.TryGetValue(WmicCpuInfoKeyNames.Name, out string name)) + { + processorModelNames.Add(name); + processorCount++; + } + + if (processor.TryGetValue(WmicCpuInfoKeyNames.MaxClockSpeed, out string frequencyValue) + && double.TryParse(frequencyValue, out double frequency) + && frequency > 0) + { + nominalFrequency = nominalFrequency == 0 ? frequency : Math.Min(nominalFrequency, frequency); + maxFrequency = Math.Max(maxFrequency, frequency); + } + } + + string? processorName = processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null; + Frequency? maxFrequencyActual = maxFrequency > 0 && processorCount > 0 + ? Frequency.FromMHz(maxFrequency) : null; + + Frequency? nominalFrequencyActual = nominalFrequency > 0 && processorCount > 0 ? + Frequency.FromMHz(nominalFrequency) : null; + + return new CpuInfo + { + ProcessorName = processorName, + PhysicalProcessorCount = processorCount > 0 ? processorCount : null, + PhysicalCoreCount = physicalCoreCount > 0 ? physicalCoreCount : null, + LogicalCoreCount = logicalCoreCount > 0 ? logicalCoreCount : null, + NominalFrequencyHz = nominalFrequencyActual?.Hertz.RoundToLong(), + MaxFrequencyHz = maxFrequencyActual?.Hertz.RoundToLong() + }; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/WindowsCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WindowsCpuDetector.cs new file mode 100644 index 0000000000..1de238b93f --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WindowsCpuDetector.cs @@ -0,0 +1,4 @@ +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +internal class WindowsCpuDetector() : CpuDetector(new MosCpuDetector(), new PowershellWmiCpuDetector(), + new WmicCpuDetector()); \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuDetector.cs new file mode 100644 index 0000000000..fe3434ae7f --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuDetector.cs @@ -0,0 +1,36 @@ +using System.IO; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Portability; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +/// +/// CPU information from output of the `wmic cpu get Name, NumberOfCores, NumberOfLogicalProcessors /Format:List` command. +/// Windows only. +/// +/// WMIC is deprecated by Microsoft starting with Windows 10 21H1 (including Windows Server), and it is not known whether it still ships with Windows by default. +/// WMIC may be removed in a future version of Windows. See +internal class WmicCpuDetector : ICpuDetector +{ + private const string DefaultWmicPath = @"C:\Windows\System32\wbem\WMIC.exe"; + + public bool IsApplicable() => OsDetector.IsWindows(); + + public CpuInfo? Detect() + { + if (!IsApplicable()) return null; + + const string argList = $"{WmicCpuInfoKeyNames.Name}, " + + $"{WmicCpuInfoKeyNames.NumberOfCores}, " + + $"{WmicCpuInfoKeyNames.NumberOfLogicalProcessors}, " + + $"{WmicCpuInfoKeyNames.MaxClockSpeed}"; + string wmicPath = File.Exists(DefaultWmicPath) ? DefaultWmicPath : "wmic"; + string? wmicOutput = ProcessHelper.RunAndReadOutput(wmicPath, $"cpu get {argList} /Format:List"); + + if (string.IsNullOrEmpty(wmicOutput)) + return null; + + return WmicCpuInfoParser.Parse(wmicOutput); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoKeyNames.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoKeyNames.cs new file mode 100644 index 0000000000..65d8ecf7d8 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoKeyNames.cs @@ -0,0 +1,9 @@ +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +internal static class WmicCpuInfoKeyNames +{ + internal const string NumberOfLogicalProcessors = "NumberOfLogicalProcessors"; + internal const string NumberOfCores = "NumberOfCores"; + internal const string Name = "Name"; + internal const string MaxClockSpeed = "MaxClockSpeed"; +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoParser.cs b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoParser.cs new file mode 100644 index 0000000000..295d76262e --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/Windows/WmicCpuInfoParser.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using Perfolizer.Horology; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.Windows; + +internal static class WmicCpuInfoParser +{ + /// + /// Parses wmic output and returns + /// + /// Output of `wmic cpu get Name, NumberOfCores, NumberOfLogicalProcessors /Format:List` + internal static CpuInfo Parse(string wmicOutput) + { + HashSet processorModelNames = new HashSet(); + int physicalCoreCount = 0; + int logicalCoreCount = 0; + int processorsCount = 0; + double maxFrequency = 0.0; + double nominalFrequency = 0.0; + + List> processors = SectionsHelper.ParseSections(wmicOutput, '='); + foreach (var processor in processors) + { + if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfCores, out string numberOfCoresValue) && + int.TryParse(numberOfCoresValue, out int numberOfCores) && + numberOfCores > 0) + physicalCoreCount += numberOfCores; + + if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfLogicalProcessors, out string numberOfLogicalValue) && + int.TryParse(numberOfLogicalValue, out int numberOfLogical) && + numberOfLogical > 0) + logicalCoreCount += numberOfLogical; + + if (processor.TryGetValue(WmicCpuInfoKeyNames.Name, out string name)) + { + processorModelNames.Add(name); + processorsCount++; + } + + if (processor.TryGetValue(WmicCpuInfoKeyNames.MaxClockSpeed, out string frequencyValue) + && double.TryParse(frequencyValue, out double frequency) + && frequency > 0) + { + nominalFrequency = nominalFrequency == 0 ? frequency : Math.Min(nominalFrequency, frequency); + maxFrequency = Math.Max(maxFrequency, frequency); + } + } + + string? processorName = processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null; + Frequency? maxFrequencyActual = maxFrequency > 0 && processorsCount > 0 + ? Frequency.FromMHz(maxFrequency) + : null; + + Frequency? nominalFrequencyActual = nominalFrequency > 0 && processorsCount > 0 ? Frequency.FromMHz(nominalFrequency) : null; + + return new CpuInfo + { + ProcessorName = processorName, + PhysicalProcessorCount = processorsCount > 0 ? processorsCount : null, + PhysicalCoreCount = physicalCoreCount > 0 ? physicalCoreCount : null, + LogicalCoreCount = logicalCoreCount > 0 ? logicalCoreCount : null, + NominalFrequencyHz = nominalFrequencyActual?.Hertz.RoundToLong(), + MaxFrequencyHz = maxFrequencyActual?.Hertz.RoundToLong() + }; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/macOS/MacOsCpuDetector.cs b/src/BenchmarkDotNet/Detectors/Cpu/macOS/MacOsCpuDetector.cs new file mode 100644 index 0000000000..2f540e82d3 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/macOS/MacOsCpuDetector.cs @@ -0,0 +1,25 @@ +using BenchmarkDotNet.Helpers; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.macOS; + +/// +/// CPU information from output of the `sysctl -a` command. +/// MacOSX only. +/// +internal class MacOsCpuDetector : ICpuDetector +{ + public bool IsApplicable() => OsDetector.IsMacOS(); + + public CpuInfo? Detect() + { + if (!IsApplicable()) return null; + + string? sysctlOutput = ProcessHelper.RunAndReadOutput("sysctl", "-a"); + + if (sysctlOutput is null) + return null; + + return SysctlCpuInfoParser.Parse(sysctlOutput); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/Cpu/macOS/SysctlCpuInfoParser.cs b/src/BenchmarkDotNet/Detectors/Cpu/macOS/SysctlCpuInfoParser.cs new file mode 100644 index 0000000000..5cd4d11c8a --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/Cpu/macOS/SysctlCpuInfoParser.cs @@ -0,0 +1,60 @@ +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors.Cpu.macOS; + +internal static class SysctlCpuInfoParser +{ + private static class Sysctl + { + internal const string ProcessorName = "machdep.cpu.brand_string"; + internal const string PhysicalProcessorCount = "hw.packages"; + internal const string PhysicalCoreCount = "hw.physicalcpu"; + internal const string LogicalCoreCount = "hw.logicalcpu"; + internal const string NominalFrequency = "hw.cpufrequency"; + internal const string MaxFrequency = "hw.cpufrequency_max"; + } + + /// Output of `sysctl -a` + [SuppressMessage("ReSharper", "StringLiteralTypo")] + internal static CpuInfo Parse(string sysctlOutput) + { + var sysctl = SectionsHelper.ParseSection(sysctlOutput, ':'); + string processorName = sysctl.GetValueOrDefault(Sysctl.ProcessorName); + int? physicalProcessorCount = PositiveIntValue(sysctl, Sysctl.PhysicalProcessorCount); + int? physicalCoreCount = PositiveIntValue(sysctl, Sysctl.PhysicalCoreCount); + int? logicalCoreCount = PositiveIntValue(sysctl, Sysctl.LogicalCoreCount); + long? nominalFrequency = PositiveLongValue(sysctl, Sysctl.NominalFrequency); + long? maxFrequency = PositiveLongValue(sysctl, Sysctl.MaxFrequency); + return new CpuInfo + { + ProcessorName = processorName, + PhysicalProcessorCount = physicalProcessorCount, + PhysicalCoreCount = physicalCoreCount, + LogicalCoreCount = logicalCoreCount, + NominalFrequencyHz = nominalFrequency, + MaxFrequencyHz = maxFrequency + }; + } + + private static int? PositiveIntValue(Dictionary sysctl, string keyName) + { + if (sysctl.TryGetValue(keyName, out string value) && + int.TryParse(value, out int result) && + result > 0) + return result; + return null; + } + + private static long? PositiveLongValue(Dictionary sysctl, string keyName) + { + if (sysctl.TryGetValue(keyName, out string value) && + long.TryParse(value, out long result) && + result > 0) + return result; + return null; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/CpuDetector.cs b/src/BenchmarkDotNet/Detectors/CpuDetector.cs new file mode 100644 index 0000000000..0cb583a95a --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/CpuDetector.cs @@ -0,0 +1,29 @@ +using System; +using System.Linq; +using BenchmarkDotNet.Detectors.Cpu; +using BenchmarkDotNet.Detectors.Cpu.Linux; +using BenchmarkDotNet.Detectors.Cpu.macOS; +using BenchmarkDotNet.Detectors.Cpu.Windows; +using BenchmarkDotNet.Extensions; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Detectors; + +public class CpuDetector(params ICpuDetector[] detectors) : ICpuDetector +{ + public static CpuDetector CrossPlatform => new( + new WindowsCpuDetector(), + new LinuxCpuDetector(), + new MacOsCpuDetector()); + + private static readonly Lazy LazyCpu = new(() => CrossPlatform.Detect()); + public static CpuInfo? Cpu => LazyCpu.Value; + + public bool IsApplicable() => detectors.Any(loader => loader.IsApplicable()); + + public CpuInfo? Detect() => detectors + .Where(loader => loader.IsApplicable()) + .Select(loader => loader.Detect()) + .WhereNotNull() + .FirstOrDefault(); +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Detectors/OsDetector.cs b/src/BenchmarkDotNet/Detectors/OsDetector.cs new file mode 100644 index 0000000000..5deeea03a6 --- /dev/null +++ b/src/BenchmarkDotNet/Detectors/OsDetector.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using System.IO; +using BenchmarkDotNet.Helpers; +using Microsoft.Win32; +using System.Runtime.InteropServices; +using BenchmarkDotNet.Extensions; +using Perfolizer.Models; +using static System.Runtime.InteropServices.RuntimeInformation; +using RuntimeEnvironment = Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment; + +namespace BenchmarkDotNet.Detectors; + +public class OsDetector +{ + public static readonly OsDetector Instance = new(); + private OsDetector() { } + + internal static string ExecutableExtension => IsWindows() ? ".exe" : string.Empty; + internal static string ScriptFileExtension => IsWindows() ? ".bat" : ".sh"; + + private readonly Lazy os = new(ResolveOs); + public static OsInfo GetOs() => Instance.os.Value; + + private static OsInfo ResolveOs() + { + if (IsMacOS()) + { + string systemVersion = ExternalToolsHelper.MacSystemProfilerData.Value.GetValueOrDefault("System Version") ?? ""; + string kernelVersion = ExternalToolsHelper.MacSystemProfilerData.Value.GetValueOrDefault("Kernel Version") ?? ""; + return new OsInfo + { + Name = "macOS", + Version = systemVersion, + KernelVersion = kernelVersion + }; + } + + if (IsLinux()) + { + try + { + string version = LinuxOsReleaseHelper.GetNameByOsRelease(File.ReadAllLines("/etc/os-release")); + bool wsl = IsUnderWsl(); + return new OsInfo + { + Name = "Linux", + Version = version, + Container = wsl ? "WSL" : null + }; + } + catch (Exception) + { + // Ignore + } + } + + string operatingSystem = RuntimeEnvironment.OperatingSystem; + string operatingSystemVersion = RuntimeEnvironment.OperatingSystemVersion; + if (IsWindows()) + { + int? ubr = GetWindowsUbr(); + if (ubr != null) + operatingSystemVersion += $".{ubr}"; + } + return new OsInfo + { + Name = operatingSystem, + Version = operatingSystemVersion + }; + } + + private static bool IsUnderWsl() + { + if (!IsLinux()) + return false; + try + { + return File.Exists("/proc/sys/fs/binfmt_misc/WSLInterop"); // https://superuser.com/a/1749811 + } + catch (Exception) + { + return false; + } + } + + // TODO: Introduce a common util API for registry calls, use it also in BenchmarkDotNet.Toolchains.CsProj.GetCurrentVersionBasedOnWindowsRegistry + /// + /// On Windows, this method returns UBR (Update Build Revision) based on Registry. + /// Returns null if the value is not available + /// + /// + private static int? GetWindowsUbr() + { + if (IsWindows()) + { + try + { + using (var baseKey = RegistryKey.OpenBaseKey(RegistryHive.LocalMachine, RegistryView.Registry32)) + using (var ndpKey = baseKey.OpenSubKey(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion")) + { + if (ndpKey == null) + return null; + + return Convert.ToInt32(ndpKey.GetValue("UBR")); + } + } + catch (Exception) + { + return null; + } + } + return null; + } + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("windows")] +#endif + internal static bool IsWindows() => +#if NET6_0_OR_GREATER + OperatingSystem.IsWindows(); // prefer linker-friendly OperatingSystem APIs +#else + IsOSPlatform(OSPlatform.Windows); +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("linux")] +#endif + internal static bool IsLinux() => +#if NET6_0_OR_GREATER + OperatingSystem.IsLinux(); +#else + IsOSPlatform(OSPlatform.Linux); +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("macos")] +#endif + // ReSharper disable once InconsistentNaming + internal static bool IsMacOS() => +#if NET6_0_OR_GREATER + OperatingSystem.IsMacOS(); +#else + IsOSPlatform(OSPlatform.OSX); +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("android")] +#endif + internal static bool IsAndroid() => +#if NET6_0_OR_GREATER + OperatingSystem.IsAndroid(); +#else + Type.GetType("Java.Lang.Object, Mono.Android") != null; +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("ios")] +#endif + // ReSharper disable once InconsistentNaming + internal static bool IsIOS() => +#if NET6_0_OR_GREATER + OperatingSystem.IsIOS(); +#else + Type.GetType("Foundation.NSObject, Xamarin.iOS") != null; +#endif + +#if NET6_0_OR_GREATER + [System.Runtime.Versioning.SupportedOSPlatformGuard("tvos")] +#endif + // ReSharper disable once InconsistentNaming + internal static bool IsTvOS() => +#if NET6_0_OR_GREATER + OperatingSystem.IsTvOS(); +#else + IsOSPlatform(OSPlatform.Create("TVOS")); +#endif +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs b/src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs index 68591ad8c8..6d7fb7b1db 100644 --- a/src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs +++ b/src/BenchmarkDotNet/Diagnosers/DiagnoserResults.cs @@ -2,6 +2,7 @@ using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.Results; +using System.Collections.Generic; using System.Linq; namespace BenchmarkDotNet.Diagnosers @@ -16,6 +17,7 @@ public DiagnoserResults(BenchmarkCase benchmarkCase, ExecuteResult executeResult ThreadingStats = executeResult.ThreadingStats; BuildResult = buildResult; ExceptionFrequency = executeResult.ExceptionFrequency; + Measurements = executeResult.Measurements; } public BenchmarkCase BenchmarkCase { get; } @@ -29,5 +31,7 @@ public DiagnoserResults(BenchmarkCase benchmarkCase, ExecuteResult executeResult public double ExceptionFrequency { get; } public BuildResult BuildResult { get; } + + public IReadOnlyList Measurements { get; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs b/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs index 1e96f8e495..40e8f231e7 100644 --- a/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs +++ b/src/BenchmarkDotNet/Diagnosers/DiagnosersLoader.cs @@ -5,6 +5,7 @@ using System.Reflection; using System.Threading; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; @@ -40,11 +41,11 @@ private static IEnumerable LoadDiagnosers() { yield return EventPipeProfiler.Default; - if (RuntimeInformation.IsLinux()) + if (OsDetector.IsLinux()) yield return PerfCollectProfiler.Default; } - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) yield break; foreach (var windowsDiagnoser in LoadWindowsDiagnosers()) diff --git a/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs index 782e895d3e..574d1e54f1 100644 --- a/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/ExceptionDiagnoser.cs @@ -1,4 +1,5 @@ using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; @@ -14,9 +15,11 @@ namespace BenchmarkDotNet.Diagnosers { public class ExceptionDiagnoser : IDiagnoser { - public static readonly ExceptionDiagnoser Default = new ExceptionDiagnoser(); + public static readonly ExceptionDiagnoser Default = new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: true)); - private ExceptionDiagnoser() { } + public ExceptionDiagnoser(ExceptionDiagnoserConfig config) => Config = config; + + public ExceptionDiagnoserConfig Config { get; } public IEnumerable Ids => new[] { nameof(ExceptionDiagnoser) }; @@ -32,14 +35,18 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } public IEnumerable ProcessResults(DiagnoserResults results) { - yield return new Metric(ExceptionsFrequencyMetricDescriptor.Instance, results.ExceptionFrequency); + yield return new Metric(new ExceptionsFrequencyMetricDescriptor(Config), results.ExceptionFrequency); } public IEnumerable Validate(ValidationParameters validationParameters) => Enumerable.Empty(); - private class ExceptionsFrequencyMetricDescriptor : IMetricDescriptor + internal class ExceptionsFrequencyMetricDescriptor : IMetricDescriptor { - internal static readonly IMetricDescriptor Instance = new ExceptionsFrequencyMetricDescriptor(); + public ExceptionDiagnoserConfig Config { get; } + public ExceptionsFrequencyMetricDescriptor(ExceptionDiagnoserConfig config = null) + { + Config = config; + } public string Id => "ExceptionFrequency"; public string DisplayName => Column.Exceptions; @@ -49,7 +56,13 @@ private class ExceptionsFrequencyMetricDescriptor : IMetricDescriptor public string Unit => "Count"; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; - public bool GetIsAvailable(Metric metric) => true; + public bool GetIsAvailable(Metric metric) + { + if (Config == null) + return metric.Value > 0; + else + return Config.DisplayExceptionsIfZeroValue || metric.Value > 0; + } } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs index 6dd5bdd2e9..9a3d348665 100644 --- a/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs +++ b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Runtime.InteropServices; using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; @@ -31,8 +32,8 @@ public class PerfCollectProfiler : IProfiler private readonly PerfCollectProfilerConfig config; private readonly DateTime creationTime = DateTime.Now; - private readonly Dictionary benchmarkToTraceFile = new (); - private readonly HashSet cliPathWithSymbolsInstalled = new (); + private readonly Dictionary benchmarkToTraceFile = new(); + private readonly HashSet cliPathWithSymbolsInstalled = new(); private FileInfo perfCollectFile; private Process perfCollectProcess; @@ -53,7 +54,7 @@ public class PerfCollectProfiler : IProfiler public IEnumerable Validate(ValidationParameters validationParameters) { - if (!RuntimeInformation.IsLinux()) + if (!OsDetector.IsLinux()) { yield return new ValidationError(true, "The PerfCollectProfiler works only on Linux!"); yield break; @@ -227,7 +228,7 @@ private void EnsureSymbolsForNativeRuntime(DiagnoserActionParameters parameters) ILogger logger = parameters.Config.GetCompositeLogger(); // We install the tool in a dedicated directory in order to always use latest version and avoid issues with broken existing configs. string toolPath = Path.Combine(Path.GetTempPath(), "BenchmarkDotNet", "symbols"); - DotNetCliCommand cliCommand = new ( + DotNetCliCommand cliCommand = new( cliPath: cliPath, arguments: $"tool install dotnet-symbol --tool-path \"{toolPath}\"", generateResult: null, @@ -252,7 +253,7 @@ private void EnsureSymbolsForNativeRuntime(DiagnoserActionParameters parameters) } private FileInfo GetTraceFile(DiagnoserActionParameters parameters, string extension) - => new (ArtifactFileNameHelper.GetTraceFilePath(parameters, creationTime, extension) + => new(ArtifactFileNameHelper.GetTraceFilePath(parameters, creationTime, extension) .Replace(" ", "_")); // perfcollect does not allow for spaces in the trace file name } } diff --git a/src/BenchmarkDotNet/Diagnosers/SnapshotProfilerBase.cs b/src/BenchmarkDotNet/Diagnosers/SnapshotProfilerBase.cs new file mode 100644 index 0000000000..3105ceeeb9 --- /dev/null +++ b/src/BenchmarkDotNet/Diagnosers/SnapshotProfilerBase.cs @@ -0,0 +1,195 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.IO; +using System.Linq; +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Validators; + +namespace BenchmarkDotNet.Diagnosers; + +public abstract class SnapshotProfilerBase : IProfiler +{ + public abstract string ShortName { get; } + + protected abstract void InitTool(Progress progress); + protected abstract void AttachToCurrentProcess(string snapshotFile); + protected abstract void AttachToProcessByPid(int pid, string snapshotFile); + protected abstract void TakeSnapshot(); + protected abstract void Detach(); + + protected abstract string CreateSnapshotFilePath(DiagnoserActionParameters parameters); + protected abstract string GetRunnerPath(); + internal abstract bool IsSupported(RuntimeMoniker runtimeMoniker); + + private readonly List snapshotFilePaths = []; + + public IEnumerable Ids => [ShortName]; + public IEnumerable Exporters => []; + public IEnumerable Analysers => []; + + public RunMode GetRunMode(BenchmarkCase benchmarkCase) => + IsSupported(benchmarkCase.Job.Environment.GetRuntime().RuntimeMoniker) ? RunMode.ExtraRun : RunMode.None; + + public void Handle(HostSignal signal, DiagnoserActionParameters parameters) + { + var logger = parameters.Config.GetCompositeLogger(); + var job = parameters.BenchmarkCase.Job; + + var runtimeMoniker = job.Environment.GetRuntime().RuntimeMoniker; + if (!IsSupported(runtimeMoniker)) + { + logger.WriteLineError($"Runtime '{runtimeMoniker}' is not supported by dotMemory"); + return; + } + + switch (signal) + { + case HostSignal.BeforeAnythingElse: + Init(logger); + break; + case HostSignal.BeforeActualRun: + string snapshotFilePath = Start(logger, parameters); + snapshotFilePaths.Add(snapshotFilePath); + break; + case HostSignal.AfterActualRun: + Stop(logger); + break; + } + } + + public IEnumerable Validate(ValidationParameters validationParameters) + { + var runtimeMonikers = validationParameters.Benchmarks.Select(b => b.Job.Environment.GetRuntime().RuntimeMoniker).Distinct(); + foreach (var runtimeMoniker in runtimeMonikers) + if (!IsSupported(runtimeMoniker)) + yield return new ValidationError(true, $"Runtime '{runtimeMoniker}' is not supported by dotMemory"); + } + + public IEnumerable ProcessResults(DiagnoserResults results) => ImmutableArray.Empty; + + public void DisplayResults(ILogger logger) + { + if (snapshotFilePaths.Count != 0) + { + logger.WriteLineInfo($"The following {ShortName} snapshots were generated:"); + foreach (string snapshotFilePath in snapshotFilePaths) + logger.WriteLineInfo($"* {snapshotFilePath}"); + } + } + + private void Init(ILogger logger) + { + try + { + logger.WriteLineInfo($"Ensuring that {ShortName} prerequisite is installed..."); + var progress = new Progress(logger, $"Installing {ShortName}"); + InitTool(progress); + logger.WriteLineInfo($"{ShortName} prerequisite is installed"); + logger.WriteLineInfo($"{ShortName} runner path: {GetRunnerPath()}"); + } + catch (Exception e) + { + logger.WriteLineError(e.ToString()); + } + } + + private string Start(ILogger logger, DiagnoserActionParameters parameters) + { + string snapshotFilePath = CreateSnapshotFilePath(parameters); + string? snapshotDirectory = Path.GetDirectoryName(snapshotFilePath); + logger.WriteLineInfo($"Target snapshot file: {snapshotFilePath}"); + if (!Directory.Exists(snapshotDirectory) && snapshotDirectory != null) + { + try + { + Directory.CreateDirectory(snapshotDirectory); + } + catch (Exception e) + { + logger.WriteLineError($"Failed to create directory: {snapshotDirectory}"); + logger.WriteLineError(e.ToString()); + } + } + + try + { + logger.WriteLineInfo($"Attaching {ShortName} to the process..."); + Attach(parameters, snapshotFilePath); + logger.WriteLineInfo($"{ShortName} is successfully attached"); + } + catch (Exception e) + { + logger.WriteLineError(e.ToString()); + return snapshotFilePath; + } + + return snapshotFilePath; + } + + private void Stop(ILogger logger) + { + try + { + logger.WriteLineInfo($"Taking {ShortName} snapshot..."); + TakeSnapshot(); + logger.WriteLineInfo($"{ShortName} snapshot is successfully taken"); + } + catch (Exception e) + { + logger.WriteLineError(e.ToString()); + } + + try + { + logger.WriteLineInfo($"Detaching {ShortName} from the process..."); + Detach(); + logger.WriteLineInfo($"{ShortName} is successfully detached"); + } + catch (Exception e) + { + logger.WriteLineError(e.ToString()); + } + } + + + private void Attach(DiagnoserActionParameters parameters, string snapshotFile) + { + int pid = parameters.Process.Id; + int currentPid = Process.GetCurrentProcess().Id; + if (pid != currentPid) + AttachToProcessByPid(pid, snapshotFile); + else + AttachToCurrentProcess(snapshotFile); + } + + protected class Progress(ILogger logger, string title) : IProgress + { + private static readonly TimeSpan ReportInterval = TimeSpan.FromSeconds(0.1); + + private int lastProgress; + private Stopwatch? stopwatch; + + public void Report(double value) + { + int progress = (int)Math.Floor(value); + bool needToReport = stopwatch == null || + (stopwatch != null && stopwatch?.Elapsed > ReportInterval) || + progress == 100; + + if (lastProgress != progress && needToReport) + { + logger.WriteLineInfo($"{title}: {progress}%"); + lastProgress = progress; + stopwatch = Stopwatch.StartNew(); + } + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs index 057a0bb624..36b2f942c3 100644 --- a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs @@ -15,9 +15,10 @@ namespace BenchmarkDotNet.Diagnosers { public class ThreadingDiagnoser : IDiagnoser { - public static readonly ThreadingDiagnoser Default = new ThreadingDiagnoser(); + public static readonly ThreadingDiagnoser Default = new ThreadingDiagnoser(new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: true, displayLockContentionWhenZero: true)); - private ThreadingDiagnoser() { } + public ThreadingDiagnoser(ThreadingDiagnoserConfig config) => Config = config; + public ThreadingDiagnoserConfig Config { get; } public IEnumerable Ids => new[] { nameof(ThreadingDiagnoser) }; @@ -33,8 +34,9 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } public IEnumerable ProcessResults(DiagnoserResults results) { - yield return new Metric(CompletedWorkItemCountMetricDescriptor.Instance, results.ThreadingStats.CompletedWorkItemCount / (double)results.ThreadingStats.TotalOperations); - yield return new Metric(LockContentionCountMetricDescriptor.Instance, results.ThreadingStats.LockContentionCount / (double)results.ThreadingStats.TotalOperations); + + yield return new Metric(new CompletedWorkItemCountMetricDescriptor(Config), results.ThreadingStats.CompletedWorkItemCount / (double)results.ThreadingStats.TotalOperations); + yield return new Metric(new LockContentionCountMetricDescriptor(Config), results.ThreadingStats.LockContentionCount / (double)results.ThreadingStats.TotalOperations); } public IEnumerable Validate(ValidationParameters validationParameters) @@ -50,10 +52,15 @@ public IEnumerable Validate(ValidationParameters validationPara } } - private class CompletedWorkItemCountMetricDescriptor : IMetricDescriptor + internal class CompletedWorkItemCountMetricDescriptor : IMetricDescriptor { internal static readonly IMetricDescriptor Instance = new CompletedWorkItemCountMetricDescriptor(); + private ThreadingDiagnoserConfig Config { get; } + public CompletedWorkItemCountMetricDescriptor(ThreadingDiagnoserConfig config = null) + { + Config = config; + } public string Id => "CompletedWorkItemCount"; public string DisplayName => Column.CompletedWorkItems; public string Legend => "The number of work items that have been processed in ThreadPool (per single operation)"; @@ -62,13 +69,26 @@ private class CompletedWorkItemCountMetricDescriptor : IMetricDescriptor public string Unit => "Count"; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; - public bool GetIsAvailable(Metric metric) => true; + public bool GetIsAvailable(Metric metric) + { + if (Config == null) + return metric.Value > 0; + else + return Config.DisplayCompletedWorkItemCountWhenZero || metric.Value > 0; + } } - private class LockContentionCountMetricDescriptor : IMetricDescriptor + internal class LockContentionCountMetricDescriptor : IMetricDescriptor { internal static readonly IMetricDescriptor Instance = new LockContentionCountMetricDescriptor(); + private ThreadingDiagnoserConfig Config { get; } + + public LockContentionCountMetricDescriptor(ThreadingDiagnoserConfig config = null) + { + Config = config; + } + public string Id => "LockContentionCount"; public string DisplayName => Column.LockContentions; public string Legend => "The number of times there was contention upon trying to take a Monitor's lock (per single operation)"; @@ -77,7 +97,13 @@ private class LockContentionCountMetricDescriptor : IMetricDescriptor public string Unit => "Count"; public bool TheGreaterTheBetter => false; public int PriorityInCategory => 0; - public bool GetIsAvailable(Metric metric) => true; + public bool GetIsAvailable(Metric metric) + { + if (Config == null) + return metric.Value > 0; + else + return Config.DisplayLockContentionWhenZero || metric.Value > 0; + } } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoserConfig.cs b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoserConfig.cs new file mode 100644 index 0000000000..6af3da25ac --- /dev/null +++ b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoserConfig.cs @@ -0,0 +1,23 @@ +using JetBrains.Annotations; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BenchmarkDotNet.Diagnosers +{ + public class ThreadingDiagnoserConfig + { + /// Display configuration for 'LockContentionCount' when it is empty. True (displayed) by default. + /// Display configuration for 'CompletedWorkItemCount' when it is empty. True (displayed) by default. + + [PublicAPI] + public ThreadingDiagnoserConfig(bool displayLockContentionWhenZero = true, bool displayCompletedWorkItemCountWhenZero = true) + { + DisplayLockContentionWhenZero = displayLockContentionWhenZero; + DisplayCompletedWorkItemCountWhenZero = displayCompletedWorkItemCountWhenZero; + } + + public bool DisplayLockContentionWhenZero { get; } + public bool DisplayCompletedWorkItemCountWhenZero { get; } + } +} diff --git a/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs index f00bb4ed67..43cc910559 100644 --- a/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Loggers; @@ -31,10 +32,10 @@ public IEnumerable Validate(ValidationParameters validationPara => new[] { new ValidationError(false, GetErrorMessage()) }; private string GetErrorMessage() => $@"Unable to resolve {unresolved.Name} diagnoser using dynamic assembly loading. - {(RuntimeInformation.IsFullFramework || RuntimeInformation.IsWindows() + {(RuntimeInformation.IsFullFramework || OsDetector.IsWindows() ? "Please make sure that you have installed the latest BenchmarkDotNet.Diagnostics.Windows package. " + Environment.NewLine + "If you are using `dotnet build` you also need to consume one of its public types to make sure that MSBuild copies it to the output directory. " + "The alternative is to use `true` in your project file." - : $"Please make sure that it's supported on {RuntimeInformation.GetOsVersion()}")}"; + : $"Please make sure that it's supported on {OsDetector.GetOs()}")}"; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs b/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs index 16ebbde219..7c10765b8e 100644 --- a/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/Arm64Disassembler.cs @@ -139,7 +139,7 @@ public void Feed(Arm64Instruction instruction) public Arm64RegisterId RegisterId { get { return _registerId; } } } - internal class Arm64Disassembler : ClrMdV2Disassembler + internal class Arm64Disassembler : ClrMdV3Disassembler { internal sealed class RuntimeSpecificData { @@ -189,7 +189,7 @@ internal RuntimeSpecificData(State state) } } - private static readonly Dictionary runtimeSpecificData = new (); + private static readonly Dictionary runtimeSpecificData = new(); protected override IEnumerable Decode(byte[] code, ulong startAddress, State state, int depth, ClrMethod currentMethod, DisassemblySyntax syntax) { diff --git a/src/BenchmarkDotNet/Disassemblers/Arm64InstructionFormatter.cs b/src/BenchmarkDotNet/Disassemblers/Arm64InstructionFormatter.cs index deb557256c..5426dcfdd9 100644 --- a/src/BenchmarkDotNet/Disassemblers/Arm64InstructionFormatter.cs +++ b/src/BenchmarkDotNet/Disassemblers/Arm64InstructionFormatter.cs @@ -12,7 +12,7 @@ internal static class Arm64InstructionFormatter internal static string Format(Arm64Asm asm, FormatterOptions formatterOptions, bool printInstructionAddresses, uint pointerSize, IReadOnlyDictionary symbols) { - StringBuilder output = new (); + StringBuilder output = new(); Arm64Instruction instruction = asm.Instruction; if (printInstructionAddresses) diff --git a/src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs b/src/BenchmarkDotNet/Disassemblers/ClrMdV3Disassembler.cs similarity index 76% rename from src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs rename to src/BenchmarkDotNet/Disassemblers/ClrMdV3Disassembler.cs index 16b3891d24..6bd01426d8 100644 --- a/src/BenchmarkDotNet/Disassemblers/ClrMdV2Disassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/ClrMdV3Disassembler.cs @@ -7,23 +7,25 @@ using System.IO; using System.Linq; using System.Text.RegularExpressions; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Disassemblers { - // This Disassembler uses ClrMd v2x. Please keep it in sync with ClrMdV1Disassembler (if possible). - internal abstract class ClrMdV2Disassembler + // This Disassembler uses ClrMd v3x. Please keep it in sync with ClrMdV1Disassembler (if possible). + internal abstract class ClrMdV3Disassembler + { private static readonly ulong MinValidAddress = GetMinValidAddress(); private static ulong GetMinValidAddress() { // https://github.com/dotnet/BenchmarkDotNet/pull/2413#issuecomment-1688100117 - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) return ushort.MaxValue + 1; - if (RuntimeInformation.IsLinux()) + if (OsDetector.IsLinux()) return (ulong) Environment.SystemPageSize; - if (RuntimeInformation.IsMacOS()) + if (OsDetector.IsMacOS()) return RuntimeInformation.GetCurrentPlatform() switch { Environments.Platform.X86 or Environments.Platform.X64 => 4096, @@ -64,7 +66,7 @@ internal DisassemblyResult AttachAndDisassemble(Settings settings) state.Todo.Enqueue( new MethodInfo( // the Disassembler Entry Method is always parameterless, so check by name is enough - typeWithBenchmark.Methods.Single(method => method.IsPublic && method.Name == settings.MethodName), + typeWithBenchmark.Methods.Single(method => method.Attributes.HasFlag(System.Reflection.MethodAttributes.Public) && method.Name == settings.MethodName), 0)); } @@ -149,9 +151,10 @@ private DisassembledMethod DisassembleMethod(MethodInfo methodInfo, State state, if (!CanBeDisassembled(method)) { - if (method.IsPInvoke) + if (method.Attributes.HasFlag(System.Reflection.MethodAttributes.PinvokeImpl)) return CreateEmpty(method, "PInvoke method"); - if (method.IL is null || method.IL.Length == 0) + var ilInfo = method.GetILInfo(); + if (ilInfo is null || ilInfo.Length == 0) return CreateEmpty(method, "Extern method"); if (method.CompilationType == MethodCompilationType.None) return CreateEmpty(method, "Method was not JITted yet."); @@ -214,60 +217,30 @@ private IEnumerable Decode(ILToNativeMap map, State state, int depth, ClrMe private static ILToNativeMap[] GetCompleteNativeMap(ClrMethod method, ClrRuntime runtime) { - if (!TryReadNativeCodeAddresses(runtime, method, out ulong startAddress, out ulong endAddress)) + // it's better to use one single map rather than few small ones + // it's simply easier to get next instruction when decoding ;) + + var hotColdInfo = method.HotColdInfo; + if (hotColdInfo.HotSize > 0 && hotColdInfo.HotStart > 0) { - startAddress = method.NativeCode; - endAddress = ulong.MaxValue; + return hotColdInfo.ColdSize <= 0 + ? new[] { new ILToNativeMap() { StartAddress = hotColdInfo.HotStart, EndAddress = hotColdInfo.HotStart + hotColdInfo.HotSize, ILOffset = -1 } } + : new[] + { + new ILToNativeMap() { StartAddress = hotColdInfo.HotStart, EndAddress = hotColdInfo.HotStart + hotColdInfo.HotSize, ILOffset = -1 }, + new ILToNativeMap() { StartAddress = hotColdInfo.ColdStart, EndAddress = hotColdInfo.ColdStart + hotColdInfo.ColdSize, ILOffset = -1 } + }; } - ILToNativeMap[] sortedMaps = method.ILOffsetMap // CanBeDisassembled ensures that there is at least one map in ILOffsetMap - .Where(map => map.StartAddress >= startAddress && map.StartAddress < endAddress) // can be false for Tier 0 maps, EndAddress is not checked on purpose here - .Where(map => map.StartAddress < map.EndAddress) // some maps have 0 length (they don't have corresponding assembly code?) + return method.ILOffsetMap + .Where(map => map.StartAddress < map.EndAddress) // some maps have 0 length? .OrderBy(map => map.StartAddress) // we need to print in the machine code order, not IL! #536 - .Select(map => new ILToNativeMap() - { - StartAddress = map.StartAddress, - // some maps have EndAddress > codeHeaderData.MethodStart + codeHeaderData.MethodSize and contain garbage (#2074). They need to be fixed! - EndAddress = Math.Min(map.EndAddress, endAddress), - ILOffset = map.ILOffset - }) .ToArray(); - - if (sortedMaps.Length == 0) - { - // In such situation ILOffsetMap most likely describes Tier 0, while CodeHeaderData Tier 1. - // Since we care about Tier 1 (if it's present), we "fake" a Tier 1 map. - return new[] { new ILToNativeMap() { StartAddress = startAddress, EndAddress = endAddress } }; - } - else if (sortedMaps[0].StartAddress != startAddress || (sortedMaps[sortedMaps.Length - 1].EndAddress != endAddress && endAddress != ulong.MaxValue)) - { - // In such situation ILOffsetMap most likely is missing few bytes. We just "extend" it to avoid producing "bad" instructions. - return new[] { new ILToNativeMap() { StartAddress = startAddress, EndAddress = endAddress } }; - } - - return sortedMaps; } private static DisassembledMethod CreateEmpty(ClrMethod method, string reason) => DisassembledMethod.Empty(method.Signature, method.NativeCode, reason); - protected static bool TryReadNativeCodeAddresses(ClrRuntime runtime, ClrMethod method, out ulong startAddress, out ulong endAddress) - { - if (method is not null - && runtime.DacLibrary.SOSDacInterface.GetCodeHeaderData(method.NativeCode, out var codeHeaderData) == HResult.S_OK - && codeHeaderData.MethodSize > 0) // false for extern methods! - { - // HotSize can be missing or be invalid (https://github.com/microsoft/clrmd/issues/1036). - // So we fetch the method size on our own. - startAddress = codeHeaderData.MethodStart; - endAddress = codeHeaderData.MethodStart + codeHeaderData.MethodSize; - return true; - } - - startAddress = endAddress = 0; - return false; - } - protected void TryTranslateAddressToName(ulong address, bool isAddressPrecodeMD, State state, int depth, ClrMethod currentMethod) { if (!IsValidAddress(address) || state.AddressToNameMapping.ContainsKey(address)) @@ -283,18 +256,10 @@ protected void TryTranslateAddressToName(ulong address, bool isAddressPrecodeMD, } var method = runtime.GetMethodByInstructionPointer(address); - if (method is null && (address & ((uint) runtime.DataTarget.DataReader.PointerSize - 1)) == 0) - { - if (runtime.DataTarget.DataReader.ReadPointer(address, out ulong newAddress) && IsValidAddress(newAddress)) - { - method = runtime.GetMethodByInstructionPointer(newAddress); - - method = WorkaroundGetMethodByInstructionPointerBug(runtime, method, newAddress); - } - } - else + if (method is null && (address & ((uint) runtime.DataTarget.DataReader.PointerSize - 1)) == 0 + && runtime.DataTarget.DataReader.ReadPointer(address, out ulong newAddress) && IsValidAddress(newAddress)) { - method = WorkaroundGetMethodByInstructionPointerBug(runtime, method, address); + method = runtime.GetMethodByInstructionPointer(newAddress); } if (method is null) @@ -313,7 +278,7 @@ protected void TryTranslateAddressToName(ulong address, bool isAddressPrecodeMD, return; } - var methodTableName = runtime.DacLibrary.SOSDacInterface.GetMethodTableName(address); + var methodTableName = runtime.GetTypeByMethodTable(address)?.Name; if (!string.IsNullOrEmpty(methodTableName)) { state.AddressToNameMapping.Add(address, $"MT_{methodTableName}"); @@ -335,7 +300,7 @@ protected void TryTranslateAddressToName(ulong address, bool isAddressPrecodeMD, protected void FlushCachedDataIfNeeded(IDataReader dataTargetDataReader, ulong address, byte[] buffer) { - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { if (dataTargetDataReader.Read(address, buffer) <= 0) { @@ -349,13 +314,6 @@ protected void FlushCachedDataIfNeeded(IDataReader dataTargetDataReader, ulong a } } - // GetMethodByInstructionPointer sometimes returns wrong methods. - // In case given address does not belong to the methods range, null is returned. - private static ClrMethod WorkaroundGetMethodByInstructionPointerBug(ClrRuntime runtime, ClrMethod method, ulong newAddress) - => TryReadNativeCodeAddresses(runtime, method, out ulong startAddress, out ulong endAddress) && !(startAddress >= newAddress && newAddress <= endAddress) - ? null - : method; - private class SharpComparer : IEqualityComparer { public bool Equals(Sharp x, Sharp y) diff --git a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs index 2dd18a1b61..7d0f95326c 100644 --- a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs +++ b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs @@ -3,6 +3,7 @@ using System.Linq; using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Disassemblers; using BenchmarkDotNet.Disassemblers.Exporters; using BenchmarkDotNet.Engines; @@ -75,7 +76,7 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) case HostSignal.AfterAll when ShouldUseSameArchitectureDisassembler(benchmark, parameters): results.Add(benchmark, sameArchitectureDisassembler.Disassemble(parameters)); break; - case HostSignal.AfterAll when RuntimeInformation.IsWindows() && !ShouldUseMonoDisassembler(benchmark): + case HostSignal.AfterAll when OsDetector.IsWindows() && !ShouldUseMonoDisassembler(benchmark): results.Add(benchmark, windowsDifferentArchitectureDisassembler.Disassemble(parameters)); break; case HostSignal.SeparateLogic when ShouldUseMonoDisassembler(benchmark): @@ -112,7 +113,7 @@ public IEnumerable Validate(ValidationParameters validationPara if (ShouldUseClrMdDisassembler(benchmark)) { - if (RuntimeInformation.IsLinux()) + if (OsDetector.IsLinux()) { var runtime = benchmark.Job.ResolveValue(EnvironmentMode.RuntimeCharacteristic, EnvironmentResolver.Instance); @@ -143,13 +144,13 @@ private static bool ShouldUseMonoDisassembler(BenchmarkCase benchmarkCase) // when we add macOS support, RuntimeInformation.IsMacOS() needs to be added here private static bool ShouldUseClrMdDisassembler(BenchmarkCase benchmarkCase) - => !ShouldUseMonoDisassembler(benchmarkCase) && (RuntimeInformation.IsWindows() || RuntimeInformation.IsLinux()); + => !ShouldUseMonoDisassembler(benchmarkCase) && (OsDetector.IsWindows() || OsDetector.IsLinux()); private static bool ShouldUseSameArchitectureDisassembler(BenchmarkCase benchmarkCase, DiagnoserActionParameters parameters) { if (ShouldUseClrMdDisassembler(benchmarkCase)) { - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { return WindowsDisassembler.GetDisassemblerArchitecture(parameters.Process, benchmarkCase.Job.Environment.Platform) == RuntimeInformation.GetCurrentPlatform(); diff --git a/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs index f29faae037..eb0d5ce3f6 100644 --- a/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/IntelDisassembler.cs @@ -7,7 +7,7 @@ namespace BenchmarkDotNet.Disassemblers { - internal class IntelDisassembler : ClrMdV2Disassembler + internal class IntelDisassembler : ClrMdV3Disassembler { internal sealed class RuntimeSpecificData { @@ -46,7 +46,7 @@ internal RuntimeSpecificData(State state) } } - private static readonly Dictionary runtimeSpecificData = new (); + private static readonly Dictionary runtimeSpecificData = new(); protected override IEnumerable Decode(byte[] code, ulong startAddress, State state, int depth, ClrMethod currentMethod, DisassemblySyntax syntax) { diff --git a/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs index cf810cdd86..1a4835484d 100644 --- a/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/SameArchitectureDisassembler.cs @@ -8,16 +8,16 @@ namespace BenchmarkDotNet.Disassemblers internal class SameArchitectureDisassembler { private readonly DisassemblyDiagnoserConfig config; - private ClrMdV2Disassembler? clrMdV2Disassembler; + private ClrMdV3Disassembler? clrMdV3Disassembler; internal SameArchitectureDisassembler(DisassemblyDiagnoserConfig config) => this.config = config; internal DisassemblyResult Disassemble(DiagnoserActionParameters parameters) // delay the creation to avoid exceptions at DisassemblyDiagnoser ctor - => (clrMdV2Disassembler ??= CreateDisassemblerForCurrentArchitecture()) + => (clrMdV3Disassembler ??= CreateDisassemblerForCurrentArchitecture()) .AttachAndDisassemble(BuildDisassemblerSettings(parameters)); - private static ClrMdV2Disassembler CreateDisassemblerForCurrentArchitecture() + private static ClrMdV3Disassembler CreateDisassemblerForCurrentArchitecture() => RuntimeInformation.GetCurrentPlatform() switch { Platform.X86 or Platform.X64 => new IntelDisassembler(), @@ -26,7 +26,7 @@ private static ClrMdV2Disassembler CreateDisassemblerForCurrentArchitecture() }; private Settings BuildDisassemblerSettings(DiagnoserActionParameters parameters) - => new ( + => new( processId: parameters.Process.Id, typeName: $"BenchmarkDotNet.Autogenerated.Runnable_{parameters.BenchmarkId.Value}", methodName: DisassemblerConstants.DisassemblerEntryMethodName, diff --git a/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs index 6bec70162a..8051ba3463 100644 --- a/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/WindowsDisassembler.cs @@ -7,6 +7,7 @@ using System.Text; using System.Xml; using System.Xml.Serialization; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; @@ -165,7 +166,7 @@ public static bool Is64Bit(Process process) if (Environment.GetEnvironmentVariable("PROCESSOR_ARCHITECTURE") == "x86") return false; - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { IsWow64Process(process.Handle, out bool isWow64); diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index 9dc8cc616e..1fc1b96dfd 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -1,10 +1,14 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.Linq; using System.Runtime.CompilerServices; +using System.Threading; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using JetBrains.Annotations; @@ -15,8 +19,6 @@ namespace BenchmarkDotNet.Engines [UsedImplicitly] public class Engine : IEngine { - public const int MinInvokeCount = 4; - [PublicAPI] public IHost Host { get; } [PublicAPI] public Action WorkloadAction { get; } [PublicAPI] public Action Dummy1Action { get; } @@ -40,10 +42,7 @@ public class Engine : IEngine private bool EvaluateOverhead { get; } private bool MemoryRandomization { get; } - private readonly List jittingMeasurements = new (10); - private readonly EnginePilotStage pilotStage; - private readonly EngineWarmupStage warmupStage; - private readonly EngineActualStage actualStage; + private readonly List jittingMeasurements = new(10); private readonly bool includeExtraStats; private readonly Random random; @@ -79,10 +78,6 @@ internal Engine( EvaluateOverhead = targetJob.ResolveValue(AccuracyMode.EvaluateOverheadCharacteristic, Resolver); MemoryRandomization = targetJob.ResolveValue(RunMode.MemoryRandomizationCharacteristic, Resolver); - warmupStage = new EngineWarmupStage(this); - pilotStage = new EnginePilotStage(this); - actualStage = new EngineActualStage(this); - random = new Random(12345); // we are using constant seed to try to get repeatable results } @@ -102,6 +97,9 @@ public void Dispose() } } + // AggressiveOptimization forces the method to go straight to tier1 JIT, and will never be re-jitted, + // eliminating tiered JIT as a potential variable in measurements. + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] public RunResults Run() { var measurements = new List(); @@ -112,29 +110,33 @@ public RunResults Run() if (EngineEventSource.Log.IsEnabled()) EngineEventSource.Log.BenchmarkStart(BenchmarkName); - if (Strategy != RunStrategy.ColdStart) + // Enumerate the stages and run iterations in a loop to ensure each benchmark invocation is called with a constant stack size. + // #1120 + foreach (var stage in EngineStage.EnumerateStages(this, Strategy, EvaluateOverhead)) { - if (Strategy != RunStrategy.Monitoring) + if (stage.Stage == IterationStage.Actual && stage.Mode == IterationMode.Workload) { - var pilotStageResult = pilotStage.Run(); - invokeCount = pilotStageResult.PerfectInvocationCount; - measurements.AddRange(pilotStageResult.Measurements); - - if (EvaluateOverhead) - { - measurements.AddRange(warmupStage.RunOverhead(invokeCount, UnrollFactor)); - measurements.AddRange(actualStage.RunOverhead(invokeCount, UnrollFactor)); - } + Host.BeforeMainRun(); } - measurements.AddRange(warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy)); - } - - Host.BeforeMainRun(); + var stageMeasurements = stage.GetMeasurementList(); + // 1-based iterationIndex + int iterationIndex = 1; + while (stage.GetShouldRunIteration(stageMeasurements, ref invokeCount)) + { + var measurement = RunIteration(new IterationData(stage.Mode, stage.Stage, iterationIndex, invokeCount, UnrollFactor)); + stageMeasurements.Add(measurement); + ++iterationIndex; + } + measurements.AddRange(stageMeasurements); - measurements.AddRange(actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring)); + WriteLine(); - Host.AfterMainRun(); + if (stage.Stage == IterationStage.Actual && stage.Mode == IterationMode.Workload) + { + Host.AfterMainRun(); + } + } (GcStats workGcHasDone, ThreadingStats threadingStats, double exceptionFrequency) = includeExtraStats ? GetExtraStats(new IterationData(IterationMode.Workload, IterationStage.Actual, 0, invokeCount, UnrollFactor)) @@ -148,11 +150,15 @@ public RunResults Run() return new RunResults(measurements, outlierMode, workGcHasDone, threadingStats, exceptionFrequency); } + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] public Measurement RunIteration(IterationData data) { // Initialization long invokeCount = data.InvokeCount; int unrollFactor = data.UnrollFactor; + if (invokeCount % unrollFactor != 0) + throw new ArgumentOutOfRangeException(nameof(data), $"InvokeCount({invokeCount}) should be a multiple of UnrollFactor({unrollFactor})."); + long totalOperations = invokeCount * OperationsPerInvoke; bool isOverhead = data.IterationMode == IterationMode.Overhead; bool randomizeMemory = !isOverhead && MemoryRandomization; @@ -167,7 +173,7 @@ public Measurement RunIteration(IterationData data) EngineEventSource.Log.IterationStart(data.IterationMode, data.IterationStage, totalOperations); var clockSpan = randomizeMemory - ? MeasureWithRandomMemory(action, invokeCount / unrollFactor) + ? MeasureWithRandomStack(action, invokeCount / unrollFactor) : Measure(action, invokeCount / unrollFactor); if (EngineEventSource.Log.IsEnabled()) @@ -193,8 +199,8 @@ public Measurement RunIteration(IterationData data) // This is in a separate method, because stackalloc can affect code alignment, // resulting in unexpected measurements on some AMD cpus, // even if the stackalloc branch isn't executed. (#2366) - [MethodImpl(MethodImplOptions.NoInlining)] - private unsafe ClockSpan MeasureWithRandomMemory(Action action, long invokeCount) + [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)] + private unsafe ClockSpan MeasureWithRandomStack(Action action, long invokeCount) { byte* stackMemory = stackalloc byte[random.Next(32)]; var clockSpan = Measure(action, invokeCount); @@ -205,6 +211,7 @@ private unsafe ClockSpan MeasureWithRandomMemory(Action action, long invok [MethodImpl(MethodImplOptions.NoInlining)] private unsafe void Consume(byte* _) { } + [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)] private ClockSpan Measure(Action action, long invokeCount) { var clock = Clock.Start(); @@ -214,31 +221,56 @@ private ClockSpan Measure(Action action, long invokeCount) private (GcStats, ThreadingStats, double) GetExtraStats(IterationData data) { - // we enable monitoring after main target run, for this single iteration which is executed at the end - // so even if we enable AppDomain monitoring in separate process - // it does not matter, because we have already obtained the results! - EnableMonitoring(); + // Warm up the measurement functions before starting the actual measurement. + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(GcStats.ReadInitial()); + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(GcStats.ReadFinal()); IterationSetupAction(); // we run iteration setup first, so even if it allocates, it is not included in the results var initialThreadingStats = ThreadingStats.ReadInitial(); // this method might allocate var exceptionsStats = new ExceptionsStats(); // allocates exceptionsStats.StartListening(); // this method might allocate - var initialGcStats = GcStats.ReadInitial(); - WorkloadAction(data.InvokeCount / data.UnrollFactor); +#if !NET7_0_OR_GREATER + if (RuntimeInformation.IsNetCore && Environment.Version.Major is >= 3 and <= 6 && RuntimeInformation.IsTieredJitEnabled) + { + // #1542 + // We put the current thread to sleep so tiered jit can kick in, compile its stuff, + // and NOT allocate anything on the background thread when we are measuring allocations. + // This is only an issue on netcoreapp3.0 to net6.0. Tiered jit allocations were "fixed" in net7.0 + // (maybe not completely eliminated forever, but at least reduced to a point where measurements are much more stable), + // and netcoreapp2.X uses only GetAllocatedBytesForCurrentThread which doesn't capture the tiered jit allocations. + Thread.Sleep(TimeSpan.FromMilliseconds(500)); + } +#endif - exceptionsStats.Stop(); - var finalGcStats = GcStats.ReadFinal(); + // GC collect before measuring allocations. + ForceGcCollect(); + GcStats gcStats; + using (FinalizerBlocker.MaybeStart()) + { + gcStats = MeasureWithGc(data.InvokeCount / data.UnrollFactor); + } + + exceptionsStats.Stop(); // this method might (de)allocate var finalThreadingStats = ThreadingStats.ReadFinal(); IterationCleanupAction(); // we run iteration cleanup after collecting GC stats var totalOperationsCount = data.InvokeCount * OperationsPerInvoke; - GcStats gcStats = (finalGcStats - initialGcStats).WithTotalOperations(totalOperationsCount); - ThreadingStats threadingStats = (finalThreadingStats - initialThreadingStats).WithTotalOperations(data.InvokeCount * OperationsPerInvoke); + return (gcStats.WithTotalOperations(totalOperationsCount), + (finalThreadingStats - initialThreadingStats).WithTotalOperations(totalOperationsCount), + exceptionsStats.ExceptionsCount / (double)totalOperationsCount); + } - return (gcStats, threadingStats, exceptionsStats.ExceptionsCount / (double)totalOperationsCount); + // Isolate the allocation measurement and skip tier0 jit to make sure we don't get any unexpected allocations. + [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)] + private GcStats MeasureWithGc(long invokeCount) + { + var initialGcStats = GcStats.ReadInitial(); + WorkloadAction(invokeCount); + var finalGcStats = GcStats.ReadFinal(); + return finalGcStats - initialGcStats; } private void RandomizeManagedHeapMemory() @@ -267,7 +299,7 @@ private void GcCollect() ForceGcCollect(); } - private static void ForceGcCollect() + internal static void ForceGcCollect() { GC.Collect(); GC.WaitForPendingFinalizers(); @@ -278,15 +310,6 @@ private static void ForceGcCollect() public void WriteLine() => Host.WriteLine(); - private static void EnableMonitoring() - { - if (RuntimeInformation.IsOldMono) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes-in-mono - return; - - if (RuntimeInformation.IsFullFramework) - AppDomain.MonitoringIsEnabled = true; - } - [UsedImplicitly] public static class Signals { @@ -309,5 +332,71 @@ private static readonly Dictionary MessagesToSignals public static bool TryGetSignal(string message, out HostSignal signal) => MessagesToSignals.TryGetValue(message, out signal); } + + // Very long key and value so this shouldn't be used outside of unit tests. + internal const string UnitTestBlockFinalizerEnvKey = "BENCHMARKDOTNET_UNITTEST_BLOCK_FINALIZER_FOR_MEMORYDIAGNOSER"; + internal const string UnitTestBlockFinalizerEnvValue = UnitTestBlockFinalizerEnvKey + "_ACTIVE"; + + // To prevent finalizers interfering with allocation measurements for unit tests, + // we block the finalizer thread until we've completed the measurement. + // https://github.com/dotnet/runtime/issues/101536#issuecomment-2077647417 + private readonly struct FinalizerBlocker : IDisposable + { + private readonly object hangLock; + + private FinalizerBlocker(object hangLock) => this.hangLock = hangLock; + + private sealed class Impl + { + // ManualResetEvent(Slim) allocates when it is waited and yields the thread, + // so we use Monitor.Wait instead which does not allocate managed memory. + // This behavior is not documented, but was observed with the VS Profiler. + private readonly object hangLock = new(); + private readonly ManualResetEventSlim enteredFinalizerEvent = new(false); + + ~Impl() + { + lock (hangLock) + { + enteredFinalizerEvent.Set(); + Monitor.Wait(hangLock); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + internal static (object hangLock, ManualResetEventSlim enteredFinalizerEvent) CreateWeakly() + { + var impl = new Impl(); + return (impl.hangLock, impl.enteredFinalizerEvent); + } + } + + internal static FinalizerBlocker MaybeStart() + { + if (Environment.GetEnvironmentVariable(UnitTestBlockFinalizerEnvKey) != UnitTestBlockFinalizerEnvValue) + { + return default; + } + var (hangLock, enteredFinalizerEvent) = Impl.CreateWeakly(); + do + { + GC.Collect(); + // Do NOT call GC.WaitForPendingFinalizers. + } + while (!enteredFinalizerEvent.IsSet); + return new FinalizerBlocker(hangLock); + } + + public void Dispose() + { + if (hangLock is not null) + { + lock (hangLock) + { + Monitor.Pulse(hangLock); + } + } + } + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineActualStage.cs b/src/BenchmarkDotNet/Engines/EngineActualStage.cs new file mode 100644 index 0000000000..173f4e3432 --- /dev/null +++ b/src/BenchmarkDotNet/Engines/EngineActualStage.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Mathematics; +using BenchmarkDotNet.Reports; +using Perfolizer.Horology; +using Perfolizer.Mathematics.OutlierDetection; + +namespace BenchmarkDotNet.Engines +{ + internal abstract class EngineActualStage(IterationMode iterationMode) : EngineStage(IterationStage.Actual, iterationMode) + { + internal const int MaxOverheadIterationCount = 20; + + internal static EngineActualStage GetOverhead(IEngine engine) + => new EngineActualStageAuto(engine.TargetJob, engine.Resolver, IterationMode.Overhead); + + internal static EngineActualStage GetWorkload(IEngine engine, RunStrategy strategy) + { + var targetJob = engine.TargetJob; + int? iterationCount = targetJob.ResolveValueAsNullable(RunMode.IterationCountCharacteristic); + const int DefaultWorkloadCount = 10; + return iterationCount == null && strategy != RunStrategy.Monitoring + ? new EngineActualStageAuto(targetJob, engine.Resolver, IterationMode.Workload) + : new EngineActualStageSpecific(iterationCount ?? DefaultWorkloadCount, IterationMode.Workload); + } + } + + internal sealed class EngineActualStageAuto : EngineActualStage + { + private readonly double maxRelativeError; + private readonly TimeInterval? maxAbsoluteError; + private readonly OutlierMode outlierMode; + private readonly int minIterationCount; + private readonly int maxIterationCount; + private readonly List measurementsForStatistics; + private int iterationCounter = 0; + + public EngineActualStageAuto(Job targetJob, IResolver resolver, IterationMode iterationMode) : base(iterationMode) + { + maxRelativeError = targetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, resolver); + maxAbsoluteError = targetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); + outlierMode = targetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, resolver); + minIterationCount = targetJob.ResolveValue(RunMode.MinIterationCountCharacteristic, resolver); + maxIterationCount = targetJob.ResolveValue(RunMode.MaxIterationCountCharacteristic, resolver); + measurementsForStatistics = GetMeasurementList(); + } + + internal override List GetMeasurementList() => new(maxIterationCount); + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + { + if (measurements.Count == 0) + { + return true; + } + + const double MaxOverheadRelativeError = 0.05; + bool isOverhead = Mode == IterationMode.Overhead; + double effectiveMaxRelativeError = isOverhead ? MaxOverheadRelativeError : maxRelativeError; + iterationCounter++; + var measurement = measurements[measurements.Count - 1]; + measurementsForStatistics.Add(measurement); + + var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, outlierMode); + double actualError = statistics.LegacyConfidenceInterval.Margin; + + double maxError1 = effectiveMaxRelativeError * statistics.Mean; + double maxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue; + double maxError = Math.Min(maxError1, maxError2); + + if (iterationCounter >= minIterationCount && actualError < maxError) + { + return false; + } + + if (iterationCounter >= maxIterationCount || isOverhead && iterationCounter >= MaxOverheadIterationCount) + { + return false; + } + + return true; + } + } + + internal sealed class EngineActualStageSpecific(int maxIterationCount, IterationMode iterationMode) : EngineActualStage(iterationMode) + { + private int iterationCount = 0; + + internal override List GetMeasurementList() => new(maxIterationCount); + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + => ++iterationCount <= maxIterationCount; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs b/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs deleted file mode 100644 index 6394db3d1e..0000000000 --- a/src/BenchmarkDotNet/Engines/EngineGeneralStage.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System; -using System.Collections.Generic; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Mathematics; -using BenchmarkDotNet.Reports; -using Perfolizer.Horology; -using Perfolizer.Mathematics.OutlierDetection; - -namespace BenchmarkDotNet.Engines -{ - public class EngineActualStage : EngineStage - { - internal const int MaxOverheadIterationCount = 20; - private const double MaxOverheadRelativeError = 0.05; - private const int DefaultWorkloadCount = 10; - - private readonly int? iterationCount; - private readonly double maxRelativeError; - private readonly TimeInterval? maxAbsoluteError; - private readonly OutlierMode outlierMode; - private readonly int minIterationCount; - private readonly int maxIterationCount; - - public EngineActualStage(IEngine engine) : base(engine) - { - iterationCount = engine.TargetJob.ResolveValueAsNullable(RunMode.IterationCountCharacteristic); - maxRelativeError = engine.TargetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, engine.Resolver); - maxAbsoluteError = engine.TargetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); - outlierMode = engine.TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, engine.Resolver); - minIterationCount = engine.TargetJob.ResolveValue(RunMode.MinIterationCountCharacteristic, engine.Resolver); - maxIterationCount = engine.TargetJob.ResolveValue(RunMode.MaxIterationCountCharacteristic, engine.Resolver); - } - - public IReadOnlyList RunOverhead(long invokeCount, int unrollFactor) - => RunAuto(invokeCount, IterationMode.Overhead, unrollFactor); - - public IReadOnlyList RunWorkload(long invokeCount, int unrollFactor, bool forceSpecific = false) - => Run(invokeCount, IterationMode.Workload, false, unrollFactor, forceSpecific); - - internal IReadOnlyList Run(long invokeCount, IterationMode iterationMode, bool runAuto, int unrollFactor, bool forceSpecific = false) - => (runAuto || iterationCount == null) && !forceSpecific - ? RunAuto(invokeCount, iterationMode, unrollFactor) - : RunSpecific(invokeCount, iterationMode, iterationCount ?? DefaultWorkloadCount, unrollFactor); - - private List RunAuto(long invokeCount, IterationMode iterationMode, int unrollFactor) - { - var measurements = new List(maxIterationCount); - var measurementsForStatistics = new List(maxIterationCount); - - int iterationCounter = 0; - bool isOverhead = iterationMode == IterationMode.Overhead; - double effectiveMaxRelativeError = isOverhead ? MaxOverheadRelativeError : maxRelativeError; - while (true) - { - iterationCounter++; - var measurement = RunIteration(iterationMode, IterationStage.Actual, iterationCounter, invokeCount, unrollFactor); - measurements.Add(measurement); - measurementsForStatistics.Add(measurement); - - var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, outlierMode); - double actualError = statistics.LegacyConfidenceInterval.Margin; - - double maxError1 = effectiveMaxRelativeError * statistics.Mean; - double maxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue; - double maxError = Math.Min(maxError1, maxError2); - - if (iterationCounter >= minIterationCount && actualError < maxError) - break; - - if (iterationCounter >= maxIterationCount || isOverhead && iterationCounter >= MaxOverheadIterationCount) - break; - } - WriteLine(); - - return measurements; - } - - private List RunSpecific(long invokeCount, IterationMode iterationMode, int iterationCount, int unrollFactor) - { - var measurements = new List(iterationCount); - - for (int i = 0; i < iterationCount; i++) - measurements.Add(RunIteration(iterationMode, IterationStage.Actual, i + 1, invokeCount, unrollFactor)); - - WriteLine(); - - return measurements; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs index 6adf6d964f..43b543d5c1 100644 --- a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs +++ b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; @@ -8,130 +9,105 @@ namespace BenchmarkDotNet.Engines { // TODO: use clockResolution - internal class EnginePilotStage : EngineStage + internal abstract class EnginePilotStage(Job targetJob, IResolver resolver) : EngineStage(IterationStage.Pilot, IterationMode.Workload) { - public readonly struct PilotStageResult - { - public long PerfectInvocationCount { get; } - public IReadOnlyList Measurements { get; } + internal const long MaxInvokeCount = (long.MaxValue / 2 + 1) / 2; - public PilotStageResult(long perfectInvocationCount, List measurements) - { - PerfectInvocationCount = perfectInvocationCount; - Measurements = measurements; - } + protected readonly int unrollFactor = targetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, resolver); + protected readonly int minInvokeCount = targetJob.ResolveValue(AccuracyMode.MinInvokeCountCharacteristic, resolver); - public PilotStageResult(long perfectInvocationCount) - { - PerfectInvocationCount = perfectInvocationCount; - Measurements = Array.Empty(); - } + protected long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor; + + internal static EnginePilotStage GetStage(IEngine engine) + { + var targetJob = engine.TargetJob; + // If InvocationCount is specified, pilot stage should be skipped + return targetJob.HasValue(RunMode.InvocationCountCharacteristic) ? null + // Here we want to guess "perfect" amount of invocation + : targetJob.HasValue(RunMode.IterationTimeCharacteristic) ? new EnginePilotStageSpecific(targetJob, engine.Resolver) + : new EnginePilotStageAuto(targetJob, engine.Resolver); } + } - internal const long MaxInvokeCount = (long.MaxValue / 2 + 1) / 2; + internal sealed class EnginePilotStageAuto(Job targetJob, IResolver resolver) : EnginePilotStage(targetJob, resolver) + { + private readonly TimeInterval minIterationTime = targetJob.ResolveValue(AccuracyMode.MinIterationTimeCharacteristic, resolver); + private readonly double maxRelativeError = targetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, resolver); + private readonly TimeInterval? maxAbsoluteError = targetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); + private readonly double resolution = targetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, resolver).GetResolution().Nanoseconds; - private readonly int unrollFactor; - private readonly TimeInterval minIterationTime; - private readonly int minInvokeCount; - private readonly double maxRelativeError; - private readonly TimeInterval? maxAbsoluteError; - private readonly double targetIterationTime; - private readonly double resolution; + internal override List GetMeasurementList() => []; - public EnginePilotStage(IEngine engine) : base(engine) + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) { - unrollFactor = engine.TargetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, engine.Resolver); - minIterationTime = engine.TargetJob.ResolveValue(AccuracyMode.MinIterationTimeCharacteristic, engine.Resolver); - minInvokeCount = engine.TargetJob.ResolveValue(AccuracyMode.MinInvokeCountCharacteristic, engine.Resolver); - maxRelativeError = engine.TargetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, engine.Resolver); - maxAbsoluteError = engine.TargetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic); - targetIterationTime = engine.TargetJob.ResolveValue(RunMode.IterationTimeCharacteristic, engine.Resolver).ToNanoseconds(); - resolution = engine.TargetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, engine.Resolver).GetResolution().Nanoseconds; - } + if (measurements.Count == 0) + { + invokeCount = Autocorrect(minInvokeCount); + return true; + } - /// Perfect invocation count - public PilotStageResult Run() - { - // If InvocationCount is specified, pilot stage should be skipped - if (TargetJob.HasValue(RunMode.InvocationCountCharacteristic)) - return new PilotStageResult(TargetJob.Run.InvocationCount); + var measurement = measurements[measurements.Count - 1]; + double iterationTime = measurement.Nanoseconds; + double operationError = 2.0 * resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision - // Here we want to guess "perfect" amount of invocation - return TargetJob.HasValue(RunMode.IterationTimeCharacteristic) - ? RunSpecific() - : RunAuto(); - } + // Max acceptable operation error + double operationMaxError1 = iterationTime / invokeCount * maxRelativeError; + double operationMaxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue; + double operationMaxError = Math.Min(operationMaxError1, operationMaxError2); - /// - /// A case where we don't have specific iteration time. - /// - private PilotStageResult RunAuto() - { - long invokeCount = Autocorrect(minInvokeCount); - var measurements = new List(); + bool isFinished = operationError < operationMaxError && iterationTime >= minIterationTime.Nanoseconds; + if (isFinished || invokeCount >= MaxInvokeCount) + { + return false; + } - int iterationCounter = 0; - while (true) + if (unrollFactor == 1 && invokeCount < EnvironmentResolver.DefaultUnrollFactorForThroughput) { - iterationCounter++; - var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor); - measurements.Add(measurement); - double iterationTime = measurement.Nanoseconds; - double operationError = 2.0 * resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision - - // Max acceptable operation error - double operationMaxError1 = iterationTime / invokeCount * maxRelativeError; - double operationMaxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue; - double operationMaxError = Math.Min(operationMaxError1, operationMaxError2); - - bool isFinished = operationError < operationMaxError && iterationTime >= minIterationTime.Nanoseconds; - if (isFinished) - break; - if (invokeCount >= MaxInvokeCount) - break; - - if (unrollFactor == 1 && invokeCount < EnvironmentResolver.DefaultUnrollFactorForThroughput) - invokeCount += 1; - else - invokeCount *= 2; + ++invokeCount; + } + else + { + invokeCount *= 2; } - WriteLine(); - return new PilotStageResult(invokeCount, measurements); + return true; } + } - /// - /// A case where we have specific iteration time. - /// - private PilotStageResult RunSpecific() - { - long invokeCount = Autocorrect(Engine.MinInvokeCount); - var measurements = new List(); + internal sealed class EnginePilotStageSpecific(Job targetJob, IResolver resolver) : EnginePilotStage(targetJob, resolver) + { + private const int MinInvokeCount = 4; - int iterationCounter = 0; + private readonly double targetIterationTime = targetJob.ResolveValue(RunMode.IterationTimeCharacteristic, resolver).ToNanoseconds(); - int downCount = 0; // Amount of iterations where newInvokeCount < invokeCount - while (true) + private int _downCount = 0; // Amount of iterations where newInvokeCount < invokeCount + + internal override List GetMeasurementList() => []; + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + { + if (measurements.Count == 0) { - iterationCounter++; - var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor); - measurements.Add(measurement); - double actualIterationTime = measurement.Nanoseconds; - long newInvokeCount = Autocorrect(Math.Max(minInvokeCount, (long)Math.Round(invokeCount * targetIterationTime / actualIterationTime))); + invokeCount = Autocorrect(MinInvokeCount); + return true; + } - if (newInvokeCount < invokeCount) - downCount++; + var measurement = measurements[measurements.Count - 1]; + double actualIterationTime = measurement.Nanoseconds; + long newInvokeCount = Autocorrect(Math.Max(minInvokeCount, (long) Math.Round(invokeCount * targetIterationTime / actualIterationTime))); - if (Math.Abs(newInvokeCount - invokeCount) <= 1 || downCount >= 3) - break; + if (newInvokeCount < invokeCount) + { + _downCount++; + } - invokeCount = newInvokeCount; + if (Math.Abs(newInvokeCount - invokeCount) <= 1 || _downCount >= 3) + { + return false; } - WriteLine(); - return new PilotStageResult(invokeCount, measurements); + invokeCount = newInvokeCount; + return true; } - - private long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineStage.cs b/src/BenchmarkDotNet/Engines/EngineStage.cs index 2c3b825394..82b6bc5fb3 100644 --- a/src/BenchmarkDotNet/Engines/EngineStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineStage.cs @@ -1,48 +1,43 @@ -using System; -using System.Collections.Generic; -using BenchmarkDotNet.Jobs; +using System.Collections.Generic; +using System.Runtime.CompilerServices; using BenchmarkDotNet.Reports; namespace BenchmarkDotNet.Engines { - public class EngineStage + internal abstract class EngineStage(IterationStage stage, IterationMode mode) { - private readonly IEngine engine; + internal readonly IterationStage Stage = stage; + internal readonly IterationMode Mode = mode; - protected EngineStage(IEngine engine) => this.engine = engine; + internal abstract List GetMeasurementList(); + internal abstract bool GetShouldRunIteration(List measurements, ref long invokeCount); - protected Job TargetJob => engine.TargetJob; - - protected Measurement RunIteration(IterationMode mode, IterationStage stage, int index, long invokeCount, int unrollFactor) - { - if (invokeCount % unrollFactor != 0) - throw new ArgumentOutOfRangeException($"InvokeCount({invokeCount}) should be a multiple of UnrollFactor({unrollFactor})."); - return engine.RunIteration(new IterationData(mode, stage, index, invokeCount, unrollFactor)); - } - - internal List Run(IStoppingCriteria criteria, long invokeCount, IterationMode mode, IterationStage stage, int unrollFactor) + [MethodImpl(MethodImplOptions.NoInlining)] + internal static IEnumerable EnumerateStages(IEngine engine, RunStrategy strategy, bool evaluateOverhead) { - var measurements = new List(criteria.MaxIterationCount); - if (criteria.Evaluate(measurements).IsFinished) - { - WriteLine(); - return measurements; - } + // It might be possible to add the jitting stage to this, but it's done in EngineFactory.CreateReadyToRun for now. - int iterationCounter = 0; - while (true) + if (strategy != RunStrategy.ColdStart) { - iterationCounter++; - measurements.Add(RunIteration(mode, stage, iterationCounter, invokeCount, unrollFactor)); - if (criteria.Evaluate(measurements).IsFinished) - break; + if (strategy != RunStrategy.Monitoring) + { + var pilotStage = EnginePilotStage.GetStage(engine); + if (pilotStage != null) + { + yield return pilotStage; + } + + if (evaluateOverhead) + { + yield return EngineWarmupStage.GetOverhead(); + yield return EngineActualStage.GetOverhead(engine); + } + } + + yield return EngineWarmupStage.GetWorkload(engine, strategy); } - WriteLine(); - - return measurements; + yield return EngineActualStage.GetWorkload(engine, strategy); } - - protected void WriteLine() => engine.WriteLine(); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs b/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs index 08ec7ec729..822ea69101 100644 --- a/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs @@ -1,24 +1,78 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; namespace BenchmarkDotNet.Engines { - internal class EngineWarmupStage : EngineStage + internal abstract class EngineWarmupStage(IterationMode iterationMode) : EngineStage(IterationStage.Warmup, iterationMode) { - private readonly IEngine engine; + private const int MinOverheadIterationCount = 4; + internal const int MaxOverheadIterationCount = 10; - public EngineWarmupStage(IEngine engine) : base(engine) => this.engine = engine; + internal static EngineWarmupStage GetOverhead() + => new EngineWarmupStageAuto(IterationMode.Overhead, MinOverheadIterationCount, MaxOverheadIterationCount); - public IReadOnlyList RunOverhead(long invokeCount, int unrollFactor) - => Run(invokeCount, IterationMode.Overhead, unrollFactor, RunStrategy.Throughput); + internal static EngineWarmupStage GetWorkload(IEngine engine, RunStrategy runStrategy) + { + var job = engine.TargetJob; + var count = job.ResolveValueAsNullable(RunMode.WarmupCountCharacteristic); + if (count.HasValue && count.Value != EngineResolver.ForceAutoWarmup || runStrategy == RunStrategy.Monitoring) + { + return new EngineWarmupStageSpecific(count ?? 0, IterationMode.Workload); + } + + int minIterationCount = job.ResolveValue(RunMode.MinWarmupIterationCountCharacteristic, engine.Resolver); + int maxIterationCount = job.ResolveValue(RunMode.MaxWarmupIterationCountCharacteristic, engine.Resolver); + return new EngineWarmupStageAuto(IterationMode.Workload, minIterationCount, maxIterationCount); + } + } - public IReadOnlyList RunWorkload(long invokeCount, int unrollFactor, RunStrategy runStrategy) - => Run(invokeCount, IterationMode.Workload, unrollFactor, runStrategy); + internal sealed class EngineWarmupStageAuto(IterationMode iterationMode, int minIterationCount, int maxIterationCount) : EngineWarmupStage(iterationMode) + { + private const int MinFluctuationCount = 4; + + private readonly int minIterationCount = minIterationCount; + private readonly int maxIterationCount = maxIterationCount; - internal IReadOnlyList Run(long invokeCount, IterationMode iterationMode, int unrollFactor, RunStrategy runStrategy) + internal override List GetMeasurementList() => new(maxIterationCount); + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) { - var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(engine.TargetJob, engine.Resolver, iterationMode, runStrategy); - return Run(criteria, invokeCount, iterationMode, IterationStage.Warmup, unrollFactor); + int n = measurements.Count; + + if (n >= maxIterationCount) + { + return false; + } + if (n < minIterationCount) + { + return true; + } + + int direction = -1; // The default "pre-state" is "decrease mode" + int fluctuationCount = 0; + for (int i = 1; i < n; i++) + { + int nextDirection = Math.Sign(measurements[i].Nanoseconds - measurements[i - 1].Nanoseconds); + if (nextDirection != direction || nextDirection == 0) + { + direction = nextDirection; + fluctuationCount++; + } + } + + return fluctuationCount < MinFluctuationCount; } } + + internal sealed class EngineWarmupStageSpecific(int maxIterationCount, IterationMode iterationMode) : EngineWarmupStage(iterationMode) + { + private int iterationCount = 0; + + internal override List GetMeasurementList() => new(maxIterationCount); + + internal override bool GetShouldRunIteration(List measurements, ref long invokeCount) + => ++iterationCount <= maxIterationCount; + } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/GcStats.cs b/src/BenchmarkDotNet/Engines/GcStats.cs index ca60b0deea..3416d14ea4 100644 --- a/src/BenchmarkDotNet/Engines/GcStats.cs +++ b/src/BenchmarkDotNet/Engines/GcStats.cs @@ -1,5 +1,6 @@ using System; using System.Reflection; +using System.Runtime.CompilerServices; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -106,9 +107,10 @@ public int GetCollectionsCount(int generation) return AllocatedBytes <= AllocationQuantum ? 0L : AllocatedBytes; } + // Skip tier0 jit to make sure we don't get any unexpected allocations in this method. + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] public static GcStats ReadInitial() { - // this will force GC.Collect, so we want to do this before collecting collections counts long? allocatedBytes = GetAllocatedBytes(); return new GcStats( @@ -119,15 +121,14 @@ public static GcStats ReadInitial() 0); } + // Skip tier0 jit to make sure we don't get any unexpected allocations in this method. + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] public static GcStats ReadFinal() { return new GcStats( GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2), - - // this will force GC.Collect, so we want to do this after collecting collections counts - // to exclude this single full forced collection from results GetAllocatedBytes(), 0); } @@ -136,17 +137,16 @@ public static GcStats ReadFinal() public static GcStats FromForced(int forcedFullGarbageCollections) => new GcStats(forcedFullGarbageCollections, forcedFullGarbageCollections, forcedFullGarbageCollections, 0, 0); + // Skip tier0 jit to make sure we don't get any unexpected allocations in this method. + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] private static long? GetAllocatedBytes() { // we have no tests for WASM and don't want to risk introducing a new bug (https://github.com/dotnet/BenchmarkDotNet/issues/2226) if (RuntimeInformation.IsWasm) return null; - // "This instance Int64 property returns the number of bytes that have been allocated by a specific - // AppDomain. The number is accurate as of the last garbage collection." - CLR via C# - // so we enforce GC.Collect here just to make sure we get accurate results - GC.Collect(); - + // Do NOT call GC.Collect() here, as it causes finalizers to run and possibly allocate. https://github.com/dotnet/runtime/issues/101536#issuecomment-2077533242 + // Instead, we call it before we start the measurement in the Engine. #if NET6_0_OR_GREATER return GC.GetTotalAllocatedBytes(precise: true); #else @@ -218,9 +218,7 @@ private static long CalculateAllocationQuantumSize() break; } - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); + Engine.ForceGcCollect(); result = GC.GetTotalMemory(false); var tmp = new object(); diff --git a/src/BenchmarkDotNet/Engines/HostSignal.cs b/src/BenchmarkDotNet/Engines/HostSignal.cs index f4a2f75a22..cf423d2268 100644 --- a/src/BenchmarkDotNet/Engines/HostSignal.cs +++ b/src/BenchmarkDotNet/Engines/HostSignal.cs @@ -7,6 +7,11 @@ public enum HostSignal /// BeforeProcessStart, + /// + /// right after we start the benchmarking process + /// + AfterProcessStart, + /// /// before jitting, warmup /// diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/AutoWarmupStoppingCriteria.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/AutoWarmupStoppingCriteria.cs deleted file mode 100644 index e2282c7baa..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/AutoWarmupStoppingCriteria.cs +++ /dev/null @@ -1,97 +0,0 @@ -using System; -using System.Collections.Generic; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Engines -{ - /// - /// Automatically choose the best number of iterations during the warmup stage. - /// - public class AutoWarmupStoppingCriteria : StoppingCriteriaBase - { - private const int DefaultMinFluctuationCount = 4; - - private readonly int minIterationCount; - private readonly int maxIterationCount; - private readonly int minFluctuationCount; - - private readonly string maxIterationMessage; - private readonly string minFluctuationMessage; - - /// - /// The idea of the implementation is simple: if measurements monotonously decrease or increase, the steady state is not achieved; - /// we should continue the warmup stage. The evaluation method counts "fluctuations" - /// (3 consecutive measurements A,B,C where (A>B AND B<C) OR (A<B AND B>C)) until the required amount of flotations is observed. - /// - /// - /// The minimum number of iterations. - /// We are always going to do at least iterations regardless of fluctuations. - /// - /// - /// The maximum number of iterations. - /// We are always going to do at most iterations regardless of fluctuations. - /// - /// - /// The required number of fluctuations. - /// If the required number of fluctuations is achieved but the number of iterations less than , - /// we need more iterations. - /// If the required number of fluctuations is not achieved but the number of iterations equal to , - /// we should stop the iterations. - /// - public AutoWarmupStoppingCriteria(int minIterationCount, int maxIterationCount, int minFluctuationCount = DefaultMinFluctuationCount) - { - this.minIterationCount = minIterationCount; - this.maxIterationCount = maxIterationCount; - this.minFluctuationCount = minFluctuationCount; - - maxIterationMessage = $"The maximum amount of iteration ({maxIterationCount}) is achieved"; - minFluctuationMessage = $"The minimum amount of fluctuation ({minFluctuationCount}) and " + - $"the minimum amount of iterations ({minIterationCount}) are achieved"; - } - - public override StoppingResult Evaluate(IReadOnlyList measurements) - { - int n = measurements.Count; - - if (n >= maxIterationCount) - return StoppingResult.CreateFinished(maxIterationMessage); - if (n < minIterationCount) - return StoppingResult.NotFinished; - - int direction = -1; // The default "pre-state" is "decrease mode" - int fluctuationCount = 0; - for (int i = 1; i < n; i++) - { - int nextDirection = Math.Sign(measurements[i].Nanoseconds - measurements[i - 1].Nanoseconds); - if (nextDirection != direction || nextDirection == 0) - { - direction = nextDirection; - fluctuationCount++; - } - } - - return fluctuationCount >= minFluctuationCount - ? StoppingResult.CreateFinished(minFluctuationMessage) - : StoppingResult.NotFinished; - } - - protected override string GetTitle() => $"{nameof(AutoWarmupStoppingCriteria)}(" + - $"{nameof(minIterationCount)}={minIterationCount}, " + - $"{nameof(maxIterationCount)}={maxIterationCount}, " + - $"{nameof(minFluctuationCount)}={minFluctuationCount})"; - - protected override int GetMaxIterationCount() => maxIterationCount; - - protected override IEnumerable GetWarnings() - { - if (minIterationCount < 0) - yield return $"Min Iteration Count ({minIterationCount}) is negative"; - if (maxIterationCount < 0) - yield return $"Max Iteration Count ({maxIterationCount}) is negative"; - if (minFluctuationCount < 0) - yield return $"Min Fluctuation Count ({minFluctuationCount}) is negative"; - if (minIterationCount > maxIterationCount) - yield return $"Min Iteration Count ({minFluctuationCount}) is greater than Max Iteration Count ({maxIterationCount})"; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/DefaultStoppingCriteriaFactory.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/DefaultStoppingCriteriaFactory.cs deleted file mode 100644 index 356fc7362e..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/DefaultStoppingCriteriaFactory.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System; -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Jobs; - -namespace BenchmarkDotNet.Engines -{ - public class DefaultStoppingCriteriaFactory - { - public static readonly DefaultStoppingCriteriaFactory Instance = new DefaultStoppingCriteriaFactory(); - - private const int MinOverheadIterationCount = 4; - internal const int MaxOverheadIterationCount = 10; - - public virtual IStoppingCriteria CreateWarmupWorkload(Job job, IResolver resolver, RunStrategy runStrategy) - { - var count = job.ResolveValueAsNullable(RunMode.WarmupCountCharacteristic); - if (count.HasValue && count.Value != EngineResolver.ForceAutoWarmup || runStrategy == RunStrategy.Monitoring) - return new FixedStoppingCriteria(count ?? 0); - - int minIterationCount = job.ResolveValue(RunMode.MinWarmupIterationCountCharacteristic, resolver); - int maxIterationCount = job.ResolveValue(RunMode.MaxWarmupIterationCountCharacteristic, resolver); - return new AutoWarmupStoppingCriteria(minIterationCount, maxIterationCount); - } - - public virtual IStoppingCriteria CreateWarmupOverhead() - { - return new AutoWarmupStoppingCriteria(MinOverheadIterationCount, MaxOverheadIterationCount); - } - - public virtual IStoppingCriteria CreateWarmup(Job job, IResolver resolver, IterationMode mode, RunStrategy runStrategy) - { - switch (mode) - { - case IterationMode.Overhead: - return CreateWarmupOverhead(); - case IterationMode.Workload: - return CreateWarmupWorkload(job, resolver, runStrategy); - default: - throw new ArgumentOutOfRangeException(nameof(mode), mode, null); - } - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/FixedStoppingCriteria.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/FixedStoppingCriteria.cs deleted file mode 100644 index 847bb41a59..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/FixedStoppingCriteria.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Engines -{ - /// - /// Stopping criteria which require a specific amount of iterations. - /// - public class FixedStoppingCriteria : StoppingCriteriaBase - { - private readonly int iterationCount; - - private readonly string message; - - public FixedStoppingCriteria(int iterationCount) - { - this.iterationCount = iterationCount; - - message = $"The required amount of iteration ({iterationCount}) is achieved"; - } - - public override StoppingResult Evaluate(IReadOnlyList measurements) - => measurements.Count >= MaxIterationCount - ? StoppingResult.CreateFinished(message) - : StoppingResult.NotFinished; - - protected override string GetTitle() => $"{nameof(FixedStoppingCriteria)}({nameof(iterationCount)}={iterationCount})"; - - protected override int GetMaxIterationCount() => iterationCount; - - protected override IEnumerable GetWarnings() - { - if (iterationCount < 0) - yield return $"Iteration count ({iterationCount}) is negative"; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/IStoppingCriteria.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/IStoppingCriteria.cs deleted file mode 100644 index 53550ef919..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/IStoppingCriteria.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Engines -{ - /// - /// A stopping criteria checks when it's time to terminate iteration in the current stage. - /// - public interface IStoppingCriteria - { - /// - /// Checks do we have enough iterations - /// - StoppingResult Evaluate(IReadOnlyList measurements); - - /// - /// Title which can be used in logs and diagnostics methods - /// - string Title { get; } - - /// - /// The maximum possible count of iterations. - /// Engine needs this value for setting the maximum capacity of the returned list of measurements. - /// The correct capacity helps to avoid infrastructure allocations during benchmarking. - /// - int MaxIterationCount { get; } - - /// - /// An array of user-friendly warnings which notify about incorrect parameters. - /// - IReadOnlyList Warnings { get; } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingCriteriaBase.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingCriteriaBase.cs deleted file mode 100644 index 5956e37760..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingCriteriaBase.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Engines -{ - public abstract class StoppingCriteriaBase : IStoppingCriteria - { - private readonly Lazy lazyTitle; - private readonly Lazy lazyMaxIterationCount; - private readonly Lazy lazyWarnings; - - public string Title => lazyTitle.Value; - public int MaxIterationCount => lazyMaxIterationCount.Value; - public IReadOnlyList Warnings => lazyWarnings.Value; - - protected StoppingCriteriaBase() - { - lazyTitle = new Lazy(GetTitle); - lazyMaxIterationCount = new Lazy(GetMaxIterationCount); - lazyWarnings = new Lazy(() => GetWarnings().ToArray()); - } - - public abstract StoppingResult Evaluate(IReadOnlyList measurements); - - protected abstract string GetTitle(); - - protected abstract int GetMaxIterationCount(); - - protected abstract IEnumerable GetWarnings(); - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingResult.cs b/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingResult.cs deleted file mode 100644 index a51287c4b4..0000000000 --- a/src/BenchmarkDotNet/Engines/StoppingCriteria/StoppingResult.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace BenchmarkDotNet.Engines -{ - public struct StoppingResult - { - public readonly bool IsFinished; - - public readonly string? Message; - - private StoppingResult(bool isFinished, string? message) - { - IsFinished = isFinished; - Message = message; - } - - public static readonly StoppingResult NotFinished = new StoppingResult(false, null); - public static StoppingResult CreateFinished(string message) => new StoppingResult(true, message); - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs b/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs index e13d50b650..f2e2f34e26 100644 --- a/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs +++ b/src/BenchmarkDotNet/Environments/BenchmarkEnvironmentInfo.cs @@ -2,10 +2,10 @@ using System.Diagnostics; using System.Linq; using System.Runtime; +using BenchmarkDotNet.Detectors.Cpu; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Portability.Cpu; using BenchmarkDotNet.Properties; using BenchmarkDotNet.Validators; using JetBrains.Annotations; diff --git a/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs b/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs index 7983f82ebc..8291354090 100644 --- a/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs +++ b/src/BenchmarkDotNet/Environments/HostEnvironmentInfo.cs @@ -2,15 +2,18 @@ using System.Collections.Generic; using System.Linq; using System.Text; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Models; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Portability.Cpu; using BenchmarkDotNet.Properties; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Toolchains.DotNetCli; using JetBrains.Annotations; +using Perfolizer.Helpers; using Perfolizer.Horology; +using Perfolizer.Models; using Perfolizer.Metrology; namespace BenchmarkDotNet.Environments @@ -32,15 +35,12 @@ public class HostEnvironmentInfo : BenchmarkEnvironmentInfo public string BenchmarkDotNetVersion { get; protected set; } - /// - /// Could be expensive - /// - public Lazy OsVersion { get; protected set; } - /// /// is expensive to call (1s) /// - public Lazy CpuInfo { get; protected set; } + public Lazy Cpu { get; protected set; } + + public Lazy Os { get; protected set; } /// /// .NET Core SDK version @@ -65,35 +65,36 @@ public class HostEnvironmentInfo : BenchmarkEnvironmentInfo public Lazy> AntivirusProducts { get; } - public Lazy VirtualMachineHypervisor { get; protected set; } + // TODO: Join with OsInfo + public Lazy VirtualMachineHypervisor { get; protected set; } protected HostEnvironmentInfo() { BenchmarkDotNetVersion = BenchmarkDotNetInfo.Instance.BrandVersion; - OsVersion = new Lazy(RuntimeInformation.GetOsVersion); - CpuInfo = new Lazy(RuntimeInformation.GetCpuInfo); ChronometerFrequency = Chronometer.Frequency; HardwareTimerKind = Chronometer.HardwareTimerKind; DotNetSdkVersion = new Lazy(DotNetCliCommandExecutor.GetDotNetSdkVersion); IsMonoInstalled = new Lazy(() => !string.IsNullOrEmpty(ProcessHelper.RunAndReadOutput("mono", "--version"))); AntivirusProducts = new Lazy>(RuntimeInformation.GetAntivirusProducts); VirtualMachineHypervisor = new Lazy(RuntimeInformation.GetVirtualMachineHypervisor); + Os = new Lazy(OsDetector.GetOs); + Cpu = new Lazy(() => CpuDetector.CrossPlatform.Detect() ?? CpuInfo.Unknown); } public new static HostEnvironmentInfo GetCurrent() => current ??= new HostEnvironmentInfo(); public override IEnumerable ToFormattedString() { - string vmName = VirtualMachineHypervisor.Value?.Name; + string? vmName = VirtualMachineHypervisor.Value?.Name; if (!string.IsNullOrEmpty(vmName)) - yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {OsVersion.Value} ({vmName})"; + yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {Os.Value.ToBrandString()} ({vmName})"; else if (RuntimeInformation.IsRunningInContainer) - yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {OsVersion.Value} (container)"; + yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {Os.Value.ToBrandString()} (container)"; else - yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {OsVersion.Value}"; + yield return $"{BenchmarkDotNetCaption} v{BenchmarkDotNetVersion}, {Os.Value.ToBrandString()}"; - yield return CpuInfoFormatter.Format(CpuInfo.Value); + yield return Cpu.Value.ToFullBrandName(); if (HardwareTimerKind != HardwareTimerKind.Unknown) { string frequency = ChronometerFrequency.ToString(FrequencyUnit.Hz, unitPresentation: UnitHelper.DefaultPresentation); @@ -104,7 +105,7 @@ public override IEnumerable ToFormattedString() if (RuntimeInformation.IsNetCore && IsDotNetCliInstalled()) { - // this wonderful version number contains words like "preview" and ... 5 segments so it can not be parsed by Version.Parse. Example: "5.0.100-preview.8.20362.3" + // this wonderful version number contains words like "preview" and ... 5 segments, so it can not be parsed by Version.Parse. Example: "5.0.100-preview.8.20362.3" if (int.TryParse(new string(DotNetSdkVersion.Value.TrimStart().TakeWhile(char.IsDigit).ToArray()), out int major) && major >= 5) yield return $".NET SDK {DotNetSdkVersion.Value}"; else @@ -131,5 +132,18 @@ public static string GetInformation() sb.AppendLine(Summary.BuildAllRuntimes(hostEnvironmentInfo, Array.Empty())); return sb.ToString(); } + + internal BdnHostInfo ToPerfonar() => new() + { + Cpu = Cpu.Value, + Os = Os.Value, + RuntimeVersion = RuntimeVersion, + HasAttachedDebugger = HasAttachedDebugger, + HasRyuJit = HasRyuJit, + Configuration = Configuration, + DotNetSdkVersion = DotNetSdkVersion.Value, + ChronometerFrequency = ChronometerFrequency.Hertz, + HardwareTimerKind = HardwareTimerKind.ToString() + }; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs b/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs index 865c6f64f3..dfeaf3fd47 100644 --- a/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs +++ b/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs @@ -17,7 +17,10 @@ private InfrastructureResolver() Register(InfrastructureMode.BuildConfigurationCharacteristic, () => InfrastructureMode.ReleaseConfigurationName); Register(InfrastructureMode.ArgumentsCharacteristic, Array.Empty); + +#pragma warning disable CS0618 // Type or member is obsolete Register(InfrastructureMode.NuGetReferencesCharacteristic, Array.Empty); +#pragma warning restore CS0618 // Type or member is obsolete } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/OsBrandStringHelper.cs b/src/BenchmarkDotNet/Environments/OsBrandStringHelper.cs deleted file mode 100644 index 045c1f7543..0000000000 --- a/src/BenchmarkDotNet/Environments/OsBrandStringHelper.cs +++ /dev/null @@ -1,278 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using BenchmarkDotNet.Extensions; - -namespace BenchmarkDotNet.Environments -{ - [SuppressMessage("ReSharper", "ClassNeverInstantiated.Global")] - public class OsBrandStringHelper - { - // See https://en.wikipedia.org/wiki/Ver_(command) - // See https://docs.microsoft.com/en-us/windows/release-health/release-information - // See https://docs.microsoft.com/en-us/windows/release-health/windows11-release-information - private static readonly Dictionary WindowsBrandVersions = new Dictionary - { - { "1.04", "1.0" }, - { "2.11", "2.0" }, - { "3", "3.0" }, - { "3.10.528", "NT 3.1" }, - { "3.11", "for Workgroups 3.11" }, - { "3.50.807", "NT 3.5" }, - { "3.51.1057", "NT 3.51" }, - { "4.00.950", "95" }, - { "4.00.1111", "95 OSR2" }, - { "4.03.1212-1214", "95 OSR2.1" }, - { "4.03.1214", "95 OSR2.5" }, - { "4.00.1381", "NT 4.0" }, - { "4.10.1998", "98" }, - { "4.10.2222", "98 SE" }, - { "4.90.2380.2", "ME Beta" }, - { "4.90.2419", "ME Beta 2" }, - { "4.90.3000", "ME" }, - { "5.00.1515", "NT 5.0 Beta" }, - { "5.00.2031", "2000 Beta 3" }, - { "5.00.2128", "2000 RC2" }, - { "5.00.2183", "2000 RC3" }, - { "5.00.2195", "2000" }, - { "5.0.2195", "2000 Professional" }, - { "5.1.2505", "XP RC1" }, - { "5.1.2600", "XP" }, - { "5.1.2600.1105-1106", "XP SP1" }, - { "5.1.2600.2180", "XP SP2" }, - { "5.2.3541", ".NET Server interim" }, - { "5.2.3590", ".NET Server Beta 3" }, - { "5.2.3660", ".NET Server RC1" }, - { "5.2.3718", ".NET Server 2003 RC2" }, - { "5.2.3763", "Server 2003 Beta" }, - { "5.2.3790", "XP Professional x64 Edition" }, - { "5.2.3790.1180", "Server 2003 SP1" }, - { "5.2.3790.1218", "Server 2003" }, - { "6.0.5048", "Longhorn" }, - { "6.0.5112", "Vista Beta 1" }, - { "6.0.5219", "Vista CTP" }, - { "6.0.5259", "Vista TAP Preview" }, - { "6.0.5270", "Vista CTP December" }, - { "6.0.5308", "Vista CTP February" }, - { "6.0.5342", "Vista CTP Refresh" }, - { "6.0.5365", "Vista April EWD" }, - { "6.0.5381", "Vista Beta 2 Preview" }, - { "6.0.5384", "Vista Beta 2" }, - { "6.0.5456", "Vista Pre-RC1 Build 5456" }, - { "6.0.5472", "Vista Pre-RC1 Build 5472" }, - { "6.0.5536", "Vista Pre-RC1 Build 5536" }, - { "6.0.5600.16384", "Vista RC1" }, - { "6.0.5700", "Vista Pre-RC2" }, - { "6.0.5728", "Vista Pre-RC2 Build 5728" }, - { "6.0.5744.16384", "Vista RC2" }, - { "6.0.5808", "Vista Pre-RTM Build 5808" }, - { "6.0.5824", "Vista Pre-RTM Build 5824" }, - { "6.0.5840", "Vista Pre-RTM Build 5840" }, - { "6.0.6000", "Vista" }, - { "6.0.6000.16386", "Vista RTM" }, - { "6.0.6001", "Vista SP1" }, - { "6.0.6002", "Vista SP2" }, - { "6.1.7600", "7" }, - { "6.1.7600.16385", "7" }, - { "6.1.7601", "7 SP1" }, - { "6.1.8400", "Home Server 2011" }, - { "6.2.8102", "8 Developer Preview" }, - { "6.2.9200", "8" }, - { "6.2.9200.16384", "8 RTM" }, - { "6.2.10211", "Phone 8" }, - { "6.3.9600", "8.1" }, - { "6.4.9841", "10 Technical Preview 1" }, - { "6.4.9860", "10 Technical Preview 2" }, - { "6.4.9879", "10 Technical Preview 3" }, - { "10.0.9926", "10 Technical Preview 4" }, - { "10.0.10041", "10 Technical Preview 5" }, - { "10.0.10049", "10 Technical Preview 6" }, - { "10.0.10240", "10 Threshold 1 [1507, RTM]" }, - { "10.0.10586", "10 Threshold 2 [1511, November Update]" }, - { "10.0.14393", "10 Redstone 1 [1607, Anniversary Update]" }, - { "10.0.15063", "10 Redstone 2 [1703, Creators Update]" }, - { "10.0.16299", "10 Redstone 3 [1709, Fall Creators Update]" }, - { "10.0.17134", "10 Redstone 4 [1803, April 2018 Update]" }, - { "10.0.17763", "10 Redstone 5 [1809, October 2018 Update]" }, - { "10.0.18362", "10 19H1 [1903, May 2019 Update]" }, - { "10.0.18363", "10 19H2 [1909, November 2019 Update]" }, - { "10.0.19041", "10 20H1 [2004, May 2020 Update]" }, - { "10.0.19042", "10 20H2 [20H2, October 2020 Update]" }, - { "10.0.19043", "10 21H1 [21H1, May 2021 Update]" }, - { "10.0.19044", "10 21H2 [21H2, November 2021 Update]" }, - { "10.0.19045", "10 22H2 [22H2, 2022 Update]" }, - { "10.0.22000", "11 21H2 [21H2, Sun Valley]" }, - { "10.0.22621", "11 22H2 [22H2, Sun Valley 2]" }, - }; - - private class Windows1XVersion - { - private string? CodeVersion { get; } - private string? CodeName { get; } - private string? MarketingName { get; } - private int BuildNumber { get; } - - private string MarketingNumber => BuildNumber >= 22000 ? "11" : "10"; - private string? ShortifiedCodeName => CodeName?.Replace(" ", ""); - private string? ShortifiedMarketingName => MarketingName?.Replace(" ", ""); - - private Windows1XVersion(string? codeVersion, string? codeName, string? marketingName, int buildNumber) - { - CodeVersion = codeVersion; - CodeName = codeName; - MarketingName = marketingName; - BuildNumber = buildNumber; - } - - private string ToFullVersion(int? ubr = null) - => ubr == null ? $"10.0.{BuildNumber}" : $"10.0.{BuildNumber}.{ubr}"; - - private static string Collapse(params string[] values) => string.Join("/", values.Where(v => !string.IsNullOrEmpty(v))); - - // The line with OsBrandString is one of the longest lines in the summary. - // When people past in on GitHub, it can be a reason of an ugly horizontal scrollbar. - // To avoid this, we are trying to minimize this line and use the minimum possible number of characters. - public string ToPrettifiedString(int? ubr) - => CodeVersion == ShortifiedCodeName - ? $"{MarketingNumber} ({Collapse(ToFullVersion(ubr), CodeVersion, ShortifiedMarketingName)})" - : $"{MarketingNumber} ({Collapse(ToFullVersion(ubr), CodeVersion, ShortifiedMarketingName, ShortifiedCodeName)})"; - - // See https://en.wikipedia.org/wiki/Windows_10_version_history - // See https://en.wikipedia.org/wiki/Windows_11_version_history - private static readonly List WellKnownVersions = new () - { - // Windows 10 - new Windows1XVersion("1507", "Threshold 1", "RTM", 10240), - new Windows1XVersion("1511", "Threshold 2", "November Update", 10586), - new Windows1XVersion("1607", "Redstone 1", "Anniversary Update", 14393), - new Windows1XVersion("1703", "Redstone 2", "Creators Update", 15063), - new Windows1XVersion("1709", "Redstone 3", "Fall Creators Update", 16299), - new Windows1XVersion("1803", "Redstone 4", "April 2018 Update", 17134), - new Windows1XVersion("1809", "Redstone 5", "October 2018 Update", 17763), - new Windows1XVersion("1903", "19H1", "May 2019 Update", 18362), - new Windows1XVersion("1909", "19H2", "November 2019 Update", 18363), - new Windows1XVersion("2004", "20H1", "May 2020 Update", 19041), - new Windows1XVersion("20H2", "20H2", "October 2020 Update", 19042), - new Windows1XVersion("21H1", "21H1", "May 2021 Update", 19043), - new Windows1XVersion("21H2", "21H2", "November 2021 Update", 19044), - new Windows1XVersion("22H2", "22H2", "2022 Update", 19045), - // Windows 11 - new Windows1XVersion("21H2", "Sun Valley", null, 22000), - new Windows1XVersion("22H2", "Sun Valley 2", "2022 Update", 22621), - new Windows1XVersion("23H2", "Sun Valley 3", "2023 Update", 22631), - }; - - public static Windows1XVersion? Resolve(string osVersionString) - { - var windows1XVersion = WellKnownVersions.FirstOrDefault(v => osVersionString == $"10.0.{v.BuildNumber}"); - if (windows1XVersion != null) - return windows1XVersion; - if (Version.TryParse(osVersionString, out var osVersion)) - { - if (osVersion.Major == 10 && osVersion.Minor == 0) - return new Windows1XVersion(null, null, null, osVersion.Build); - } - return null; - } - } - - /// - /// Transform an operation system name and version to a nice form for summary. - /// - /// Original operation system name - /// Original operation system version - /// UBR (Update Build Revision), the revision number of Windows version (if available) - /// Prettified operation system title - public static string Prettify(string osName, string osVersion, int? windowsUbr = null) - { - if (osName == "Windows") - return PrettifyWindows(osVersion, windowsUbr); - return $"{osName} {osVersion}"; - } - - private static string PrettifyWindows(string osVersion, int? windowsUbr) - { - var windows1XVersion = Windows1XVersion.Resolve(osVersion); - if (windows1XVersion != null) - return "Windows " + windows1XVersion.ToPrettifiedString(windowsUbr); - - string brandVersion = WindowsBrandVersions.GetValueOrDefault(osVersion); - string completeOsVersion = windowsUbr != null && osVersion.Count(c => c == '.') == 2 - ? osVersion + "." + windowsUbr - : osVersion; - string fullVersion = brandVersion == null ? osVersion : brandVersion + " (" + completeOsVersion + ")"; - return "Windows " + fullVersion; - } - - private class MacOSXVersion - { - private int DarwinVersion { get; } - private string CodeName { get; } - - private MacOSXVersion(int darwinVersion, string codeName) - { - DarwinVersion = darwinVersion; - CodeName = codeName; - } - - private static readonly List WellKnownVersions = new List - { - new MacOSXVersion(6, "Jaguar"), - new MacOSXVersion(7, "Panther"), - new MacOSXVersion(8, "Tiger"), - new MacOSXVersion(9, "Leopard"), - new MacOSXVersion(10, "Snow Leopard"), - new MacOSXVersion(11, "Lion"), - new MacOSXVersion(12, "Mountain Lion"), - new MacOSXVersion(13, "Mavericks"), - new MacOSXVersion(14, "Yosemite"), - new MacOSXVersion(15, "El Capitan"), - new MacOSXVersion(16, "Sierra"), - new MacOSXVersion(17, "High Sierra"), - new MacOSXVersion(18, "Mojave"), - new MacOSXVersion(19, "Catalina"), - new MacOSXVersion(20, "Big Sur"), - new MacOSXVersion(21, "Monterey"), - new MacOSXVersion(22, "Ventura"), - new MacOSXVersion(23, "Sonoma"), - new MacOSXVersion(24, "Sequoia"), - }; - - public static string? ResolveCodeName(string kernelVersion) - { - if (string.IsNullOrWhiteSpace(kernelVersion)) - return null; - - kernelVersion = kernelVersion.ToLowerInvariant().Trim(); - if (kernelVersion.StartsWith("darwin")) - kernelVersion = kernelVersion.Substring(6).Trim(); - var numbers = kernelVersion.Split('.'); - if (numbers.Length == 0) - return null; - - string majorVersionStr = numbers[0]; - if (int.TryParse(majorVersionStr, out int majorVersion)) - return WellKnownVersions.FirstOrDefault(v => v.DarwinVersion == majorVersion)?.CodeName; - return null; - } - } - - public static string PrettifyMacOSX(string systemVersion, string kernelVersion) - { - string codeName = MacOSXVersion.ResolveCodeName(kernelVersion); - if (codeName != null) - { - int firstDigitIndex = systemVersion.IndexOfAny("0123456789".ToCharArray()); - if (firstDigitIndex == -1) - return $"{systemVersion} {codeName} [{kernelVersion}]"; - - string systemVersionTitle = systemVersion.Substring(0, firstDigitIndex).Trim(); - string systemVersionNumbers = systemVersion.Substring(firstDigitIndex).Trim(); - return $"{systemVersionTitle} {codeName} {systemVersionNumbers} [{kernelVersion}]"; - } - - return $"{systemVersion} [{kernelVersion}]"; - } - } -} diff --git a/src/BenchmarkDotNet/Environments/Platform.cs b/src/BenchmarkDotNet/Environments/Platform.cs index bc8f1ad26b..434a84ff98 100644 --- a/src/BenchmarkDotNet/Environments/Platform.cs +++ b/src/BenchmarkDotNet/Environments/Platform.cs @@ -51,5 +51,10 @@ public enum Platform /// A PowerPC 64-bit (little-endian) processor architecture. /// Ppc64le, + + /// + /// A RiscV 64-bit processor architecture. + /// + RiscV64, } } diff --git a/src/BenchmarkDotNet/Environments/ProcessorBrandStringHelper.cs b/src/BenchmarkDotNet/Environments/ProcessorBrandStringHelper.cs deleted file mode 100644 index 882078df74..0000000000 --- a/src/BenchmarkDotNet/Environments/ProcessorBrandStringHelper.cs +++ /dev/null @@ -1,155 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Text.RegularExpressions; -using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Environments -{ - public static class ProcessorBrandStringHelper - { - /// - /// Transform a processor brand string to a nice form for summary. - /// - /// The CPU information - /// Whether to include determined max frequency information - /// Prettified version - public static string Prettify(CpuInfo cpuInfo, bool includeMaxFrequency = false) - { - if (cpuInfo == null || string.IsNullOrEmpty(cpuInfo.ProcessorName)) - { - return "Unknown processor"; - } - - // Remove parts which don't provide any useful information for user - var processorName = cpuInfo.ProcessorName.Replace("@", "").Replace("(R)", "").Replace("(TM)", ""); - - // If we have found physical core(s), we can safely assume we can drop extra info from brand - if (cpuInfo.PhysicalCoreCount.HasValue && cpuInfo.PhysicalCoreCount.Value > 0) - processorName = Regex.Replace(processorName, @"(\w+?-Core Processor)", "").Trim(); - - string frequencyString = GetBrandStyledActualFrequency(cpuInfo.NominalFrequency); - if (includeMaxFrequency && frequencyString != null && !processorName.Contains(frequencyString)) - { - // show Max only if there's already a frequency name to differentiate the two - string maxFrequency = processorName.Contains("Hz") ? $"(Max: {frequencyString})" : frequencyString; - processorName = $"{processorName} {maxFrequency}"; - } - - // Remove double spaces - processorName = Regex.Replace(processorName.Trim(), @"\s+", " "); - - // Add microarchitecture name if known - string microarchitecture = ParseMicroarchitecture(processorName); - if (microarchitecture != null) - processorName = $"{processorName} ({microarchitecture})"; - - return processorName; - } - - /// - /// Presents actual processor's frequency into brand string format - /// - /// - private static string GetBrandStyledActualFrequency(Frequency? frequency) - { - if (frequency == null) - return null; - return $"{frequency.Value.ToGHz().ToString("N2", DefaultCultureInfo.Instance)}GHz"; - } - - /// - /// Parse a processor name and tries to return a microarchitecture name. - /// Works only for well-known microarchitectures. - /// - private static string? ParseMicroarchitecture(string processorName) - { - if (processorName.StartsWith("Intel Core")) - { - string model = processorName.Substring("Intel Core".Length).Trim(); - - // Core i3/5/7/9 - if ( - model.Length > 4 && - model[0] == 'i' && - (model[1] == '3' || model[1] == '5' || model[1] == '7' || model[1] == '9') && - (model[2] == '-' || model[2] == ' ')) - { - string modelNumber = model.Substring(3); - if (modelNumber.StartsWith("CPU")) - modelNumber = modelNumber.Substring(3).Trim(); - if (modelNumber.Contains("CPU")) - modelNumber = modelNumber.Substring(0, modelNumber.IndexOf("CPU", StringComparison.Ordinal)).Trim(); - return ParseIntelCoreMicroarchitecture(modelNumber); - } - } - - return null; - } - - private static readonly Lazy> KnownMicroarchitectures = new Lazy>(() => - { - var data = ResourceHelper.LoadResource("BenchmarkDotNet.Environments.microarchitectures.txt").Split('\r', '\n'); - var dictionary = new Dictionary(); - string? currentMicroarchitecture = null; - foreach (string line in data) - { - if (line.StartsWith("//") || string.IsNullOrWhiteSpace(line)) - continue; - if (line.StartsWith("#")) - { - currentMicroarchitecture = line.Substring(1).Trim(); - continue; - } - - string modelNumber = line.Trim(); - if (dictionary.ContainsKey(modelNumber)) - throw new Exception($"{modelNumber} is defined twice in microarchitectures.txt"); - if (currentMicroarchitecture == null) - throw new Exception($"{modelNumber} doesn't have defined microarchitecture in microarchitectures.txt"); - dictionary[modelNumber] = currentMicroarchitecture; - } - - return dictionary; - }); - - // see http://www.intel.com/content/www/us/en/processors/processor-numbers.html - [SuppressMessage("ReSharper", "StringLiteralTypo")] - internal static string? ParseIntelCoreMicroarchitecture(string modelNumber) - { - if (KnownMicroarchitectures.Value.TryGetValue(modelNumber, out string? microarchitecture)) - return microarchitecture; - - if (modelNumber.Length >= 3 && modelNumber.Substring(0, 3).All(char.IsDigit) && - (modelNumber.Length == 3 || !char.IsDigit(modelNumber[3]))) - return "Nehalem"; - if (modelNumber.Length >= 4 && modelNumber.Substring(0, 4).All(char.IsDigit)) - { - char generation = modelNumber[0]; - switch (generation) - { - case '2': - return "Sandy Bridge"; - case '3': - return "Ivy Bridge"; - case '4': - return "Haswell"; - case '5': - return "Broadwell"; - case '6': - return "Skylake"; - case '7': - return "Kaby Lake"; - case '8': - return "Coffee Lake"; - default: - return null; - } - } - return null; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs index be5a84cce2..21bb2821df 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs @@ -1,4 +1,5 @@ using System; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -44,7 +45,7 @@ public static ClrRuntime CreateForLocalFullNetFrameworkBuild(string version) internal static ClrRuntime GetCurrentVersion() { - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { throw new NotSupportedException(".NET Framework supports Windows OS only."); } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs index d86e4b0f40..472b629405 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs @@ -11,18 +11,19 @@ namespace BenchmarkDotNet.Environments { public class CoreRuntime : Runtime { - public static readonly CoreRuntime Core20 = new (RuntimeMoniker.NetCoreApp20, "netcoreapp2.0", ".NET Core 2.0"); - public static readonly CoreRuntime Core21 = new (RuntimeMoniker.NetCoreApp21, "netcoreapp2.1", ".NET Core 2.1"); - public static readonly CoreRuntime Core22 = new (RuntimeMoniker.NetCoreApp22, "netcoreapp2.2", ".NET Core 2.2"); - public static readonly CoreRuntime Core30 = new (RuntimeMoniker.NetCoreApp30, "netcoreapp3.0", ".NET Core 3.0"); - public static readonly CoreRuntime Core31 = new (RuntimeMoniker.NetCoreApp31, "netcoreapp3.1", ".NET Core 3.1"); - public static readonly CoreRuntime Core50 = new (RuntimeMoniker.Net50, "net5.0", ".NET 5.0"); - public static readonly CoreRuntime Core60 = new (RuntimeMoniker.Net60, "net6.0", ".NET 6.0"); - public static readonly CoreRuntime Core70 = new (RuntimeMoniker.Net70, "net7.0", ".NET 7.0"); - public static readonly CoreRuntime Core80 = new (RuntimeMoniker.Net80, "net8.0", ".NET 8.0"); - public static readonly CoreRuntime Core90 = new (RuntimeMoniker.Net90, "net9.0", ".NET 9.0"); - - public static CoreRuntime Latest => Core90; // when dotnet/runtime branches for 10.0, this will need to get updated + public static readonly CoreRuntime Core20 = new(RuntimeMoniker.NetCoreApp20, "netcoreapp2.0", ".NET Core 2.0"); + public static readonly CoreRuntime Core21 = new(RuntimeMoniker.NetCoreApp21, "netcoreapp2.1", ".NET Core 2.1"); + public static readonly CoreRuntime Core22 = new(RuntimeMoniker.NetCoreApp22, "netcoreapp2.2", ".NET Core 2.2"); + public static readonly CoreRuntime Core30 = new(RuntimeMoniker.NetCoreApp30, "netcoreapp3.0", ".NET Core 3.0"); + public static readonly CoreRuntime Core31 = new(RuntimeMoniker.NetCoreApp31, "netcoreapp3.1", ".NET Core 3.1"); + public static readonly CoreRuntime Core50 = new(RuntimeMoniker.Net50, "net5.0", ".NET 5.0"); + public static readonly CoreRuntime Core60 = new(RuntimeMoniker.Net60, "net6.0", ".NET 6.0"); + public static readonly CoreRuntime Core70 = new(RuntimeMoniker.Net70, "net7.0", ".NET 7.0"); + public static readonly CoreRuntime Core80 = new(RuntimeMoniker.Net80, "net8.0", ".NET 8.0"); + public static readonly CoreRuntime Core90 = new(RuntimeMoniker.Net90, "net9.0", ".NET 9.0"); + public static readonly CoreRuntime Core10_0 = new(RuntimeMoniker.Net10_0, "net10.0", ".NET 10.0"); + + public static CoreRuntime Latest => Core10_0; // when dotnet/runtime branches for 11.0, this will need to get updated private CoreRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName) : base(runtimeMoniker, msBuildMoniker, displayName) @@ -74,6 +75,7 @@ internal static CoreRuntime FromVersion(Version version) case Version v when v.Major == 7 && v.Minor == 0: return GetPlatformSpecific(Core70); case Version v when v.Major == 8 && v.Minor == 0: return GetPlatformSpecific(Core80); case Version v when v.Major == 9 && v.Minor == 0: return GetPlatformSpecific(Core90); + case Version v when v.Major == 10 && v.Minor == 0: return GetPlatformSpecific(Core10_0); default: return CreateForNewVersion($"net{version.Major}.{version.Minor}", $".NET {version.Major}.{version.Minor}"); } @@ -97,11 +99,26 @@ internal static bool TryGetVersion(out Version? version) return true; } - var systemPrivateCoreLib = FileVersionInfo.GetVersionInfo(typeof(object).Assembly.Location); - // systemPrivateCoreLib.Product*Part properties return 0 so we have to implement some ugly parsing... - if (TryGetVersionFromProductInfo(systemPrivateCoreLib.ProductVersion, systemPrivateCoreLib.ProductName, out version)) + string coreclrLocation = typeof(object).Assembly.Location; + // Single-file publish has empty assembly location. + if (!string.IsNullOrEmpty(coreclrLocation)) { - return true; + var systemPrivateCoreLib = FileVersionInfo.GetVersionInfo(coreclrLocation); + // systemPrivateCoreLib.Product*Part properties return 0 so we have to implement some ugly parsing... + if (TryGetVersionFromProductInfo(systemPrivateCoreLib.ProductVersion, systemPrivateCoreLib.ProductName, out version)) + { + return true; + } + } + else + { + // .Net Core 3.X supports single-file publish, .Net Core 2.X does not. + // .Net Core 3.X fixed the version in FrameworkDescription, so we don't need to handle the case of 4.6.x in this branch. + var frameworkDescriptionVersion = GetParsableVersionPart(GetVersionFromFrameworkDescription()); + if (Version.TryParse(frameworkDescriptionVersion, out version)) + { + return true; + } } // it's OK to use this method only after checking the previous ones @@ -123,6 +140,14 @@ internal static bool TryGetVersion(out Version? version) return false; } + internal static string GetVersionFromFrameworkDescription() + { + // .NET 10.0.0-preview.5.25277.114 -> 10.0.0-preview.5.25277.114 + // .NET Core 3.1.32 -> 3.1.32 + string frameworkDescription = System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription; + return new string(frameworkDescription.SkipWhile(c => !char.IsDigit(c)).ToArray()); + } + // sample input: // for dotnet run: C:\Program Files\dotnet\shared\Microsoft.NETCore.App\2.1.12\ // for dotnet publish: C:\Users\adsitnik\source\repos\ConsoleApp25\ConsoleApp25\bin\Release\netcoreapp2.0\win-x64\publish\ @@ -192,7 +217,7 @@ internal static bool TryGetVersionFromFrameworkName(string frameworkName, out Ve } // Version.TryParse does not handle thing like 3.0.0-WORD - private static string GetParsableVersionPart(string fullVersionName) => new string(fullVersionName.TakeWhile(c => char.IsDigit(c) || c == '.').ToArray()); + internal static string GetParsableVersionPart(string fullVersionName) => new string(fullVersionName.TakeWhile(c => char.IsDigit(c) || c == '.').ToArray()); private static CoreRuntime GetPlatformSpecific(CoreRuntime fallback) { diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs index 5862d1390c..c1c37db2cd 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs @@ -5,11 +5,12 @@ namespace BenchmarkDotNet.Environments { public class MonoRuntime : Runtime, IEquatable { - public static readonly MonoRuntime Default = new ("Mono"); - public static readonly MonoRuntime Mono60 = new ("Mono with .NET 6.0", RuntimeMoniker.Mono60, "net6.0", isDotNetBuiltIn: true); - public static readonly MonoRuntime Mono70 = new ("Mono with .NET 7.0", RuntimeMoniker.Mono70, "net7.0", isDotNetBuiltIn: true); - public static readonly MonoRuntime Mono80 = new ("Mono with .NET 8.0", RuntimeMoniker.Mono80, "net8.0", isDotNetBuiltIn: true); - public static readonly MonoRuntime Mono90 = new ("Mono with .NET 9.0", RuntimeMoniker.Mono90, "net9.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Default = new("Mono"); + public static readonly MonoRuntime Mono60 = new("Mono with .NET 6.0", RuntimeMoniker.Mono60, "net6.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Mono70 = new("Mono with .NET 7.0", RuntimeMoniker.Mono70, "net7.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Mono80 = new("Mono with .NET 8.0", RuntimeMoniker.Mono80, "net8.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Mono90 = new("Mono with .NET 9.0", RuntimeMoniker.Mono90, "net9.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Mono10_0 = new("Mono with .NET 10.0", RuntimeMoniker.Mono10_0, "net10.0", isDotNetBuiltIn: true); public string CustomPath { get; } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs index f1f49c5dca..bff018558e 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs @@ -22,6 +22,10 @@ public class NativeAotRuntime : Runtime /// NativeAOT compiled as net9.0 /// public static readonly NativeAotRuntime Net90 = new NativeAotRuntime(RuntimeMoniker.NativeAot90, "net9.0", "NativeAOT 9.0"); + /// + /// NativeAOT compiled as net10.0 + /// + public static readonly NativeAotRuntime Net10_0 = new NativeAotRuntime(RuntimeMoniker.NativeAot10_0, "net10.0", "NativeAOT 10.0"); public override bool IsAOT => true; @@ -48,6 +52,7 @@ public static NativeAotRuntime GetCurrentVersion() case Version v when v.Major == 7 && v.Minor == 0: return Net70; case Version v when v.Major == 8 && v.Minor == 0: return Net80; case Version v when v.Major == 9 && v.Minor == 0: return Net90; + case Version v when v.Major == 10 && v.Minor == 0: return Net10_0; default: return new NativeAotRuntime(RuntimeMoniker.NotRecognized, $"net{version.Major}.{version.Minor}", $"NativeAOT {version.Major}.{version.Minor}"); } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs b/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs index b695de321c..4a46e6456d 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs @@ -18,7 +18,7 @@ public abstract class Runtime : IEquatable public RuntimeMoniker RuntimeMoniker { get; } /// - /// MsBuild Target Framework Moniker, example: net462, netcoreapp2.1 + /// MsBuild Target Framework Moniker, example: net462, net8.0 /// public string MsBuildMoniker { get; } diff --git a/src/BenchmarkDotNet/Environments/microarchitectures.txt b/src/BenchmarkDotNet/Environments/microarchitectures.txt deleted file mode 100644 index f06d295390..0000000000 --- a/src/BenchmarkDotNet/Environments/microarchitectures.txt +++ /dev/null @@ -1,183 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Kaby Lake -// See: https://en.wikipedia.org/wiki/Kaby_Lake -//////////////////////////////////////////////////////////////////////////////// - -# Kaby Lake -7Y30 -7Y32 -7Y54 -7Y57 -7Y75 -3865U -3965U -4410Y -4415U -7100 -7100H -7100T -7100U -7101E -7101TE -7130U -7167U -7200U -7260U -7267U -7287U -7300 -7300HQ -7300T -7300U -7320 -7350K -7360U -7400 -7400T -7440HQ -7500 -7500T -7500U -7560U -7567U -7600 -7600K -7600T -7600U -7640X -7660U -7700 -7700HQ -7700K -7700T -7740X -7820HK -7820HQ -7920HQ -8130U -E3-1220 v6 -E3-1225 v6 -E3-1230 v6 -E3-1240 v6 -E3-1245 v6 -E3-1270 v6 -E3-1275 v6 -E3-1280 v6 -E3-1285 v6 -E3-1505L v6 -E3-1505M v6 -E3-1535M v6 -G3930 -G3930T -G3950 -G4560 -G4560T -G4600 -G4600T -G4620 - -# Kaby Lake R -8250U -8350U -8550U -8650U - -# Kaby Lake G -8305G -8705G -8706G -8709G -8809G - -# Amber Lake Y -8100Y -8200Y -8210Y -8500Y - -//////////////////////////////////////////////////////////////////////////////// -// Coffee Lake -// See: https://en.wikipedia.org/wiki/Coffee_Lake -//////////////////////////////////////////////////////////////////////////////// - -# Coffee Lake -610 -2104G -2124 -2124G -2126G -2134 -2136 -2144G -2146G -2174G -2176G -2176M -2186G -2186M -8086K -8100 -8100H -8100T -8109U -8259U -8269U -8300 -8300H -8300T -8350K -8400 -8400B -8400H -8400T -8500 -8500B -8500T -8559U -8600 -8600K -8600T -8700 -8700B -8700K -8700T -8750H -8850H -8950HK -9350KF -9400 -9400F -9600K -9600KF -9700K -9700KF -9900K -9900KF -G4900 -G4900T -G4920 -G5400 -G5400T -G5500 -G5500T -G5600 - -//////////////////////////////////////////////////////////////////////////////// -// Cannon Lake -// See: https://en.wikipedia.org/wiki/Cannon_Lake_(microarchitecture) -//////////////////////////////////////////////////////////////////////////////// - -# Cannon Lake -8121U - -//////////////////////////////////////////////////////////////////////////////// -// Whiskey Lake -// See: https://en.wikipedia.org/wiki/Whiskey_Lake_(microarchitecture) -//////////////////////////////////////////////////////////////////////////////// - -# Whiskey Lake -8565U -8265U -8145U -5405U -4205U \ No newline at end of file diff --git a/src/BenchmarkDotNet/Exporters/ExporterBase.cs b/src/BenchmarkDotNet/Exporters/ExporterBase.cs index 17a8d432a6..a7604b2864 100644 --- a/src/BenchmarkDotNet/Exporters/ExporterBase.cs +++ b/src/BenchmarkDotNet/Exporters/ExporterBase.cs @@ -48,7 +48,7 @@ public IEnumerable ExportToFiles(Summary summary, ILogger consoleLogger) return new[] { filePath }; } - internal string GetArtifactFullName(Summary summary) + public string GetArtifactFullName(Summary summary) { string fileName = GetFileName(summary); return $"{Path.Combine(summary.ResultsDirectoryPath, fileName)}-{FileCaption}{FileNameSuffix}.{FileExtension}"; diff --git a/src/BenchmarkDotNet/Exporters/Json/JsonExporterBase.cs b/src/BenchmarkDotNet/Exporters/Json/JsonExporterBase.cs index 8ce4472a5a..3727ecac80 100644 --- a/src/BenchmarkDotNet/Exporters/Json/JsonExporterBase.cs +++ b/src/BenchmarkDotNet/Exporters/Json/JsonExporterBase.cs @@ -3,8 +3,9 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; +using Perfolizer.Helpers; using Perfolizer.Horology; -using JsonSerializer = SimpleJson.SimpleJson; +using SimpleJson; namespace BenchmarkDotNet.Exporters.Json { @@ -23,8 +24,8 @@ protected JsonExporterBase(bool indentJson = false, bool excludeMeasurements = f public override void ExportToLog(Summary summary, ILogger logger) { - JsonSerializer.CurrentJsonSerializerStrategy.Indent = IndentJson; - logger.WriteLine(JsonSerializer.SerializeObject(GetDataToSerialize(summary))); + SimpleJsonSerializer.CurrentJsonSerializerStrategy.Indent = IndentJson; + logger.WriteLine(SimpleJsonSerializer.SerializeObject(GetDataToSerialize(summary))); } protected virtual IReadOnlyDictionary GetDataToSerialize(Summary summary) @@ -47,11 +48,11 @@ protected virtual IReadOnlyDictionary GetDataToSerialize(HostEnv { { nameof(HostEnvironmentInfo.BenchmarkDotNetCaption), HostEnvironmentInfo.BenchmarkDotNetCaption }, { nameof(environmentInfo.BenchmarkDotNetVersion), environmentInfo.BenchmarkDotNetVersion }, - { "OsVersion", environmentInfo.OsVersion.Value }, - { "ProcessorName", ProcessorBrandStringHelper.Prettify(environmentInfo.CpuInfo.Value) }, - { "PhysicalProcessorCount", environmentInfo.CpuInfo.Value?.PhysicalProcessorCount }, - { "PhysicalCoreCount", environmentInfo.CpuInfo.Value?.PhysicalCoreCount }, - { "LogicalCoreCount", environmentInfo.CpuInfo.Value?.LogicalCoreCount }, + { "OsVersion", environmentInfo.Os.Value.ToBrandString() }, + { "ProcessorName", environmentInfo.Cpu.Value.ToShortBrandName() }, + { "PhysicalProcessorCount", environmentInfo.Cpu.Value?.PhysicalProcessorCount }, + { "PhysicalCoreCount", environmentInfo.Cpu.Value?.PhysicalCoreCount }, + { "LogicalCoreCount", environmentInfo.Cpu.Value?.LogicalCoreCount }, { nameof(environmentInfo.RuntimeVersion), environmentInfo.RuntimeVersion }, { nameof(environmentInfo.Architecture), environmentInfo.Architecture }, { nameof(environmentInfo.HasAttachedDebugger), environmentInfo.HasAttachedDebugger }, @@ -83,40 +84,40 @@ protected virtual IReadOnlyDictionary GetDataToSerialize(Benchma { "Statistics", report.ResultStatistics } }; - // We show MemoryDiagnoser's results only if it is being used - if (report.BenchmarkCase.Config.HasMemoryDiagnoser()) + // We show MemoryDiagnoser's results only if it is being used + if (report.BenchmarkCase.Config.HasMemoryDiagnoser()) + { + benchmark.Add("Memory", new { - benchmark.Add("Memory", new + report.GcStats.Gen0Collections, + report.GcStats.Gen1Collections, + report.GcStats.Gen2Collections, + report.GcStats.TotalOperations, + BytesAllocatedPerOperation = report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) + }); + } + + if (ExcludeMeasurements == false) + { + // We construct Measurements manually, so that we can have the IterationMode enum as text, rather than an integer + benchmark.Add("Measurements", + report.AllMeasurements.Select(m => new { - report.GcStats.Gen0Collections, - report.GcStats.Gen1Collections, - report.GcStats.Gen2Collections, - report.GcStats.TotalOperations, - BytesAllocatedPerOperation = report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) - }); - } + IterationMode = m.IterationMode.ToString(), + IterationStage = m.IterationStage.ToString(), + m.LaunchIndex, + m.IterationIndex, + m.Operations, + m.Nanoseconds + })); - if (ExcludeMeasurements == false) + if (report.Metrics.Any()) { - // We construct Measurements manually, so that we can have the IterationMode enum as text, rather than an integer - benchmark.Add("Measurements", - report.AllMeasurements.Select(m => new - { - IterationMode = m.IterationMode.ToString(), - IterationStage = m.IterationStage.ToString(), - m.LaunchIndex, - m.IterationIndex, - m.Operations, - m.Nanoseconds - })); - - if (report.Metrics.Any()) - { - benchmark.Add("Metrics", report.Metrics.Values); - } + benchmark.Add("Metrics", report.Metrics.Values); } + } - return benchmark; + return benchmark; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Exporters/Json/SimpleJson.cs b/src/BenchmarkDotNet/Exporters/Json/SimpleJsonSerializer.cs similarity index 99% rename from src/BenchmarkDotNet/Exporters/Json/SimpleJson.cs rename to src/BenchmarkDotNet/Exporters/Json/SimpleJsonSerializer.cs index 5670d64588..e461c8a5c6 100644 --- a/src/BenchmarkDotNet/Exporters/Json/SimpleJson.cs +++ b/src/BenchmarkDotNet/Exporters/Json/SimpleJsonSerializer.cs @@ -105,7 +105,7 @@ public JsonArray(int capacity) : base(capacity) { } /// The json representation of the array. public override string ToString() { - return SimpleJson.SerializeObject(this) ?? string.Empty; + return SimpleJsonSerializer.SerializeObject(this) ?? string.Empty; } } @@ -344,7 +344,7 @@ IEnumerator IEnumerable.GetEnumerator() /// public override string ToString() { - return SimpleJson.SerializeObject(this); + return SimpleJsonSerializer.SerializeObject(this); } #if SIMPLE_JSON_DYNAMIC @@ -501,7 +501,7 @@ namespace SimpleJson #else public #endif - static class SimpleJson + static class SimpleJsonSerializer { private const int TOKEN_NONE = 0; private const int TOKEN_CURLY_OPEN = 1; @@ -522,7 +522,7 @@ static class SimpleJson private static readonly char[] EscapeCharacters = new char[] { '"', '\\', '\b', '\f', '\n', '\r', '\t' }; private static readonly string EscapeCharactersString = new string(EscapeCharacters); - static SimpleJson() + static SimpleJsonSerializer() { EscapeTable = new char[93]; EscapeTable['"'] = '"'; diff --git a/src/BenchmarkDotNet/Exporters/PerfonarJsonExporter.cs b/src/BenchmarkDotNet/Exporters/PerfonarJsonExporter.cs new file mode 100644 index 0000000000..ce8287fb2d --- /dev/null +++ b/src/BenchmarkDotNet/Exporters/PerfonarJsonExporter.cs @@ -0,0 +1,18 @@ +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using Perfolizer.Json; + +namespace BenchmarkDotNet.Exporters; + +/// +/// IMPORTANT: Not fully implemented yet +/// +internal class PerfonarJsonExporter(LightJsonSettings? jsonSettings = null) : ExporterBase +{ + protected override string FileExtension => "perfonar.json"; + + public override void ExportToLog(Summary summary, ILogger logger) + { + logger.WriteLine(LightJsonSerializer.Serialize(summary.ToPerfonar(), jsonSettings)); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Exporters/PerfonarMdExporter.cs b/src/BenchmarkDotNet/Exporters/PerfonarMdExporter.cs new file mode 100644 index 0000000000..d2d4e77a79 --- /dev/null +++ b/src/BenchmarkDotNet/Exporters/PerfonarMdExporter.cs @@ -0,0 +1,21 @@ +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; + +using Perfolizer.Perfonar.Tables; + +namespace BenchmarkDotNet.Exporters; + +/// +/// IMPORTANT: Not fully implemented yet +/// +internal class PerfonarMdExporter : ExporterBase +{ + protected override string FileExtension => "perfonar.md"; + + public override void ExportToLog(Summary summary, ILogger logger) + { + var table = new PerfonarTable(summary.ToPerfonar(), summary.GetDefaultTableConfig()); + string markdown = table.ToMarkdown(new PerfonarTableStyle()); + logger.WriteLine(markdown); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Exporters/RPlotExporter.cs b/src/BenchmarkDotNet/Exporters/RPlotExporter.cs index 5c23f3b598..c90338f490 100644 --- a/src/BenchmarkDotNet/Exporters/RPlotExporter.cs +++ b/src/BenchmarkDotNet/Exporters/RPlotExporter.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Exporters.Csv; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; @@ -82,7 +83,7 @@ public void ExportToLog(Summary summary, ILogger logger) private static bool TryFindRScript(ILogger consoleLogger, out string? rscriptPath) { - string rscriptExecutable = RuntimeInformation.IsWindows() ? "Rscript.exe" : "Rscript"; + string rscriptExecutable = OsDetector.IsWindows() ? "Rscript.exe" : "Rscript"; rscriptPath = null; string rHome = Environment.GetEnvironmentVariable("R_HOME"); @@ -101,7 +102,7 @@ private static bool TryFindRScript(ILogger consoleLogger, out string? rscriptPat if (rscriptPath != null) return true; - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { string programFiles = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles); string programFilesR = Path.Combine(programFiles, "R"); diff --git a/src/BenchmarkDotNet/Exporters/Xml/SummaryDto.cs b/src/BenchmarkDotNet/Exporters/Xml/SummaryDto.cs index 79f8e51f1e..3087250b52 100644 --- a/src/BenchmarkDotNet/Exporters/Xml/SummaryDto.cs +++ b/src/BenchmarkDotNet/Exporters/Xml/SummaryDto.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Reports; using JetBrains.Annotations; +using Perfolizer.Helpers; using Perfolizer.Horology; // ReSharper disable UnusedMember.Global @@ -34,11 +35,11 @@ internal class HostEnvironmentInfoDto { public string BenchmarkDotNetCaption => HostEnvironmentInfo.BenchmarkDotNetCaption; public string BenchmarkDotNetVersion => hei.BenchmarkDotNetVersion; - public string OsVersion => hei.OsVersion.Value; - public string ProcessorName => ProcessorBrandStringHelper.Prettify(hei.CpuInfo.Value); - public string PhysicalProcessorCount => hei.CpuInfo.Value?.PhysicalProcessorCount?.ToString(); - public string PhysicalCoreCount => hei.CpuInfo.Value?.PhysicalCoreCount?.ToString(); - public string LogicalCoreCount => hei.CpuInfo.Value?.LogicalCoreCount?.ToString(); + public string OsVersion => hei.Os.Value.ToBrandString(); + public string ProcessorName => hei.Cpu.Value.ToShortBrandName(); + public string PhysicalProcessorCount => hei.Cpu.Value?.PhysicalProcessorCount?.ToString(); + public string PhysicalCoreCount => hei.Cpu.Value?.PhysicalCoreCount?.ToString(); + public string LogicalCoreCount => hei.Cpu.Value?.LogicalCoreCount?.ToString(); public string RuntimeVersion => hei.RuntimeVersion; public string Architecture => hei.Architecture; public bool HasAttachedDebugger => hei.HasAttachedDebugger; @@ -72,6 +73,7 @@ internal class BenchmarkReportDto public string Parameters => report.BenchmarkCase.Parameters.PrintInfo; public Statistics Statistics => report.ResultStatistics; public IEnumerable Metrics => report.Metrics.Values; + public GcStats Memory => new GcStats() { Gen0Collections = report.GcStats.Gen0Collections, @@ -80,6 +82,7 @@ internal class BenchmarkReportDto TotalOperations = report.GcStats.TotalOperations, BytesAllocatedPerOperation = report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) }; + [PublicAPI] public IEnumerable Measurements { get; } private readonly BenchmarkReport report; @@ -105,4 +108,4 @@ internal struct GcStats public long TotalOperations { get; set; } public long? BytesAllocatedPerOperation { get; set; } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Extensions/MathExtensions.cs b/src/BenchmarkDotNet/Extensions/MathExtensions.cs new file mode 100644 index 0000000000..86a17f93f5 --- /dev/null +++ b/src/BenchmarkDotNet/Extensions/MathExtensions.cs @@ -0,0 +1,9 @@ +using System; + +namespace BenchmarkDotNet.Extensions; + +internal static class MathExtensions +{ + public static int RoundToInt(this double x) => (int)Math.Round(x); + public static long RoundToLong(this double x) => (long)Math.Round(x); +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs index 912babeb39..4d0854ba03 100644 --- a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.IO; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; @@ -86,7 +87,7 @@ public static bool TrySetAffinity( if (logger == null) throw new ArgumentNullException(nameof(logger)); - if (!RuntimeInformation.IsWindows() && !RuntimeInformation.IsLinux()) + if (!OsDetector.IsWindows() && !OsDetector.IsLinux()) return false; try @@ -108,7 +109,7 @@ public static bool TrySetAffinity( if (process == null) throw new ArgumentNullException(nameof(process)); - if (!RuntimeInformation.IsWindows() && !RuntimeInformation.IsLinux()) + if (!OsDetector.IsWindows() && !OsDetector.IsLinux()) return null; try @@ -124,7 +125,7 @@ public static bool TrySetAffinity( internal static void SetEnvironmentVariables(this ProcessStartInfo start, BenchmarkCase benchmarkCase, IResolver resolver) { if (benchmarkCase.Job.Environment.Runtime is ClrRuntime clrRuntime && !string.IsNullOrEmpty(clrRuntime.Version)) - start.EnvironmentVariables["COMPLUS_Version"] = clrRuntime.Version; + SetClrEnvironmentVariables(start, "Version", clrRuntime.Version); if (benchmarkCase.Job.Environment.Runtime is MonoRuntime monoRuntime && !string.IsNullOrEmpty(monoRuntime.MonoBclPath)) start.EnvironmentVariables["MONO_PATH"] = monoRuntime.MonoBclPath; @@ -132,10 +133,10 @@ internal static void SetEnvironmentVariables(this ProcessStartInfo start, Benchm if (benchmarkCase.Config.HasPerfCollectProfiler()) { // enable tracing configuration inside of CoreCLR (https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/linux-performance-tracing.md#collecting-a-trace) - start.EnvironmentVariables["COMPlus_PerfMapEnabled"] = "1"; - start.EnvironmentVariables["COMPlus_EnableEventLog"] = "1"; + SetClrEnvironmentVariables(start, "PerfMapEnabled", "1"); + SetClrEnvironmentVariables(start, "EnableEventLog", "1"); // enable BDN Event Source (https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/linux-performance-tracing.md#filtering) - start.EnvironmentVariables["COMPlus_EventSourceFilter"] = EngineEventSource.SourceName; + SetClrEnvironmentVariables(start, "EventSourceFilter", EngineEventSource.SourceName); // workaround for https://github.com/dotnet/runtime/issues/71786, will be solved by next perf version start.EnvironmentVariables["DOTNET_EnableWriteXorExecute"] = "0"; } @@ -163,7 +164,7 @@ internal static void SetEnvironmentVariables(this ProcessStartInfo start, Benchm public static void KillTree(this Process process, TimeSpan timeout) { - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { RunProcessAndIgnoreOutput("taskkill", $"/T /F /PID {process.Id}", timeout); } @@ -257,21 +258,27 @@ private static void SetCoreRunEnvironmentVariables(this ProcessStartInfo start, { var gcMode = benchmarkCase.Job.Environment.Gc; - start.EnvironmentVariables["COMPlus_gcServer"] = gcMode.ResolveValue(GcMode.ServerCharacteristic, resolver) ? "1" : "0"; - start.EnvironmentVariables["COMPlus_gcConcurrent"] = gcMode.ResolveValue(GcMode.ConcurrentCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "gcServer", gcMode.ResolveValue(GcMode.ServerCharacteristic, resolver) ? "1" : "0"); + SetClrEnvironmentVariables(start, "gcConcurrent", gcMode.ResolveValue(GcMode.ConcurrentCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.CpuGroupsCharacteristic)) - start.EnvironmentVariables["COMPlus_GCCpuGroup"] = gcMode.ResolveValue(GcMode.CpuGroupsCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "GCCpuGroup", gcMode.ResolveValue(GcMode.CpuGroupsCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.AllowVeryLargeObjectsCharacteristic)) - start.EnvironmentVariables["COMPlus_gcAllowVeryLargeObjects"] = gcMode.ResolveValue(GcMode.AllowVeryLargeObjectsCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "gcAllowVeryLargeObjects", gcMode.ResolveValue(GcMode.AllowVeryLargeObjectsCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.RetainVmCharacteristic)) - start.EnvironmentVariables["COMPlus_GCRetainVM"] = gcMode.ResolveValue(GcMode.RetainVmCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "GCRetainVM", gcMode.ResolveValue(GcMode.RetainVmCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.NoAffinitizeCharacteristic)) - start.EnvironmentVariables["COMPlus_GCNoAffinitize"] = gcMode.ResolveValue(GcMode.NoAffinitizeCharacteristic, resolver) ? "1" : "0"; + SetClrEnvironmentVariables(start, "GCNoAffinitize", gcMode.ResolveValue(GcMode.NoAffinitizeCharacteristic, resolver) ? "1" : "0"); if (gcMode.HasValue(GcMode.HeapAffinitizeMaskCharacteristic)) - start.EnvironmentVariables["COMPlus_GCHeapAffinitizeMask"] = gcMode.HeapAffinitizeMask.ToString("X"); + SetClrEnvironmentVariables(start, "GCHeapAffinitizeMask", gcMode.HeapAffinitizeMask.ToString("X")); if (gcMode.HasValue(GcMode.HeapCountCharacteristic)) - start.EnvironmentVariables["COMPlus_GCHeapCount"] = gcMode.HeapCount.ToString("X"); + SetClrEnvironmentVariables(start, "GCHeapCount", gcMode.HeapCount.ToString("X")); + } + + private static void SetClrEnvironmentVariables(ProcessStartInfo start, string suffix, string value) + { + start.EnvironmentVariables[$"DOTNET_{suffix}"] = value; + start.EnvironmentVariables[$"COMPlus_{suffix}"] = value; } } } diff --git a/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs b/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs index cf1f71a166..0a1a7c22fc 100644 --- a/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/ReflectionExtensions.cs @@ -155,10 +155,7 @@ internal static bool ContainsRunnableBenchmarks(this Type type) { var typeInfo = type.GetTypeInfo(); - if (typeInfo.IsAbstract - || typeInfo.IsSealed - || typeInfo.IsNotPublic - || typeInfo.IsGenericType && !IsRunnableGenericType(typeInfo)) + if (typeInfo.IsAbstract || typeInfo.IsGenericType && !IsRunnableGenericType(typeInfo)) return false; return typeInfo.GetBenchmarks().Any(); diff --git a/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs b/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs index 5ba51eebdd..e3fbdb04e0 100644 --- a/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs @@ -47,6 +47,8 @@ internal static Runtime GetRuntime(this RuntimeMoniker runtimeMoniker) return CoreRuntime.Core80; case RuntimeMoniker.Net90: return CoreRuntime.Core90; + case RuntimeMoniker.Net10_0: + return CoreRuntime.Core10_0; case RuntimeMoniker.Mono: return MonoRuntime.Default; case RuntimeMoniker.NativeAot60: @@ -57,6 +59,8 @@ internal static Runtime GetRuntime(this RuntimeMoniker runtimeMoniker) return NativeAotRuntime.Net80; case RuntimeMoniker.NativeAot90: return NativeAotRuntime.Net90; + case RuntimeMoniker.NativeAot10_0: + return NativeAotRuntime.Net10_0; case RuntimeMoniker.Mono60: return MonoRuntime.Mono60; case RuntimeMoniker.Mono70: @@ -65,9 +69,57 @@ internal static Runtime GetRuntime(this RuntimeMoniker runtimeMoniker) return MonoRuntime.Mono80; case RuntimeMoniker.Mono90: return MonoRuntime.Mono90; + case RuntimeMoniker.Mono10_0: + return MonoRuntime.Mono10_0; default: throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "Runtime Moniker not supported"); } } + + internal static Version GetRuntimeVersion(this RuntimeMoniker runtimeMoniker) => runtimeMoniker switch + { + RuntimeMoniker.Net461 => new Version(4, 6, 1), + RuntimeMoniker.Net462 => new Version(4, 6, 2), + RuntimeMoniker.Net47 => new Version(4, 7), + RuntimeMoniker.Net471 => new Version(4, 7, 1), + RuntimeMoniker.Net472 => new Version(4, 7, 2), + RuntimeMoniker.Net48 => new Version(4, 8), + RuntimeMoniker.Net481 => new Version(4, 8, 1), + RuntimeMoniker.NetCoreApp20 => new Version(2, 0), + RuntimeMoniker.NetCoreApp21 => new Version(2, 1), + RuntimeMoniker.NetCoreApp22 => new Version(2, 2), + RuntimeMoniker.NetCoreApp30 => new Version(3, 0), + RuntimeMoniker.NetCoreApp31 => new Version(3, 1), + RuntimeMoniker.Net50 => new Version(5, 0), + RuntimeMoniker.Net60 => new Version(6, 0), + RuntimeMoniker.Net70 => new Version(7, 0), + RuntimeMoniker.Net80 => new Version(8, 0), + RuntimeMoniker.Net90 => new Version(9, 0), + RuntimeMoniker.Net10_0 => new Version(10, 0), + RuntimeMoniker.NativeAot60 => new Version(6, 0), + RuntimeMoniker.NativeAot70 => new Version(7, 0), + RuntimeMoniker.NativeAot80 => new Version(8, 0), + RuntimeMoniker.NativeAot90 => new Version(9, 0), + RuntimeMoniker.NativeAot10_0 => new Version(10, 0), + RuntimeMoniker.Mono60 => new Version(6, 0), + RuntimeMoniker.Mono70 => new Version(7, 0), + RuntimeMoniker.Mono80 => new Version(8, 0), + RuntimeMoniker.Mono90 => new Version(9, 0), + RuntimeMoniker.Mono10_0 => new Version(10, 0), + RuntimeMoniker.Wasm => Portability.RuntimeInformation.IsNetCore && CoreRuntime.TryGetVersion(out var version) ? version : new Version(5, 0), + RuntimeMoniker.WasmNet50 => new Version(5, 0), + RuntimeMoniker.WasmNet60 => new Version(6, 0), + RuntimeMoniker.WasmNet70 => new Version(7, 0), + RuntimeMoniker.WasmNet80 => new Version(8, 0), + RuntimeMoniker.WasmNet90 => new Version(9, 0), + RuntimeMoniker.WasmNet10_0 => new Version(10, 0), + RuntimeMoniker.MonoAOTLLVM => Portability.RuntimeInformation.IsNetCore && CoreRuntime.TryGetVersion(out var version) ? version : new Version(6, 0), + RuntimeMoniker.MonoAOTLLVMNet60 => new Version(6, 0), + RuntimeMoniker.MonoAOTLLVMNet70 => new Version(7, 0), + RuntimeMoniker.MonoAOTLLVMNet80 => new Version(8, 0), + RuntimeMoniker.MonoAOTLLVMNet90 => new Version(9, 0), + RuntimeMoniker.MonoAOTLLVMNet10_0 => new Version(10, 0), + _ => throw new NotImplementedException($"{nameof(GetRuntimeVersion)} not implemented for {runtimeMoniker}") + }; } } diff --git a/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs b/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs index 62b4d31d94..d98ca46e58 100644 --- a/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs +++ b/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Linq; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; @@ -17,7 +18,7 @@ internal static class ArtifactFileNameHelper internal static string GetTraceFilePath(DiagnoserActionParameters details, DateTime creationTime, string fileExtension) { - return GetFilePath(details, null, creationTime, fileExtension, "userheap.etl".Length); + return GetFilePath(details, null, creationTime, fileExtension, "userheap.etl".Length - fileExtension.Length); } internal static string GetFilePath(DiagnoserActionParameters details, string? subfolder, DateTime? creationTime, string fileExtension, int reserve) @@ -26,7 +27,7 @@ internal static string GetFilePath(DiagnoserActionParameters details, string? su // long paths can be enabled on Windows but it does not mean that everything is going to work fine.. // so we always use 260 as limit on Windows - int limit = RuntimeInformation.IsWindows() + int limit = OsDetector.IsWindows() ? WindowsOldPathLimit - reserve : CommonSenseLimit; @@ -68,7 +69,7 @@ private static string GetLimitedFilePath(DiagnoserActionParameters details, stri private static string GetFilePath(string fileName, DiagnoserActionParameters details, string? subfolder, DateTime? creationTime, string fileExtension) { - // if we run for more than one toolchain, the output file name should contain the name too so we can differ net462 vs netcoreapp2.1 etc + // if we run for more than one toolchain, the output file name should contain the name too so we can differ net462 vs net8.0 etc if (details.Config.GetJobs().Select(job => ToolchainExtensions.GetToolchain(job)).Distinct().Count() > 1) fileName += $"-{details.BenchmarkCase.Job.Environment.Runtime?.Name ?? details.BenchmarkCase.GetToolchain()?.Name ?? details.BenchmarkCase.Job.Id}"; diff --git a/src/BenchmarkDotNet/Helpers/AwaitHelper.cs b/src/BenchmarkDotNet/Helpers/AwaitHelper.cs index 8d16fb716a..98837a7ea6 100644 --- a/src/BenchmarkDotNet/Helpers/AwaitHelper.cs +++ b/src/BenchmarkDotNet/Helpers/AwaitHelper.cs @@ -22,7 +22,7 @@ private class ValueTaskWaiter private ValueTaskWaiter() { - resetEvent = new (); + resetEvent = new(); awaiterCallback = resetEvent.Set; } diff --git a/src/BenchmarkDotNet/Helpers/DisposeAtProcessTermination.cs b/src/BenchmarkDotNet/Helpers/DisposeAtProcessTermination.cs new file mode 100644 index 0000000000..bc627a4a1b --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/DisposeAtProcessTermination.cs @@ -0,0 +1,70 @@ +using BenchmarkDotNet.Detectors; +using System; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Helpers +{ + /// + /// Ensures that explicit Dispose is called at termination of the Process. + /// + /// + /// + /// This class exists to help in reverting system state where C#'s using statement does not + /// suffice. I.e. when Benchmark's process is aborted via Ctrl-C, Ctrl-Break or via click on the + /// X in the upper right of Window. + /// + /// + /// Usage: Derive your clas that changes system state of this class. Revert system state in + /// override of implementation. + /// Use your class in C#'s using statement, to ensure system state is reverted in normal situations. + /// This class ensures your override is also called at process 'abort'. + /// + /// + /// Note: This class is explicitly not responsible for cleanup of Native resources. Of course, + /// derived classes can cleanup their Native resources (usually managed via + /// derived classes), by delegating explicit Disposal to their + /// fields. + /// + /// + public abstract class DisposeAtProcessTermination : IDisposable + { + private static readonly bool ConsoleSupportsCancelKeyPress + = !(OsDetector.IsAndroid() || OsDetector.IsIOS() || OsDetector.IsTvOS() || Portability.RuntimeInformation.IsWasm); + + protected DisposeAtProcessTermination() + { + AppDomain.CurrentDomain.ProcessExit += OnProcessExit; + if (ConsoleSupportsCancelKeyPress) + { + // Cancel key presses are not supported by .NET or do not exist for these + Console.CancelKeyPress += OnCancelKeyPress; + } + + // It does not make sense to include a Finalizer. We do not manage any native resource and: + // as we are subscribed to static events, it would never be called. + } + + /// + /// Called when the user presses Ctrl-C or Ctrl-Break. + /// + private void OnCancelKeyPress(object? sender, ConsoleCancelEventArgs e) + { + if (!e.Cancel) { Dispose(); } + } + + /// + /// Called when the user clicks on the X in the upper right corner to close the Benchmark's Window. + /// + private void OnProcessExit(object? sender, EventArgs e) => Dispose(); + + public virtual void Dispose() + { + AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; + if (ConsoleSupportsCancelKeyPress) + { + // Cancel key presses are not supported by .NET or do not exist for these + Console.CancelKeyPress -= OnCancelKeyPress; + } + } + } +} diff --git a/src/BenchmarkDotNet/Helpers/ExternalToolsHelper.cs b/src/BenchmarkDotNet/Helpers/ExternalToolsHelper.cs index 3802608047..bd5274c6b3 100644 --- a/src/BenchmarkDotNet/Helpers/ExternalToolsHelper.cs +++ b/src/BenchmarkDotNet/Helpers/ExternalToolsHelper.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Helpers @@ -11,7 +12,7 @@ public static class ExternalToolsHelper /// MacOSX only. /// public static readonly Lazy> MacSystemProfilerData = - LazyParse(RuntimeInformation.IsMacOS, "system_profiler", "SPSoftwareDataType", s => SectionsHelper.ParseSection(s, ':')); + LazyParse(OsDetector.IsMacOS, "system_profiler", "SPSoftwareDataType", s => SectionsHelper.ParseSection(s, ':')); private static Lazy LazyParse(Func isAvailable, string fileName, string arguments, Func parseFunc) { diff --git a/src/BenchmarkDotNet/Helpers/PowerShellLocator.cs b/src/BenchmarkDotNet/Helpers/PowerShellLocator.cs new file mode 100644 index 0000000000..28b8e910f7 --- /dev/null +++ b/src/BenchmarkDotNet/Helpers/PowerShellLocator.cs @@ -0,0 +1,81 @@ +using System; +using System.IO; +using System.Linq; +using System.Runtime.Versioning; +using System.Text.RegularExpressions; +using BenchmarkDotNet.Detectors; + +namespace BenchmarkDotNet.Helpers; + +/// +/// Locates PowerShell on a system, currently only supports on Windows. +/// +internal class PowerShellLocator +{ + private static readonly string WindowsPowershellPath = + $"{Environment.SystemDirectory}{Path.DirectorySeparatorChar}WindowsPowerShell{Path.DirectorySeparatorChar}" + + $"v1.0{Path.DirectorySeparatorChar}powershell.exe"; + +#if NET6_0_OR_GREATER + [SupportedOSPlatform("windows")] +#endif + internal static string? LocateOnWindows() + { + if (OsDetector.IsWindows() == false) + return null; + + string powershellPath; + + try + { + string programFiles = Environment.Is64BitOperatingSystem + ? Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles) + : Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86); + + string powershell7PlusPath = $"{programFiles}{Path.DirectorySeparatorChar}Powershell{Path.DirectorySeparatorChar}"; + + bool checkForPowershell7Plus = true; + + if (Directory.Exists(powershell7PlusPath)) + { + string[] subDirectories = Directory.EnumerateDirectories(powershell7PlusPath, "*", SearchOption.AllDirectories) + .ToArray(); + + //Use the highest number string directory for PowerShell so that we get the newest major PowerShell version + // Example version directories are 6, 7, and in the future 8. + string? subDirectory = subDirectories.Where(x => Regex.IsMatch(x, "[0-9]")) + .Select(x => x) + .OrderByDescending(x => x) + .FirstOrDefault(); + + if (subDirectory is not null) + { + powershell7PlusPath = $"{subDirectory}{Path.DirectorySeparatorChar}pwsh.exe"; + } + else + { + checkForPowershell7Plus = false; + } + } + + // Optimistically, use Cross-platform new PowerShell when available but fallback to Windows PowerShell if not available. + if (checkForPowershell7Plus) + { + powershellPath = File.Exists(powershell7PlusPath) ? powershell7PlusPath : WindowsPowershellPath; + } + else + { + powershellPath = WindowsPowershellPath; + } + + if (File.Exists(powershellPath) == false) + powershellPath = "PowerShell"; + } + catch + { + powershellPath = "PowerShell"; + } + + return powershellPath; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/ProcessHelper.cs b/src/BenchmarkDotNet/Helpers/ProcessHelper.cs index 1b75c6392f..caa3410b21 100644 --- a/src/BenchmarkDotNet/Helpers/ProcessHelper.cs +++ b/src/BenchmarkDotNet/Helpers/ProcessHelper.cs @@ -12,7 +12,8 @@ internal static class ProcessHelper /// Run external process and return the console output. /// In the case of any exception, null will be returned. /// - internal static string? RunAndReadOutput(string fileName, string arguments = "", ILogger? logger = null) + internal static string? RunAndReadOutput(string fileName, string arguments = "", ILogger? logger = null, + Dictionary? environmentVariables = null) { var processStartInfo = new ProcessStartInfo { @@ -24,6 +25,9 @@ internal static class ProcessHelper RedirectStandardOutput = true, RedirectStandardError = true }; + if (environmentVariables != null) + foreach (var variable in environmentVariables) + processStartInfo.Environment[variable.Key] = variable.Value; using (var process = new Process { StartInfo = processStartInfo }) using (new ConsoleExitHandler(process, logger ?? NullLogger.Instance)) { diff --git a/src/BenchmarkDotNet/Helpers/SectionsHelper.cs b/src/BenchmarkDotNet/Helpers/SectionsHelper.cs index 15c54fd8f9..e16442d0fb 100644 --- a/src/BenchmarkDotNet/Helpers/SectionsHelper.cs +++ b/src/BenchmarkDotNet/Helpers/SectionsHelper.cs @@ -32,5 +32,14 @@ public static List> ParseSections(string? content, ch .Where(s => s.Count > 0) .ToList(); } + + public static List> ParseSectionsForPowershellWmi(string? content, char separator) + { + return + Regex.Split(content ?? "", "(\r*\n)") + .Select(s => ParseSection(s, separator)) + .Where(s => s.Count > 0) + .ToList(); + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/TaskbarProgress.cs b/src/BenchmarkDotNet/Helpers/TaskbarProgress.cs index f66e21cdf6..e3910a0924 100644 --- a/src/BenchmarkDotNet/Helpers/TaskbarProgress.cs +++ b/src/BenchmarkDotNet/Helpers/TaskbarProgress.cs @@ -1,5 +1,6 @@ using System; using System.Runtime.InteropServices; +using BenchmarkDotNet.Detectors; namespace BenchmarkDotNet.Helpers { @@ -13,9 +14,9 @@ internal enum TaskbarProgressState Warning = Paused } - internal class TaskbarProgress : IDisposable + internal class TaskbarProgress : DisposeAtProcessTermination { - private static readonly bool OsVersionIsSupported = Portability.RuntimeInformation.IsWindows() + private static readonly bool OsVersionIsSupported = OsDetector.IsWindows() // Must be windows 7 or greater && Environment.OSVersion.Version >= new Version(6, 1); @@ -30,10 +31,6 @@ internal TaskbarProgress(TaskbarProgressState initialTaskbarState) { com = Com.MaybeCreateInstanceAndSetInitialState(initialTaskbarState); terminal = Terminal.MaybeCreateInstanceAndSetInitialState(initialTaskbarState); - if (IsEnabled) - { - Console.CancelKeyPress += OnConsoleCancelEvent; - } } } @@ -50,23 +47,20 @@ internal void SetProgress(float progressValue) { throw new ArgumentOutOfRangeException(nameof(progressValue), "progressValue must be between 0 and 1 inclusive."); } - uint value = (uint) (progressValue * 100); + uint value = (uint)(progressValue * 100); com?.SetValue(value); terminal?.SetValue(value); } - private void OnConsoleCancelEvent(object sender, ConsoleCancelEventArgs e) - => Dispose(); - - public void Dispose() + public override void Dispose() { if (IsEnabled) { SetState(TaskbarProgressState.NoProgress); com = null; terminal = null; - Console.CancelKeyPress -= OnConsoleCancelEvent; } + base.Dispose(); } private sealed class Com diff --git a/src/BenchmarkDotNet/Helpers/UnitHelper.cs b/src/BenchmarkDotNet/Helpers/UnitHelper.cs index ae76faf101..9575401776 100644 --- a/src/BenchmarkDotNet/Helpers/UnitHelper.cs +++ b/src/BenchmarkDotNet/Helpers/UnitHelper.cs @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Helpers; public static class UnitHelper { - public static readonly UnitPresentation DefaultPresentation = new (true, 0, gap: true); + public static readonly UnitPresentation DefaultPresentation = new(true, 0, gap: true); public static string ToDefaultString(this TimeInterval timeInterval, string? format = null) => timeInterval.ToString(format, null, DefaultPresentation); } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/UserInteractionHelper.cs b/src/BenchmarkDotNet/Helpers/UserInteractionHelper.cs index 7951d168dd..73e83d6ada 100644 --- a/src/BenchmarkDotNet/Helpers/UserInteractionHelper.cs +++ b/src/BenchmarkDotNet/Helpers/UserInteractionHelper.cs @@ -1,3 +1,4 @@ +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Helpers @@ -19,7 +20,7 @@ internal static class UserInteractionHelper /// public static string EscapeCommandExample(string input) { - return !RuntimeInformation.IsWindows() && input.IndexOf('*') >= 0 ? $"'{input}'" : input; + return !OsDetector.IsWindows() && input.IndexOf('*') >= 0 ? $"'{input}'" : input; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Helpers/XUnitHelper.cs b/src/BenchmarkDotNet/Helpers/XUnitHelper.cs index 559113a8f4..7f7415bdfa 100644 --- a/src/BenchmarkDotNet/Helpers/XUnitHelper.cs +++ b/src/BenchmarkDotNet/Helpers/XUnitHelper.cs @@ -6,5 +6,5 @@ namespace BenchmarkDotNet.Helpers; internal static class XUnitHelper { public static Lazy IsIntegrationTest = - new (() => AppDomain.CurrentDomain.GetAssemblies().Any(assembly => assembly.GetName().Name == "BenchmarkDotNet.IntegrationTests")); + new(() => AppDomain.CurrentDomain.GetAssemblies().Any(assembly => assembly.GetName().Name == "BenchmarkDotNet.IntegrationTests")); } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs b/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs index 8fe04e5764..0d5113db48 100644 --- a/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs +++ b/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Models; using BenchmarkDotNet.Portability; using JetBrains.Annotations; @@ -14,7 +16,10 @@ public sealed class EnvironmentMode : JobMode public static readonly Characteristic RuntimeCharacteristic = CreateCharacteristic(nameof(Runtime)); public static readonly Characteristic AffinityCharacteristic = CreateCharacteristic(nameof(Affinity)); public static readonly Characteristic GcCharacteristic = CreateCharacteristic(nameof(Gc)); - public static readonly Characteristic> EnvironmentVariablesCharacteristic = CreateCharacteristic>(nameof(EnvironmentVariables)); + + public static readonly Characteristic> EnvironmentVariablesCharacteristic = + CreateCharacteristic>(nameof(EnvironmentVariables)); + public static readonly Characteristic PowerPlanModeCharacteristic = CreateCharacteristic(nameof(PowerPlanMode)); public static readonly Characteristic LargeAddressAwareCharacteristic = CreateCharacteristic(nameof(LargeAddressAware)); @@ -60,7 +65,7 @@ public Jit Jit /// /// Runtime /// - public Runtime Runtime + public Runtime? Runtime { get { return RuntimeCharacteristic[this]; } set { RuntimeCharacteristic[this] = value; } @@ -107,7 +112,7 @@ public bool LargeAddressAware get => LargeAddressAwareCharacteristic[this]; set { - if (value && !RuntimeInformation.IsWindows()) + if (value && !OsDetector.IsWindows()) { throw new NotSupportedException("LargeAddressAware is a Windows-specific concept."); } @@ -133,5 +138,12 @@ public void SetEnvironmentVariable(EnvironmentVariable variable) } internal Runtime GetRuntime() => HasValue(RuntimeCharacteristic) ? Runtime : RuntimeInformation.GetCurrentRuntime(); + + internal BdnEnvironment ToPerfonar() => new() + { + Jit = HasValue(JitCharacteristic) ? Jit : null, + Runtime = HasValue(RuntimeCharacteristic) ? Runtime?.RuntimeMoniker : null, + Affinity = HasValue(AffinityCharacteristic) ? (int)Affinity : null + }; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs index 5e5f8b5369..277627d2e9 100644 --- a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs +++ b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.ComponentModel; using System.Diagnostics.CodeAnalysis; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; @@ -18,6 +20,9 @@ public sealed class InfrastructureMode : JobMode public static readonly Characteristic EngineFactoryCharacteristic = CreateCharacteristic(nameof(EngineFactory)); public static readonly Characteristic BuildConfigurationCharacteristic = CreateCharacteristic(nameof(BuildConfiguration)); public static readonly Characteristic> ArgumentsCharacteristic = CreateCharacteristic>(nameof(Arguments)); + + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This will soon be removed")] public static readonly Characteristic> NuGetReferencesCharacteristic = CreateCharacteristic>(nameof(NuGetReferences)); public static readonly InfrastructureMode InProcess = new InfrastructureMode(InProcessEmitToolchain.Instance); @@ -64,6 +69,8 @@ public IReadOnlyList Arguments set => ArgumentsCharacteristic[this] = value; } + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This will soon be removed")] public IReadOnlyCollection NuGetReferences { get => NuGetReferencesCharacteristic[this]; diff --git a/src/BenchmarkDotNet/Jobs/JobComparer.cs b/src/BenchmarkDotNet/Jobs/JobComparer.cs index 0a0c287523..ac1539ce7c 100644 --- a/src/BenchmarkDotNet/Jobs/JobComparer.cs +++ b/src/BenchmarkDotNet/Jobs/JobComparer.cs @@ -1,12 +1,23 @@ -using System; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Order; +using System; using System.Collections.Generic; -using BenchmarkDotNet.Characteristics; namespace BenchmarkDotNet.Jobs { internal class JobComparer : IComparer, IEqualityComparer { - public static readonly JobComparer Instance = new JobComparer(); + private readonly IComparer Comparer; + + public static readonly JobComparer Instance = new JobComparer(JobOrderPolicy.Default); + public static readonly JobComparer Numeric = new JobComparer(JobOrderPolicy.Numeric); + + public JobComparer(JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default) + { + Comparer = jobOrderPolicy == JobOrderPolicy.Default + ? StringComparer.Ordinal + : new NumericStringComparer(); // TODO: Use `StringComparer.Create(CultureInfo.InvariantCulture, CompareOptions.NumericOrdering)` for .NET10 or greater. + } public int Compare(Job x, Job y) { @@ -39,7 +50,7 @@ public int Compare(Job x, Job y) continue; } - int compare = string.CompareOrdinal( + int compare = Comparer.Compare( presenter.ToPresentation(x, characteristic), presenter.ToPresentation(y, characteristic)); if (compare != 0) @@ -52,5 +63,79 @@ public int Compare(Job x, Job y) public bool Equals(Job x, Job y) => Compare(x, y) == 0; public int GetHashCode(Job obj) => obj.Id.GetHashCode(); + + internal class NumericStringComparer : IComparer + { + public int Compare(string? x, string? y) + { + if (ReferenceEquals(x, y)) return 0; + if (x == null) return -1; + if (y == null) return 1; + + ReadOnlySpan spanX = x.AsSpan(); + ReadOnlySpan spanY = y.AsSpan(); + + int i = 0, j = 0; + + while (i < spanX.Length && j < spanY.Length) + { + char cx = spanX[i]; + char cy = spanY[j]; + + if (!char.IsDigit(cx) || !char.IsDigit(cy)) + { + int cmp = cx.CompareTo(cy); + if (cmp != 0) + return cmp; + + i++; + j++; + continue; + } + + int ixStart = i; + int iyStart = j; + + // Skip leading zeros + while (ixStart < spanX.Length && spanX[ixStart] == '0') ixStart++; + while (iyStart < spanY.Length && spanY[iyStart] == '0') iyStart++; + + int ix = ixStart; + int iy = iyStart; + + // Skip digits + while (ix < spanX.Length && char.IsDigit(spanX[ix])) ix++; + while (iy < spanY.Length && char.IsDigit(spanY[iy])) iy++; + + int lenX = ix - ixStart; + int lenY = iy - iyStart; + + // Compare by digits length + if (lenX != lenY) + return lenX.CompareTo(lenY); + + // Compare digits + for (int k = 0; k < lenX; k++) + { + int cmp = spanX[ixStart + k].CompareTo(spanY[iyStart + k]); + if (cmp != 0) + return cmp; + } + + // Compare by leading zeros + int leadingZerosX = ixStart - i; + int leadingZerosY = iyStart - j; + if (leadingZerosX != leadingZerosY) + return 0; // Leading zero differences are ignored (`CompareOptions.NumericOrdering` behavior of .NET) + + // Move to the next character after the digits + i = ix; + j = iy; + } + + // Compare remaining chars + return (spanX.Length - i).CompareTo(spanY.Length - j); + } + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 2c778df589..0424528ad4 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -339,6 +339,8 @@ public static Job WithEnvironmentVariable(this Job job, string key, string value /// (optional)Indicate the URI of the NuGet package source to use during the restore operation. /// (optional)Allows prerelease packages to be installed. /// + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This method will soon be removed, please start using .WithMsBuildArguments() instead.")] public static Job WithNuGet(this Job job, string packageName, string? packageVersion = null, Uri? source = null, bool prerelease = false) => job.WithCore(j => j.Infrastructure.NuGetReferences = new NuGetReferenceList(j.Infrastructure.NuGetReferences ?? Array.Empty()) @@ -352,9 +354,14 @@ public static Job WithNuGet(this Job job, string packageName, string? packageVer /// /// A collection of NuGet dependencies /// + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This method will soon be removed, please start using .WithMsBuildArguments() instead.")] public static Job WithNuGet(this Job job, NuGetReferenceList nuGetReferences) => job.WithCore(j => j.Infrastructure.NuGetReferences = nuGetReferences); + public static Job WithMsBuildArguments(this Job job, params string[] msBuildArguments) + => job.WithArguments([.. msBuildArguments.Select(a => new MsBuildArgument(a))]); + // Accuracy /// /// Maximum acceptable error for a benchmark (by default, BenchmarkDotNet continue iterations until the actual error is less than the specified error). @@ -437,8 +444,10 @@ private static Job WithCore(this Job job, Action updateCallback) return newJob; } - internal static bool HasDynamicBuildCharacteristic(this Job job) => - job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic) + internal static bool HasDynamicBuildCharacteristic(this Job job) +#pragma warning disable CS0618 // Type or member is obsolete + => job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic) +#pragma warning restore CS0618 // Type or member is obsolete || job.HasValue(InfrastructureMode.BuildConfigurationCharacteristic) || job.HasValue(InfrastructureMode.ArgumentsCharacteristic); } diff --git a/src/BenchmarkDotNet/Jobs/JobIdGenerator.cs b/src/BenchmarkDotNet/Jobs/JobIdGenerator.cs index fa8796d112..a208b3c9b7 100644 --- a/src/BenchmarkDotNet/Jobs/JobIdGenerator.cs +++ b/src/BenchmarkDotNet/Jobs/JobIdGenerator.cs @@ -10,12 +10,23 @@ public static string GenerateRandomId(Job job) string presentation = CharacteristicSetPresenter.Display.ToPresentation(job); if (presentation == "") return "DefaultJob"; - int seed = presentation.GetHashCode(); + int seed = GetStableHashCode(presentation); var random = new Random(seed); string id = ""; for (int i = 0; i < 6; i++) - id += (char) ('A' + random.Next(26)); + id += (char)('A' + random.Next(26)); return "Job-" + id; } + + // Compute string hash value with DJB2 algorithm. + private static int GetStableHashCode(string value) + { + uint hash = 5381; + foreach (char c in value) + { + hash = ((hash << 5) + hash) + c; // hash * 32 + hash + c + } + return unchecked((int)hash); + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/NugetReference.cs b/src/BenchmarkDotNet/Jobs/NugetReference.cs index 0db3a028de..89dc4c3734 100644 --- a/src/BenchmarkDotNet/Jobs/NugetReference.cs +++ b/src/BenchmarkDotNet/Jobs/NugetReference.cs @@ -1,8 +1,11 @@ using System; +using System.ComponentModel; using System.Text.RegularExpressions; namespace BenchmarkDotNet.Jobs { + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This type will soon be removed")] public class NuGetReference : IEquatable { public NuGetReference(string packageName, string packageVersion, Uri? source = null, bool prerelease = false) diff --git a/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs b/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs index 13c727c28f..025353b24f 100644 --- a/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs +++ b/src/BenchmarkDotNet/Jobs/NugetReferenceList.cs @@ -1,12 +1,15 @@ using System; using System.Collections; using System.Collections.Generic; +using System.ComponentModel; namespace BenchmarkDotNet.Jobs { /// /// An ordered list of NuGet references. Does not allow duplicate references with the same PackageName. /// + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This type will soon be removed")] public class NuGetReferenceList : IReadOnlyCollection { private readonly List references = new List(); diff --git a/src/BenchmarkDotNet/Jobs/RunMode.cs b/src/BenchmarkDotNet/Jobs/RunMode.cs index a76b5dc832..3e1e56c22f 100644 --- a/src/BenchmarkDotNet/Jobs/RunMode.cs +++ b/src/BenchmarkDotNet/Jobs/RunMode.cs @@ -2,6 +2,8 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Models; using Perfolizer.Horology; namespace BenchmarkDotNet.Jobs @@ -9,7 +11,8 @@ namespace BenchmarkDotNet.Jobs [SuppressMessage("ReSharper", "UnusedMember.Global")] public sealed class RunMode : JobMode { - public static readonly Characteristic RunStrategyCharacteristic = Characteristic.Create(nameof(RunStrategy), RunStrategy.Throughput); + public static readonly Characteristic RunStrategyCharacteristic = + Characteristic.Create(nameof(RunStrategy), RunStrategy.Throughput); public static readonly Characteristic LaunchCountCharacteristic = CreateCharacteristic(nameof(LaunchCount)); public static readonly Characteristic InvocationCountCharacteristic = CreateCharacteristic(nameof(InvocationCount)); @@ -61,13 +64,9 @@ public sealed class RunMode : JobMode }.Freeze(); - public RunMode() : this(null) - { - } + public RunMode() : this(null) { } - private RunMode(string id) : base(id) - { - } + private RunMode(string id) : base(id) { } /// /// Available values: Throughput and ColdStart. @@ -191,5 +190,21 @@ public bool MemoryRandomization get => MemoryRandomizationCharacteristic[this]; set => MemoryRandomizationCharacteristic[this] = value; } + + internal BdnExecution ToPerfonar() => new() + { + LaunchCount = HasValue(LaunchCountCharacteristic) ? LaunchCount : null, + WarmupCount = HasValue(WarmupCountCharacteristic) ? WarmupCount : null, + IterationCount = HasValue(IterationCountCharacteristic) ? IterationCount : null, + IterationTimeMs = HasValue(IterationTimeCharacteristic) ? IterationTime.ToMilliseconds().RoundToLong() : null, + InvocationCount = HasValue(InvocationCountCharacteristic) ? InvocationCount : null, + UnrollFactor = HasValue(UnrollFactorCharacteristic) ? UnrollFactor : null, + MinIterationCount = HasValue(MinIterationCountCharacteristic) ? MinIterationCount : null, + MaxIterationCount = HasValue(MaxIterationCountCharacteristic) ? MaxIterationCount : null, + MinWarmupIterationCount = HasValue(MinWarmupIterationCountCharacteristic) ? MinWarmupIterationCount : null, + MaxWarmupIterationCount = HasValue(MaxWarmupIterationCountCharacteristic) ? MaxWarmupIterationCount : null, + MemoryRandomization = HasValue(MemoryRandomizationCharacteristic) ? MemoryRandomization : null, + RunStrategy = HasValue(RunStrategyCharacteristic) ? RunStrategy : null, + }; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Loggers/Broker.cs b/src/BenchmarkDotNet/Loggers/Broker.cs index cec11a3091..277cd3c430 100644 --- a/src/BenchmarkDotNet/Loggers/Broker.cs +++ b/src/BenchmarkDotNet/Loggers/Broker.cs @@ -15,9 +15,7 @@ internal class Broker { private readonly ILogger logger; private readonly Process process; - private readonly IDiagnoser diagnoser; private readonly AnonymousPipeServerStream inputFromBenchmark, acknowledgments; - private readonly DiagnoserActionParameters diagnoserActionParameters; private readonly ManualResetEvent finished; public Broker(ILogger logger, Process process, IDiagnoser diagnoser, @@ -25,10 +23,10 @@ public Broker(ILogger logger, Process process, IDiagnoser diagnoser, { this.logger = logger; this.process = process; - this.diagnoser = diagnoser; + this.Diagnoser = diagnoser; this.inputFromBenchmark = inputFromBenchmark; this.acknowledgments = acknowledgments; - diagnoserActionParameters = new DiagnoserActionParameters(process, benchmarkCase, benchmarkId); + DiagnoserActionParameters = new DiagnoserActionParameters(process, benchmarkCase, benchmarkId); finished = new ManualResetEvent(false); Results = new List(); @@ -38,6 +36,10 @@ public Broker(ILogger logger, Process process, IDiagnoser diagnoser, process.Exited += OnProcessExited; } + internal IDiagnoser Diagnoser { get; } + + internal DiagnoserActionParameters DiagnoserActionParameters { get; } + internal List Results { get; } internal List PrefixedOutput { get; } @@ -74,8 +76,8 @@ private void OnProcessExited(object sender, EventArgs e) private void ProcessDataBlocking() { - using StreamReader reader = new (inputFromBenchmark, AnonymousPipesHost.UTF8NoBOM, detectEncodingFromByteOrderMarks: false); - using StreamWriter writer = new (acknowledgments, AnonymousPipesHost.UTF8NoBOM, bufferSize: 1); + using StreamReader reader = new(inputFromBenchmark, AnonymousPipesHost.UTF8NoBOM, detectEncodingFromByteOrderMarks: false); + using StreamWriter writer = new(acknowledgments, AnonymousPipesHost.UTF8NoBOM, bufferSize: 1); // Flush the data to the Stream after each write, otherwise the client will wait for input endlessly! writer.AutoFlush = true; @@ -90,7 +92,7 @@ private void ProcessDataBlocking() } else if (Engine.Signals.TryGetSignal(line, out var signal)) { - diagnoser?.Handle(signal, diagnoserActionParameters); + Diagnoser?.Handle(signal, DiagnoserActionParameters); writer.WriteLine(Engine.Signals.Acknowledgment); diff --git a/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs b/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs index 20aa2f7a54..d201f5da75 100644 --- a/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs +++ b/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Portability; using JetBrains.Annotations; @@ -15,7 +16,7 @@ public sealed class ConsoleLogger : ILogger public static readonly ILogger Ascii = new ConsoleLogger(false); public static readonly ILogger Unicode = new ConsoleLogger(true); private static readonly bool ConsoleSupportsColors - = !(RuntimeInformation.IsAndroid() || RuntimeInformation.IsIOS() || RuntimeInformation.IsWasm || RuntimeInformation.IsTvOS()); + = !(OsDetector.IsAndroid() || OsDetector.IsIOS() || RuntimeInformation.IsWasm || OsDetector.IsTvOS()); private readonly bool unicodeSupport; private readonly Dictionary colorScheme; diff --git a/src/BenchmarkDotNet/Models/BdnBenchmark.cs b/src/BenchmarkDotNet/Models/BdnBenchmark.cs new file mode 100644 index 0000000000..19eb4e9674 --- /dev/null +++ b/src/BenchmarkDotNet/Models/BdnBenchmark.cs @@ -0,0 +1,36 @@ +using System.Text; +using BenchmarkDotNet.Extensions; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Models; + +internal class BdnBenchmark : BenchmarkInfo +{ + public string Namespace { get; set; } = ""; + public string Type { get; set; } = ""; + public string Method { get; set; } = ""; + public string Parameters { get; set; } = ""; + public string? HardwareIntrinsics { get; set; } = ""; + + // TODO: Improve + public override string? GetDisplay() + { + if (Display != null) return Display; + + var builder = new StringBuilder(); + builder.Append($"{Namespace}"); + if (Type.IsNotBlank()) + { + if (builder.Length > 0) + builder.Append('.'); + builder.Append(Type); + } + if (Method.IsNotBlank()) + { + if (builder.Length > 0) + builder.Append('.'); + builder.Append(Method); + } + return builder.ToString(); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Models/BdnEnvironment.cs b/src/BenchmarkDotNet/Models/BdnEnvironment.cs new file mode 100644 index 0000000000..1256d83c6f --- /dev/null +++ b/src/BenchmarkDotNet/Models/BdnEnvironment.cs @@ -0,0 +1,12 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Models; + +internal class BdnEnvironment : EnvironmentInfo +{ + public RuntimeMoniker? Runtime { get; set; } + public Jit? Jit { get; set; } + public int? Affinity { get; set; } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Models/BdnExecution.cs b/src/BenchmarkDotNet/Models/BdnExecution.cs new file mode 100644 index 0000000000..0031c3897e --- /dev/null +++ b/src/BenchmarkDotNet/Models/BdnExecution.cs @@ -0,0 +1,83 @@ +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Engines; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Models; + +internal class BdnExecution : ExecutionInfo +{ + /// + /// Available values: Throughput and ColdStart. + /// Throughput: default strategy which allows to get good precision level. + /// ColdStart: should be used only for measuring cold start of the application or testing purpose. + /// Monitoring: no overhead evaluating, with several target iterations. Perfect for macrobenchmarks without a steady state with high variance. + /// + public RunStrategy? RunStrategy { get; set; } + + /// + /// How many times we should launch process with target benchmark. + /// + public int? LaunchCount { get; set; } + + /// + /// How many warmup iterations should be performed. + /// + public int? WarmupCount { get; set; } + + /// + /// How many target iterations should be performed + /// If specified, will be ignored. + /// If specified, will be ignored. + /// + public int? IterationCount { get; set; } + + /// + /// Desired time of execution of an iteration. Used by Pilot stage to estimate the number of invocations per iteration. + /// The default value is 500 milliseconds. + /// + public long? IterationTimeMs { get; set; } + + /// + /// Invocation count in a single iteration. + /// If specified, will be ignored. + /// If specified, it must be a multiple of . + /// + public long? InvocationCount { get; set; } + + /// + /// How many times the benchmark method will be invoked per one iteration of a generated loop. + /// + public int? UnrollFactor { get; set; } + + /// + /// Minimum count of target iterations that should be performed + /// The default value is 15 + /// If you set this value to below 15, then is not going to work + /// + public int? MinIterationCount { get; set; } + + /// + /// Maximum count of target iterations that should be performed + /// The default value is 100 + /// If you set this value to below 15, then is not going to work + /// + public int? MaxIterationCount { get; set; } + + /// + /// Minimum count of warmup iterations that should be performed + /// The default value is 6 + /// + public int? MinWarmupIterationCount { get; set; } + + /// + /// Maximum count of warmup iterations that should be performed + /// The default value is 50 + /// + public int? MaxWarmupIterationCount { get; set; } + + /// + /// specifies whether Engine should allocate some random-sized memory between iterations + /// it makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration + /// + public bool? MemoryRandomization { get; set; } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Models/BdnHostInfo.cs b/src/BenchmarkDotNet/Models/BdnHostInfo.cs new file mode 100644 index 0000000000..ca33e4d8ef --- /dev/null +++ b/src/BenchmarkDotNet/Models/BdnHostInfo.cs @@ -0,0 +1,14 @@ +using Perfolizer.Models; + +namespace BenchmarkDotNet.Models; + +internal class BdnHostInfo : HostInfo +{ + public string RuntimeVersion { get; set; } = ""; + public bool HasAttachedDebugger { get; set; } + public bool HasRyuJit { get; set; } + public string Configuration { get; set; } = ""; + public string DotNetSdkVersion { get; set; } = ""; + public double ChronometerFrequency { get; set; } + public string HardwareTimerKind { get; set; } = ""; +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Models/BdnLifecycle.cs b/src/BenchmarkDotNet/Models/BdnLifecycle.cs new file mode 100644 index 0000000000..a93a325977 --- /dev/null +++ b/src/BenchmarkDotNet/Models/BdnLifecycle.cs @@ -0,0 +1,55 @@ +using System; +using BenchmarkDotNet.Engines; +using JetBrains.Annotations; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Models; + +internal class BdnLifecycle : LifecycleInfo, IEquatable, IComparable +{ + public int LaunchIndex { get; set; } + public IterationStage IterationStage { get; set; } = IterationStage.Unknown; + public IterationMode IterationMode { get; set; } = IterationMode.Unknown; + + public bool Equals(BdnLifecycle? other) + { + if (ReferenceEquals(null, other)) + return false; + if (ReferenceEquals(this, other)) + return true; + return LaunchIndex == other.LaunchIndex && + IterationMode == other.IterationMode && + IterationStage == other.IterationStage; + } + + public override bool Equals(object? obj) + { + if (ReferenceEquals(null, obj)) + return false; + if (ReferenceEquals(this, obj)) + return true; + if (obj.GetType() != GetType()) + return false; + return Equals((BdnLifecycle)obj); + } + + public override int GetHashCode() => HashCode.Combine(LaunchIndex, IterationMode, IterationStage); + + public static bool operator ==(BdnLifecycle? left, BdnLifecycle? right) => Equals(left, right); + public static bool operator !=(BdnLifecycle? left, BdnLifecycle? right) => !Equals(left, right); + + public int CompareTo(BdnLifecycle? other) + { + if (ReferenceEquals(this, other)) + return 0; + if (ReferenceEquals(null, other)) + return 1; + int launchIndexComparison = LaunchIndex.CompareTo(other.LaunchIndex); + if (launchIndexComparison != 0) + return launchIndexComparison; + int iterationStageComparison = IterationStage.CompareTo(other.IterationStage); + if (iterationStageComparison != 0) + return iterationStageComparison; + return IterationMode.CompareTo(other.IterationMode); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Models/BdnSchema.cs b/src/BenchmarkDotNet/Models/BdnSchema.cs new file mode 100644 index 0000000000..3006b702de --- /dev/null +++ b/src/BenchmarkDotNet/Models/BdnSchema.cs @@ -0,0 +1,17 @@ +using Perfolizer.Perfonar.Base; + +namespace BenchmarkDotNet.Models; + +internal class BdnSchema : PerfonarSchema +{ + public static readonly BdnSchema Instance = new(); + + private BdnSchema() : base("bdn") + { + Add(); + Add(); + Add(); + Add(); + Add(); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Order/CategoryComparer.cs b/src/BenchmarkDotNet/Order/CategoryComparer.cs index a64fb7c588..ac11695a59 100644 --- a/src/BenchmarkDotNet/Order/CategoryComparer.cs +++ b/src/BenchmarkDotNet/Order/CategoryComparer.cs @@ -7,7 +7,7 @@ namespace BenchmarkDotNet.Order internal class CategoryComparer : IComparer { private const string Separator = "§"; - public static readonly CategoryComparer Instance = new (); + public static readonly CategoryComparer Instance = new(); public int Compare(string[] x, string[] y) { diff --git a/src/BenchmarkDotNet/Order/DefaultOrderer.cs b/src/BenchmarkDotNet/Order/DefaultOrderer.cs index 656de7ce98..d0dc2d475b 100644 --- a/src/BenchmarkDotNet/Order/DefaultOrderer.cs +++ b/src/BenchmarkDotNet/Order/DefaultOrderer.cs @@ -19,7 +19,7 @@ public class DefaultOrderer : IOrderer private readonly IComparer categoryComparer = CategoryComparer.Instance; private readonly IComparer paramsComparer = ParameterComparer.Instance; - private readonly IComparer jobComparer = JobComparer.Instance; + private readonly IComparer jobComparer; private readonly IComparer targetComparer; public SummaryOrderPolicy SummaryOrderPolicy { get; } @@ -27,10 +27,14 @@ public class DefaultOrderer : IOrderer public DefaultOrderer( SummaryOrderPolicy summaryOrderPolicy = SummaryOrderPolicy.Default, - MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared) + MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared, + JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default) { SummaryOrderPolicy = summaryOrderPolicy; MethodOrderPolicy = methodOrderPolicy; + jobComparer = jobOrderPolicy == JobOrderPolicy.Default + ? JobComparer.Instance + : JobComparer.Numeric; targetComparer = new DescriptorComparer(methodOrderPolicy); } diff --git a/src/BenchmarkDotNet/Order/JobOrderPolicy.cs b/src/BenchmarkDotNet/Order/JobOrderPolicy.cs new file mode 100644 index 0000000000..be1d66186d --- /dev/null +++ b/src/BenchmarkDotNet/Order/JobOrderPolicy.cs @@ -0,0 +1,14 @@ +namespace BenchmarkDotNet.Order; + +public enum JobOrderPolicy +{ + /// + /// Compare job characteristics in ordinal order. + /// + Default, + + /// + /// Compare job characteristics in numeric order. + /// + Numeric, +} diff --git a/src/BenchmarkDotNet/Parameters/SmartParamBuilder.cs b/src/BenchmarkDotNet/Parameters/SmartParamBuilder.cs index 29814d8179..49ea1b162f 100644 --- a/src/BenchmarkDotNet/Parameters/SmartParamBuilder.cs +++ b/src/BenchmarkDotNet/Parameters/SmartParamBuilder.cs @@ -99,8 +99,20 @@ public string ToSourceCode() ? $"[{argumentIndex}]" // IEnumerable : string.Empty; // IEnumerable + string methodCall; + if ((source as MethodInfo)?.IsStatic ?? (source as PropertyInfo)?.GetMethod.IsStatic ?? throw new Exception($"{nameof(source)} was not {nameof(MethodInfo)} nor {nameof(PropertyInfo)}")) + { + // If the source member is static, we need to place the fully qualified type name before it, in case the source member is from another type that this generated type does not inherit from. + methodCall = $"{source.DeclaringType.GetCorrectCSharpTypeName()}.{source.Name}"; + } + else + { + // If the source member is non-static, we mustn't include the type name, as this would be a compiler error when accessing a non-static source member in the base class of this generated type. + methodCall = source.Name; + } + // we do something like enumerable.ElementAt(sourceIndex)[argumentIndex]; - return $"{cast}BenchmarkDotNet.Parameters.ParameterExtractor.GetParameter({source.Name}{callPostfix}, {sourceIndex}){indexPostfix};"; + return $"{cast}BenchmarkDotNet.Parameters.ParameterExtractor.GetParameter({methodCall}{callPostfix}, {sourceIndex}){indexPostfix};"; } } diff --git a/src/BenchmarkDotNet/Portability/Cpu/CpuInfo.cs b/src/BenchmarkDotNet/Portability/Cpu/CpuInfo.cs deleted file mode 100644 index 8c3227a25d..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/CpuInfo.cs +++ /dev/null @@ -1,37 +0,0 @@ -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - public class CpuInfo - { - public string ProcessorName { get; } - public int? PhysicalProcessorCount { get; } - public int? PhysicalCoreCount { get; } - public int? LogicalCoreCount { get; } - public Frequency? NominalFrequency { get; } - public Frequency? MinFrequency { get; } - public Frequency? MaxFrequency { get; } - - internal CpuInfo(string processorName, Frequency? nominalFrequency) - : this(processorName, null, null, null, nominalFrequency, null, null) - { - } - - public CpuInfo(string processorName, - int? physicalProcessorCount, - int? physicalCoreCount, - int? logicalCoreCount, - Frequency? nominalFrequency, - Frequency? minFrequency, - Frequency? maxFrequency) - { - ProcessorName = processorName; - PhysicalProcessorCount = physicalProcessorCount; - PhysicalCoreCount = physicalCoreCount; - LogicalCoreCount = logicalCoreCount; - NominalFrequency = nominalFrequency; - MinFrequency = minFrequency; - MaxFrequency = maxFrequency; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/CpuInfoFormatter.cs b/src/BenchmarkDotNet/Portability/Cpu/CpuInfoFormatter.cs deleted file mode 100644 index e016ad1a4b..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/CpuInfoFormatter.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Environments; - -namespace BenchmarkDotNet.Portability.Cpu -{ - public static class CpuInfoFormatter - { - public static string Format(CpuInfo cpuInfo) - { - if (cpuInfo == null) - { - return "Unknown processor"; - } - - var parts = new List - { - ProcessorBrandStringHelper.Prettify(cpuInfo, includeMaxFrequency: true) - }; - - if (cpuInfo.PhysicalProcessorCount > 0) - parts.Add($", {cpuInfo.PhysicalProcessorCount} CPU"); - - if (cpuInfo.LogicalCoreCount == 1) - parts.Add(", 1 logical core"); - - if (cpuInfo.LogicalCoreCount > 1) - parts.Add($", {cpuInfo.LogicalCoreCount} logical cores"); - - if (cpuInfo.LogicalCoreCount > 0 && cpuInfo.PhysicalCoreCount > 0) - parts.Add(" and "); - else if (cpuInfo.PhysicalCoreCount > 0) - parts.Add(", "); - - if (cpuInfo.PhysicalCoreCount == 1) - parts.Add("1 physical core"); - if (cpuInfo.PhysicalCoreCount > 1) - parts.Add($"{cpuInfo.PhysicalCoreCount} physical cores"); - - string result = string.Join("", parts); - // The line with ProcessorBrandString is one of the longest lines in the summary. - // When people past in on GitHub, it can be a reason of an ugly horizontal scrollbar. - // To avoid this, we are trying to minimize this line and use the minimum possible number of characters. - // Here we are removing the repetitive "cores" word. - if (result.Contains("logical cores") && result.Contains("physical cores")) - result = result.Replace("logical cores", "logical"); - - return result; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/MosCpuInfoProvider.cs b/src/BenchmarkDotNet/Portability/Cpu/MosCpuInfoProvider.cs deleted file mode 100644 index 2edc7cabcb..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/MosCpuInfoProvider.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Management; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class MosCpuInfoProvider - { -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatform("windows")] -#endif - internal static readonly Lazy MosCpuInfo = new Lazy(Load); - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatform("windows")] -#endif - private static CpuInfo Load() - { - var processorModelNames = new HashSet(); - uint physicalCoreCount = 0; - uint logicalCoreCount = 0; - int processorsCount = 0; - uint nominalClockSpeed = 0; - uint maxClockSpeed = 0; - uint minClockSpeed = 0; - - - using (var mosProcessor = new ManagementObjectSearcher("SELECT * FROM Win32_Processor")) - { - foreach (var moProcessor in mosProcessor.Get().Cast()) - { - string name = moProcessor[WmicCpuInfoKeyNames.Name]?.ToString(); - if (!string.IsNullOrEmpty(name)) - { - processorModelNames.Add(name); - processorsCount++; - physicalCoreCount += (uint) moProcessor[WmicCpuInfoKeyNames.NumberOfCores]; - logicalCoreCount += (uint) moProcessor[WmicCpuInfoKeyNames.NumberOfLogicalProcessors]; - maxClockSpeed = (uint) moProcessor[WmicCpuInfoKeyNames.MaxClockSpeed]; - } - } - } - - return new CpuInfo( - processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null, - processorsCount > 0 ? processorsCount : (int?) null, - physicalCoreCount > 0 ? (int?) physicalCoreCount : null, - logicalCoreCount > 0 ? (int?) logicalCoreCount : null, - nominalClockSpeed > 0 && logicalCoreCount > 0 ? Frequency.FromMHz(nominalClockSpeed) : (Frequency?) null, - minClockSpeed > 0 && logicalCoreCount > 0 ? Frequency.FromMHz(minClockSpeed) : (Frequency?) null, - maxClockSpeed > 0 && logicalCoreCount > 0 ? Frequency.FromMHz(maxClockSpeed) : (Frequency?) null); - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoKeyNames.cs b/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoKeyNames.cs deleted file mode 100644 index 60284c5363..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoKeyNames.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class ProcCpuInfoKeyNames - { - internal const string PhysicalId = "physical id"; - internal const string CpuCores = "cpu cores"; - internal const string ModelName = "model name"; - internal const string MaxFrequency = "max freq"; - internal const string MinFrequency = "min freq"; - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoParser.cs b/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoParser.cs deleted file mode 100644 index e3d0a14c46..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoParser.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; -using BenchmarkDotNet.Helpers; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class ProcCpuInfoParser - { - internal static CpuInfo ParseOutput(string? content) - { - var logicalCores = SectionsHelper.ParseSections(content, ':'); - var processorModelNames = new HashSet(); - var processorsToPhysicalCoreCount = new Dictionary(); - - int logicalCoreCount = 0; - var nominalFrequency = Frequency.Zero; - var minFrequency = Frequency.Zero; - var maxFrequency = Frequency.Zero; - - foreach (var logicalCore in logicalCores) - { - if (logicalCore.TryGetValue(ProcCpuInfoKeyNames.PhysicalId, out string physicalId) && - logicalCore.TryGetValue(ProcCpuInfoKeyNames.CpuCores, out string cpuCoresValue) && - int.TryParse(cpuCoresValue, out int cpuCoreCount) && - cpuCoreCount > 0) - processorsToPhysicalCoreCount[physicalId] = cpuCoreCount; - - if (logicalCore.TryGetValue(ProcCpuInfoKeyNames.ModelName, out string modelName)) - { - processorModelNames.Add(modelName); - nominalFrequency = ParseFrequencyFromBrandString(modelName); - logicalCoreCount++; - } - - if (logicalCore.TryGetValue(ProcCpuInfoKeyNames.MinFrequency, out string minCpuFreqValue) - && Frequency.TryParseMHz(minCpuFreqValue, out var minCpuFreq)) - { - minFrequency = minCpuFreq; - } - - if (logicalCore.TryGetValue(ProcCpuInfoKeyNames.MaxFrequency, out string maxCpuFreqValue) - && Frequency.TryParseMHz(maxCpuFreqValue, out var maxCpuFreq)) - { - maxFrequency = maxCpuFreq; - } - } - - return new CpuInfo( - processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null, - processorsToPhysicalCoreCount.Count > 0 ? processorsToPhysicalCoreCount.Count : (int?) null, - processorsToPhysicalCoreCount.Count > 0 ? processorsToPhysicalCoreCount.Values.Sum() : (int?) null, - logicalCoreCount > 0 ? logicalCoreCount : (int?) null, - nominalFrequency > 0 ? nominalFrequency : (Frequency?) null, - minFrequency > 0 ? minFrequency : (Frequency?) null, - maxFrequency > 0 ? maxFrequency : (Frequency?) null); - } - - internal static Frequency ParseFrequencyFromBrandString(string brandString) - { - const string pattern = "(\\d.\\d+)GHz"; - var matches = Regex.Matches(brandString, pattern, RegexOptions.IgnoreCase); - if (matches.Count > 0 && matches[0].Groups.Count > 1) - { - string match = Regex.Matches(brandString, pattern, RegexOptions.IgnoreCase)[0].Groups[1].ToString(); - return Frequency.TryParseGHz(match, out var result) ? result : Frequency.Zero; - } - - return 0d; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoProvider.cs b/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoProvider.cs deleted file mode 100644 index 7cadad1ff2..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/ProcCpuInfoProvider.cs +++ /dev/null @@ -1,75 +0,0 @@ -using System; -using System.Linq; -using System.Text; -using BenchmarkDotNet.Helpers; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - /// - /// CPU information from output of the `cat /proc/info` command. - /// Linux only. - /// - internal static class ProcCpuInfoProvider - { - internal static readonly Lazy ProcCpuInfo = new (Load); - - private static CpuInfo? Load() - { - if (RuntimeInformation.IsLinux()) - { - string content = ProcessHelper.RunAndReadOutput("cat", "/proc/cpuinfo") ?? ""; - string output = GetCpuSpeed() ?? ""; - content += output; - return ProcCpuInfoParser.ParseOutput(content); - } - return null; - } - - private static string? GetCpuSpeed() - { - try - { - string[]? output = ProcessHelper.RunAndReadOutput("/bin/bash", "-c \"lscpu | grep MHz\"")? - .Split('\n') - .SelectMany(x => x.Split(':')) - .ToArray(); - - return ParseCpuFrequencies(output); - } - catch (Exception) - { - return null; - } - } - - private static string? ParseCpuFrequencies(string[]? input) - { - // Example of output we trying to parse: - // - // CPU MHz: 949.154 - // CPU max MHz: 3200,0000 - // CPU min MHz: 800,0000 - - if (input == null) - return null; - - var output = new StringBuilder(); - for (int i = 0; i + 1 < input.Length; i += 2) - { - string name = input[i].Trim(); - string value = input[i + 1].Trim(); - - if (name.EqualsWithIgnoreCase("CPU min MHz")) - if (Frequency.TryParseMHz(value.Replace(',', '.'), out var minFrequency)) - output.Append($"\n{ProcCpuInfoKeyNames.MinFrequency}\t:{minFrequency.ToMHz()}"); - - if (name.EqualsWithIgnoreCase("CPU max MHz")) - if (Frequency.TryParseMHz(value.Replace(',', '.'), out var maxFrequency)) - output.Append($"\n{ProcCpuInfoKeyNames.MaxFrequency}\t:{maxFrequency.ToMHz()}"); - } - - return output.ToString(); - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoParser.cs b/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoParser.cs deleted file mode 100644 index 7e61bbd3ec..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoParser.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Extensions; - -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class SysctlCpuInfoParser - { - [SuppressMessage("ReSharper", "StringLiteralTypo")] - internal static CpuInfo ParseOutput(string? content) - { - var sysctl = SectionsHelper.ParseSection(content, ':'); - string processorName = sysctl.GetValueOrDefault("machdep.cpu.brand_string"); - var physicalProcessorCount = GetPositiveIntValue(sysctl, "hw.packages"); - var physicalCoreCount = GetPositiveIntValue(sysctl, "hw.physicalcpu"); - var logicalCoreCount = GetPositiveIntValue(sysctl, "hw.logicalcpu"); - var nominalFrequency = GetPositiveLongValue(sysctl, "hw.cpufrequency"); - var minFrequency = GetPositiveLongValue(sysctl, "hw.cpufrequency_min"); - var maxFrequency = GetPositiveLongValue(sysctl, "hw.cpufrequency_max"); - return new CpuInfo(processorName, physicalProcessorCount, physicalCoreCount, logicalCoreCount, nominalFrequency, minFrequency, maxFrequency); - } - - private static int? GetPositiveIntValue(Dictionary sysctl, string keyName) - { - if (sysctl.TryGetValue(keyName, out string value) && - int.TryParse(value, out int result) && - result > 0) - return result; - return null; - } - - private static long? GetPositiveLongValue(Dictionary sysctl, string keyName) - { - if (sysctl.TryGetValue(keyName, out string value) && - long.TryParse(value, out long result) && - result > 0) - return result; - return null; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoProvider.cs b/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoProvider.cs deleted file mode 100644 index 32a207c1c8..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/SysctlCpuInfoProvider.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using BenchmarkDotNet.Helpers; - -namespace BenchmarkDotNet.Portability.Cpu -{ - /// - /// CPU information from output of the `sysctl -a` command. - /// MacOSX only. - /// - internal static class SysctlCpuInfoProvider - { - internal static readonly Lazy SysctlCpuInfo = new Lazy(Load); - - private static CpuInfo? Load() - { - if (RuntimeInformation.IsMacOS()) - { - string content = ProcessHelper.RunAndReadOutput("sysctl", "-a"); - return SysctlCpuInfoParser.ParseOutput(content); - } - return null; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoKeyNames.cs b/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoKeyNames.cs deleted file mode 100644 index 9dd2209304..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoKeyNames.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class WmicCpuInfoKeyNames - { - internal const string NumberOfLogicalProcessors = "NumberOfLogicalProcessors"; - internal const string NumberOfCores = "NumberOfCores"; - internal const string Name = "Name"; - internal const string MaxClockSpeed = "MaxClockSpeed"; - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoParser.cs b/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoParser.cs deleted file mode 100644 index 441b1a8b10..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoParser.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Helpers; -using Perfolizer.Horology; - -namespace BenchmarkDotNet.Portability.Cpu -{ - internal static class WmicCpuInfoParser - { - internal static CpuInfo ParseOutput(string? content) - { - var processors = SectionsHelper.ParseSections(content, '='); - - var processorModelNames = new HashSet(); - int physicalCoreCount = 0; - int logicalCoreCount = 0; - int processorsCount = 0; - - var currentClockSpeed = Frequency.Zero; - var maxClockSpeed = Frequency.Zero; - var minClockSpeed = Frequency.Zero; - - foreach (var processor in processors) - { - if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfCores, out string numberOfCoresValue) && - int.TryParse(numberOfCoresValue, out int numberOfCores) && - numberOfCores > 0) - physicalCoreCount += numberOfCores; - - if (processor.TryGetValue(WmicCpuInfoKeyNames.NumberOfLogicalProcessors, out string numberOfLogicalValue) && - int.TryParse(numberOfLogicalValue, out int numberOfLogical) && - numberOfLogical > 0) - logicalCoreCount += numberOfLogical; - - if (processor.TryGetValue(WmicCpuInfoKeyNames.Name, out string name)) - { - processorModelNames.Add(name); - processorsCount++; - } - - if (processor.TryGetValue(WmicCpuInfoKeyNames.MaxClockSpeed, out string frequencyValue) - && int.TryParse(frequencyValue, out int frequency) - && frequency > 0) - { - maxClockSpeed += frequency; - } - } - - return new CpuInfo( - processorModelNames.Count > 0 ? string.Join(", ", processorModelNames) : null, - processorsCount > 0 ? processorsCount : (int?) null, - physicalCoreCount > 0 ? physicalCoreCount : (int?) null, - logicalCoreCount > 0 ? logicalCoreCount : (int?) null, - currentClockSpeed > 0 && processorsCount > 0 ? Frequency.FromMHz(currentClockSpeed / processorsCount) : (Frequency?) null, - minClockSpeed > 0 && processorsCount > 0 ? Frequency.FromMHz(minClockSpeed / processorsCount) : (Frequency?) null, - maxClockSpeed > 0 && processorsCount > 0 ? Frequency.FromMHz(maxClockSpeed / processorsCount) : (Frequency?) null); - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoProvider.cs b/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoProvider.cs deleted file mode 100644 index d259200701..0000000000 --- a/src/BenchmarkDotNet/Portability/Cpu/WmicCpuInfoProvider.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; -using System.IO; -using BenchmarkDotNet.Helpers; - -namespace BenchmarkDotNet.Portability.Cpu -{ - /// - /// CPU information from output of the `wmic cpu get Name, NumberOfCores, NumberOfLogicalProcessors /Format:List` command. - /// Windows only. - /// - internal static class WmicCpuInfoProvider - { - internal static readonly Lazy WmicCpuInfo = new (Load); - - private const string DefaultWmicPath = @"C:\Windows\System32\wbem\WMIC.exe"; - - private static CpuInfo? Load() - { - if (RuntimeInformation.IsWindows()) - { - const string argList = $"{WmicCpuInfoKeyNames.Name}, {WmicCpuInfoKeyNames.NumberOfCores}, " + - $"{WmicCpuInfoKeyNames.NumberOfLogicalProcessors}, {WmicCpuInfoKeyNames.MaxClockSpeed}"; - string wmicPath = File.Exists(DefaultWmicPath) ? DefaultWmicPath : "wmic"; - string content = ProcessHelper.RunAndReadOutput(wmicPath, $"cpu get {argList} /Format:List"); - return WmicCpuInfoParser.ParseOutput(content); - } - return null; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs index 6064b24dfa..9b8348a125 100644 --- a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs +++ b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs @@ -2,20 +2,16 @@ using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; -using System.IO; using System.Linq; using System.Management; using System.Reflection; using System.Runtime.InteropServices; using System.Text.RegularExpressions; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Portability.Cpu; -using JetBrains.Annotations; -using Microsoft.Win32; using static System.Runtime.InteropServices.RuntimeInformation; -using RuntimeEnvironment = Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment; namespace BenchmarkDotNet.Portability { @@ -25,46 +21,36 @@ internal static class RuntimeInformation internal const string ReleaseConfigurationName = "RELEASE"; internal const string Unknown = "?"; + // Many of these checks allocate and/or are expensive to compute. We store the results in static readonly fields to keep Engine non-allocating. + // Static readonly fields are used instead of properties to avoid an extra getter method call that might not be tier1 jitted. + // This class is internal, so we don't need to expose these as properties. + /// /// returns true for both the old (implementation of .NET Framework) and new Mono (.NET 6+ flavour) /// - public static bool IsMono { get; } = Type.GetType("Mono.RuntimeStructs") != null; // it allocates a lot of memory, we need to check it once in order to keep Engine non-allocating! + public static readonly bool IsMono = Type.GetType("Mono.RuntimeStructs") != null; - public static bool IsOldMono { get; } = Type.GetType("Mono.Runtime") != null; + public static readonly bool IsOldMono = Type.GetType("Mono.Runtime") != null; - public static bool IsNewMono { get; } = IsMono && !IsOldMono; + public static readonly bool IsNewMono = IsMono && !IsOldMono; - public static bool IsFullFramework => + public static readonly bool IsFullFramework = #if NET6_0_OR_GREATER + // This could be const, but we want to avoid unreachable code warnings. false; #else FrameworkDescription.StartsWith(".NET Framework", StringComparison.OrdinalIgnoreCase); #endif - [PublicAPI] - public static bool IsNetNative => FrameworkDescription.StartsWith(".NET Native", StringComparison.OrdinalIgnoreCase); - - public static bool IsNetCore - => ((Environment.Version.Major >= 5) || FrameworkDescription.StartsWith(".NET Core", StringComparison.OrdinalIgnoreCase)) - && !string.IsNullOrEmpty(typeof(object).Assembly.Location); - - public static bool IsNativeAOT - => Environment.Version.Major >= 5 - && string.IsNullOrEmpty(typeof(object).Assembly.Location) // it's merged to a single .exe and .Location returns null - && !IsWasm; // Wasm also returns "" for assembly locations - #if NET6_0_OR_GREATER [System.Runtime.Versioning.SupportedOSPlatformGuard("browser")] -#endif - public static bool IsWasm => -#if NET6_0_OR_GREATER - OperatingSystem.IsBrowser(); + public static readonly bool IsWasm = OperatingSystem.IsBrowser(); #else - IsOSPlatform(OSPlatform.Create("BROWSER")); + public static readonly bool IsWasm = IsOSPlatform(OSPlatform.Create("BROWSER")); #endif #if NETSTANDARD2_0 - public static bool IsAot { get; } = IsAotMethod(); // This allocates, so we only want to call it once statically. + public static readonly bool IsAot = IsAotMethod() || FrameworkDescription.StartsWith(".NET Native", StringComparison.OrdinalIgnoreCase); private static bool IsAotMethod() { @@ -82,175 +68,34 @@ private static bool IsAotMethod() return false; } #else - public static bool IsAot => !System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeCompiled; -#endif - - public static bool IsRunningInContainer => string.Equals(Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"), "true"); - - internal static string ExecutableExtension => IsWindows() ? ".exe" : string.Empty; - - internal static string ScriptFileExtension => IsWindows() ? ".bat" : ".sh"; - - internal static string GetArchitecture() => GetCurrentPlatform().ToString(); - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("windows")] -#endif - internal static bool IsWindows() => -#if NET6_0_OR_GREATER - OperatingSystem.IsWindows(); // prefer linker-friendly OperatingSystem APIs -#else - IsOSPlatform(OSPlatform.Windows); -#endif - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("linux")] -#endif - internal static bool IsLinux() => -#if NET6_0_OR_GREATER - OperatingSystem.IsLinux(); -#else - IsOSPlatform(OSPlatform.Linux); -#endif - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("macos")] -#endif - internal static bool IsMacOS() => -#if NET6_0_OR_GREATER - OperatingSystem.IsMacOS(); -#else - IsOSPlatform(OSPlatform.OSX); -#endif - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("android")] -#endif - internal static bool IsAndroid() => -#if NET6_0_OR_GREATER - OperatingSystem.IsAndroid(); -#else - Type.GetType("Java.Lang.Object, Mono.Android") != null; + public static readonly bool IsAot = !System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeCompiled; #endif -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("ios")] -#endif - internal static bool IsIOS() => -#if NET6_0_OR_GREATER - OperatingSystem.IsIOS(); -#else - Type.GetType("Foundation.NSObject, Xamarin.iOS") != null; -#endif - -#if NET6_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatformGuard("tvos")] -#endif - internal static bool IsTvOS() => -#if NET6_0_OR_GREATER - OperatingSystem.IsTvOS(); -#else - IsOSPlatform(OSPlatform.Create("TVOS")); -#endif - - public static string GetOsVersion() - { - if (IsMacOS()) - { - string systemVersion = ExternalToolsHelper.MacSystemProfilerData.Value.GetValueOrDefault("System Version") ?? ""; - string kernelVersion = ExternalToolsHelper.MacSystemProfilerData.Value.GetValueOrDefault("Kernel Version") ?? ""; - if (!string.IsNullOrEmpty(systemVersion) && !string.IsNullOrEmpty(kernelVersion)) - return OsBrandStringHelper.PrettifyMacOSX(systemVersion, kernelVersion); - } - - if (IsLinux()) - { - string? version = GetLinuxOsVersion(); - if (version != null && version.Trim() != "") - return version; - } - - string operatingSystem = RuntimeEnvironment.OperatingSystem; - string operatingSystemVersion = RuntimeEnvironment.OperatingSystemVersion; - - return OsBrandStringHelper.Prettify( - operatingSystem, - operatingSystemVersion, - GetWindowsUbr()); - } - - private static string? GetLinuxOsVersion() - { - if (!IsLinux()) - return null; - try - { - string version = LinuxOsReleaseHelper.GetNameByOsRelease(File.ReadAllLines("/etc/os-release")); - bool wsl = IsUnderWsl(); - return wsl ? version + " WSL" : version; - } - catch (Exception) - { - return null; - } - } + public static bool IsNetCore + => ((Environment.Version.Major >= 5) || FrameworkDescription.StartsWith(".NET Core", StringComparison.OrdinalIgnoreCase)) + && !IsAot; - private static bool IsUnderWsl() - { - if (!IsLinux()) - return false; - try - { - return File.Exists("/proc/sys/fs/binfmt_misc/WSLInterop"); // https://superuser.com/a/1749811 - } - catch (Exception) - { - return false; - } - } + public static bool IsNativeAOT + => Environment.Version.Major >= 5 + && IsAot + && !IsWasm && !IsMono; // Wasm and MonoAOTLLVM are also AOT - // TODO: Introduce a common util API for registry calls, use it also in BenchmarkDotNet.Toolchains.CsProj.GetCurrentVersionBasedOnWindowsRegistry - /// - /// On Windows, this method returns UBR (Update Build Revision) based on Registry. - /// Returns null if the value is not available - /// - /// - private static int? GetWindowsUbr() - { - if (IsWindows()) - { - try - { - using (var baseKey = RegistryKey.OpenBaseKey(RegistryHive.LocalMachine, RegistryView.Registry32)) - using (var ndpKey = baseKey.OpenSubKey(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion")) - { - if (ndpKey == null) - return null; - return Convert.ToInt32(ndpKey.GetValue("UBR")); - } - } - catch (Exception) - { - return null; - } - } - return null; - } + public static readonly bool IsTieredJitEnabled = + IsNetCore + && (Environment.Version.Major < 3 + // Disabled by default in netcoreapp2.X, check if it's enabled. + ? Environment.GetEnvironmentVariable("COMPlus_TieredCompilation") == "1" + || Environment.GetEnvironmentVariable("DOTNET_TieredCompilation") == "1" + || (AppContext.TryGetSwitch("System.Runtime.TieredCompilation", out bool isEnabled) && isEnabled) + // Enabled by default in netcoreapp3.0+, check if it's disabled. + : Environment.GetEnvironmentVariable("COMPlus_TieredCompilation") != "0" + && Environment.GetEnvironmentVariable("DOTNET_TieredCompilation") != "0" + && (!AppContext.TryGetSwitch("System.Runtime.TieredCompilation", out isEnabled) || isEnabled)); - internal static CpuInfo GetCpuInfo() - { - if (IsWindows() && IsFullFramework && !IsMono) - return MosCpuInfoProvider.MosCpuInfo.Value; - if (IsWindows()) - return WmicCpuInfoProvider.WmicCpuInfo.Value; - if (IsLinux()) - return ProcCpuInfoProvider.ProcCpuInfo.Value; - if (IsMacOS()) - return SysctlCpuInfoProvider.SysctlCpuInfo.Value; + public static readonly bool IsRunningInContainer = string.Equals(Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"), "true"); - return null; - } + internal static string GetArchitecture() => GetCurrentPlatform().ToString(); internal static string GetRuntimeVersion() { @@ -315,28 +160,25 @@ internal static string GetRuntimeVersion() private static string GetNetCoreVersion() { - if (IsAndroid()) + if (OsDetector.IsAndroid()) { return $".NET {Environment.Version}"; } - else - { - var coreclrAssemblyInfo = FileVersionInfo.GetVersionInfo(typeof(object).GetTypeInfo().Assembly.Location); - var corefxAssemblyInfo = FileVersionInfo.GetVersionInfo(typeof(Regex).GetTypeInfo().Assembly.Location); - if (CoreRuntime.TryGetVersion(out var version) && version >= new Version(5, 0)) - { - // after the merge of dotnet/corefx and dotnet/coreclr into dotnet/runtime the version should always be the same - Debug.Assert(coreclrAssemblyInfo.FileVersion == corefxAssemblyInfo.FileVersion); - - return $".NET {version} ({coreclrAssemblyInfo.FileVersion})"; - } - else - { - string runtimeVersion = version != default ? version.ToString() : Unknown; + return CoreRuntime.TryGetVersion(out var version) && version.Major >= 5 + ? $".NET {version} ({GetDetailedVersion()})" + : $".NET Core {version?.ToString() ?? Unknown} ({GetDetailedVersion()})"; - return $".NET Core {runtimeVersion} (CoreCLR {coreclrAssemblyInfo.FileVersion}, CoreFX {corefxAssemblyInfo.FileVersion})"; - } + string GetDetailedVersion() + { + string coreclrLocation = typeof(object).GetTypeInfo().Assembly.Location; + // Single-file publish has empty assembly location. + if (string.IsNullOrEmpty(coreclrLocation)) + return CoreRuntime.GetVersionFromFrameworkDescription(); + // .Net Core 2.X has confusing FrameworkDescription like 4.6.X. + if (version?.Major >= 3) + return $"{CoreRuntime.GetVersionFromFrameworkDescription()}, {FileVersionInfo.GetVersionInfo(coreclrLocation).FileVersion}"; + return FileVersionInfo.GetVersionInfo(coreclrLocation).FileVersion; } } @@ -364,12 +206,13 @@ internal static Runtime GetCurrentRuntime() public static Platform GetCurrentPlatform() { // these are not part of .NET Standard 2.0, so we use hardcoded values taken from - // https://github.com/dotnet/runtime/blob/1a37caf773a3b857ccff49a31be3333d4fdc491f/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Architecture.cs#L9 + // https://github.com/dotnet/runtime/blob/080fcae7eaa8367abf7900e08ff2e52e3efea5bf/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Architecture.cs#L9 const Architecture Wasm = (Architecture)4; const Architecture S390x = (Architecture)5; const Architecture LoongArch64 = (Architecture)6; const Architecture Armv6 = (Architecture)7; const Architecture Ppc64le = (Architecture)8; + const Architecture RiscV64 = (Architecture)9; switch (ProcessArchitecture) { @@ -391,6 +234,8 @@ public static Platform GetCurrentPlatform() return Platform.Armv6; case Ppc64le: return Platform.Ppc64le; + case RiscV64: + return Platform.RiscV64; default: throw new ArgumentOutOfRangeException(); } @@ -416,14 +261,14 @@ internal static string GetJitInfo() { if (IsNativeAOT) return "NativeAOT"; - if (IsNetNative || IsAot) + if (IsAot) return "AOT"; if (IsMono || IsWasm) return ""; // There is no helpful information about JIT on Mono if (IsNetCore || HasRyuJit()) // CoreCLR supports only RyuJIT return "RyuJIT"; if (IsFullFramework) - return "LegacyJIT"; + return "LegacyJIT"; return Unknown; } @@ -476,7 +321,7 @@ public JitModule(string name, string version) internal static ICollection GetAntivirusProducts() { var products = new List(); - if (IsWindows()) + if (OsDetector.IsWindows()) { try { @@ -484,7 +329,7 @@ internal static ICollection GetAntivirusProducts() using (var data = wmi.Get()) foreach (var o in data) { - var av = (ManagementObject) o; + var av = (ManagementObject)o; if (av != null) { string name = av["displayName"].ToString(); @@ -502,11 +347,11 @@ internal static ICollection GetAntivirusProducts() return products; } - internal static VirtualMachineHypervisor GetVirtualMachineHypervisor() + internal static VirtualMachineHypervisor? GetVirtualMachineHypervisor() { - VirtualMachineHypervisor[] hypervisors = { HyperV.Default, VirtualBox.Default, VMware.Default }; + VirtualMachineHypervisor[] hypervisors = [HyperV.Default, VirtualBox.Default, VMware.Default]; - if (IsWindows()) + if (OsDetector.IsWindows()) { try { @@ -532,4 +377,4 @@ internal static VirtualMachineHypervisor GetVirtualMachineHypervisor() return null; } } -} +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs index ee7077a55e..bec0dbbc48 100644 --- a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs +++ b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs @@ -1,15 +1,12 @@ using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -#if RELEASE using BenchmarkDotNet.Properties; -#endif [assembly: Guid("cbba82d3-e650-407f-a0f0-767891d4f04c")] [assembly: CLSCompliant(true)] -#if RELEASE [assembly: InternalsVisibleTo("BenchmarkDotNet.Tests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] @@ -18,13 +15,3 @@ [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.TestAdapter,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] -#else -[assembly: InternalsVisibleTo("BenchmarkDotNet.Tests")] -[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests")] -[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows")] -[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace")] -[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotMemory")] -[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning")] -[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks")] -[assembly: InternalsVisibleTo("BenchmarkDotNet.TestAdapter")] -#endif \ No newline at end of file diff --git a/src/BenchmarkDotNet/Properties/BenchmarkDotNetInfo.cs b/src/BenchmarkDotNet/Properties/BenchmarkDotNetInfo.cs index fcb01a0055..0941fa1cc3 100644 --- a/src/BenchmarkDotNet/Properties/BenchmarkDotNetInfo.cs +++ b/src/BenchmarkDotNet/Properties/BenchmarkDotNetInfo.cs @@ -1,12 +1,15 @@ using System; using System.Reflection; using BenchmarkDotNet.Extensions; +using Perfolizer.Models; namespace BenchmarkDotNet.Properties { public class BenchmarkDotNetInfo { - private static readonly Lazy LazyInstance = new (() => + public const string BenchmarkDotNetCaption = "BenchmarkDotNet"; + + private static readonly Lazy LazyInstance = new(() => { var assembly = typeof(BenchmarkDotNetInfo).GetTypeInfo().Assembly; var assemblyVersion = assembly.GetName().Version; @@ -16,6 +19,12 @@ public class BenchmarkDotNetInfo public static BenchmarkDotNetInfo Instance { get; } = LazyInstance.Value; + public EngineInfo GetBdnEngineInfo() => new() + { + Name = BenchmarkDotNetCaption, + Version = BrandVersion + }; + public Version AssemblyVersion { get; } public string FullVersion { get; } diff --git a/src/BenchmarkDotNet/Reports/BenchmarkReport.cs b/src/BenchmarkDotNet/Reports/BenchmarkReport.cs index 744793c063..05f2a3b13e 100644 --- a/src/BenchmarkDotNet/Reports/BenchmarkReport.cs +++ b/src/BenchmarkDotNet/Reports/BenchmarkReport.cs @@ -3,10 +3,14 @@ using System.Collections.Immutable; using System.Linq; using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Models; using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.Results; using JetBrains.Annotations; +using Perfolizer.Horology; +using Perfolizer.Models; namespace BenchmarkDotNet.Reports { @@ -51,5 +55,54 @@ public BenchmarkReport( public override string ToString() => $"{BenchmarkCase.DisplayInfo}, {AllMeasurements.Count} runs"; public IReadOnlyList GetResultRuns() => AllMeasurements.Where(r => r.Is(IterationMode.Workload, IterationStage.Result)).ToList(); + + internal EntryInfo ToPerfonar() + { + var entry = new EntryInfo + { + Benchmark = new BdnBenchmark + { + Display = BenchmarkCase.DisplayInfo, + Namespace = BenchmarkCase.Descriptor.Type.Namespace ?? "", + Type = FullNameProvider.GetTypeName(BenchmarkCase.Descriptor.Type), + Method = BenchmarkCase.Descriptor.WorkloadMethod.Name, + Parameters = BenchmarkCase.Parameters.PrintInfo, + HardwareIntrinsics = this.GetHardwareIntrinsicsInfo() + }, + Job = new JobInfo + { + Environment = BenchmarkCase.Job.Environment.ToPerfonar(), + Execution = BenchmarkCase.Job.Run.ToPerfonar() + } + }; + + var lifecycles = AllMeasurements.GroupBy(m => new BdnLifecycle + { + LaunchIndex = m.LaunchIndex, + IterationMode = m.IterationMode, + IterationStage = m.IterationStage + }).OrderBy(x => x.Key).ToList(); + foreach (var lifecycleGroup in lifecycles) + { + var measurementsEntry = new EntryInfo + { + Lifecycle = lifecycleGroup.Key + }; + + foreach (var measurement in lifecycleGroup.ToList()) + { + measurementsEntry.Add(new EntryInfo + { + IterationIndex = measurement.IterationIndex, + InvocationCount = measurement.Operations, + Value = measurement.Nanoseconds / measurement.Operations, + Unit = TimeUnit.Nanosecond + }); + } + entry.Add(measurementsEntry); + } + + return entry; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index 3822ed8acb..343a0301e9 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -12,6 +12,10 @@ using BenchmarkDotNet.Validators; using JetBrains.Annotations; using Perfolizer.Horology; +using Perfolizer.Models; +using Perfolizer.Perfonar; +using Perfolizer.Perfonar.Base; +using Perfolizer.Perfonar.Tables; namespace BenchmarkDotNet.Reports { @@ -62,7 +66,8 @@ public Summary( DisplayPrecisionManager = new DisplayPrecisionManager(this); Orderer = GetConfiguredOrdererOrDefaultOne(reports.Select(report => report.BenchmarkCase.Config)); - BenchmarksCases = Orderer.GetSummaryOrder(reports.Select(report => report.BenchmarkCase).ToImmutableArray(), this).ToImmutableArray(); // we sort it first + BenchmarksCases = + Orderer.GetSummaryOrder(reports.Select(report => report.BenchmarkCase).ToImmutableArray(), this).ToImmutableArray(); // we sort it first Reports = BenchmarksCases.Select(b => ReportMap[b]).ToImmutableArray(); // we use sorted collection to re-create reports list BaseliningStrategy = BaseliningStrategy.Create(BenchmarksCases); Style = (summaryStyle ?? GetConfiguredSummaryStyleOrDefaultOne(BenchmarksCases)).WithCultureInfo(cultureInfo); @@ -84,8 +89,10 @@ public Summary( public bool IsMultipleRuntimes => isMultipleRuntimes ??= BenchmarksCases.Length > 1 ? BenchmarksCases.Select(benchmark => benchmark.GetRuntime()).Distinct().Count() > 1 : false; - internal static Summary ValidationFailed(string title, string resultsDirectoryPath, string logFilePath, ImmutableArray? validationErrors = null) - => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, validationErrors ?? ImmutableArray.Empty, ImmutableArray.Empty); + internal static Summary ValidationFailed(string title, string resultsDirectoryPath, string logFilePath, + ImmutableArray? validationErrors = null) + => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, + DefaultCultureInfo.Instance, validationErrors ?? ImmutableArray.Empty, ImmutableArray.Empty); internal static Summary Join(List summaries, ClockSpan clockSpan) => new Summary( @@ -152,7 +159,7 @@ private static IOrderer GetConfiguredOrdererOrDefaultOne(IEnumerable config.Orderer != DefaultOrderer.Instance) .Select(config => config.Orderer) .Distinct() - .SingleOrDefault() + .FirstOrDefault() ?? DefaultOrderer.Instance; private static SummaryStyle GetConfiguredSummaryStyleOrDefaultOne(ImmutableArray benchmarkCases) @@ -165,7 +172,41 @@ private static SummaryStyle GetConfiguredSummaryStyleOrDefaultOne(ImmutableArray #nullable enable .Select(benchmark => benchmark.Config.SummaryStyle) .Distinct() - .SingleOrDefault() + .FirstOrDefault() ?? SummaryStyle.Default; + + internal PerfonarTableConfig GetDefaultTableConfig() + { + return new PerfonarTableConfig + { + ColumnDefinitions = + [ + new PerfonarColumnDefinition(".engine") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PerfonarColumnDefinition(".host.os") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PerfonarColumnDefinition(".host.cpu") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PerfonarColumnDefinition(".benchmark") { Cloud = "secondary" }, + new PerfonarColumnDefinition(".job") { Cloud = "secondary", Compressed = true }, + new PerfonarColumnDefinition("=center"), + new PerfonarColumnDefinition("=spread") + ] + }; + } + + // TODO: GcStats + internal EntryInfo ToPerfonar() + { + var root = new EntryInfo + { + Engine = new EngineInfo + { + Name = HostEnvironmentInfo.BenchmarkDotNetCaption, + Version = HostEnvironmentInfo.BenchmarkDotNetVersion, + }, + Host = HostEnvironmentInfo.ToPerfonar(), + }; + foreach (var benchmarkReport in Reports) + root.Add(benchmarkReport.ToPerfonar()); + return root; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs index dbe350b2c8..6ff57f8f87 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs @@ -198,7 +198,9 @@ IEnumerable GetDefinitions(Func((attribute, parameterType) => { - var paramsValues = GetValidValuesForParamsSource(type, attribute.Name); + var targetType = attribute.Type ?? type; + + var paramsValues = GetValidValuesForParamsSource(targetType, attribute.Name); return SmartParamBuilder.CreateForParams(parameterType, paramsValues.source, paramsValues.values); }); @@ -208,7 +210,7 @@ IEnumerable GetDefinitions(Func GetArgumentsDefinitions(MethodInfo benchmark, Type target, SummaryStyle summaryStyle) + private static IEnumerable GetArgumentsDefinitions(MethodInfo benchmark, Type benchmarkType, SummaryStyle summaryStyle) { var argumentsAttributes = benchmark.GetCustomAttributes(); int priority = argumentsAttributes.Select(attribute => attribute.Priority).Sum(); @@ -244,8 +246,9 @@ private static IEnumerable GetArgumentsDefinitions(MethodInf yield break; var argumentsSourceAttribute = benchmark.GetCustomAttribute(); + var targetType = argumentsSourceAttribute.Type ?? benchmarkType; - var valuesInfo = GetValidValuesForParamsSource(target, argumentsSourceAttribute.Name); + var valuesInfo = GetValidValuesForParamsSource(targetType, argumentsSourceAttribute.Name); for (int sourceIndex = 0; sourceIndex < valuesInfo.values.Length; sourceIndex++) yield return SmartParamBuilder.CreateForArguments(benchmark, parameterDefinitions, valuesInfo, sourceIndex, summaryStyle); } @@ -263,16 +266,7 @@ private static void AssertMethodIsAccessible(string methodType, MethodInfo metho { if (!methodInfo.IsPublic) throw new InvalidBenchmarkDeclarationException($"{methodType} method {methodInfo.Name} has incorrect access modifiers.\nMethod must be public."); - - var declaringType = methodInfo.DeclaringType; - - while (declaringType != null) - { - if (!declaringType.GetTypeInfo().IsPublic && !declaringType.GetTypeInfo().IsNestedPublic) - throw new InvalidBenchmarkDeclarationException($"{declaringType.FullName} containing {methodType} method {methodInfo.Name} has incorrect access modifiers.\nDeclaring type must be public."); - - declaringType = declaringType.DeclaringType; - } + /* Moved the code that verifies if DeclaringType of a given MethodInfo (a method) is publicly accessible to CompilationValidator */ } private static void AssertMethodIsNotGeneric(string methodType, MethodInfo methodInfo) @@ -312,25 +306,25 @@ private static object Map(object providedValue, Type type) return providedValue; } - private static (MemberInfo source, object[] values) GetValidValuesForParamsSource(Type parentType, string sourceName) + private static (MemberInfo source, object[] values) GetValidValuesForParamsSource(Type sourceType, string sourceName) { - var paramsSourceMethod = parentType.GetAllMethods().SingleOrDefault(method => method.Name == sourceName && method.IsPublic); + var paramsSourceMethod = sourceType.GetAllMethods().SingleOrDefault(method => method.Name == sourceName && method.IsPublic); if (paramsSourceMethod != default) return (paramsSourceMethod, ToArray( - paramsSourceMethod.Invoke(paramsSourceMethod.IsStatic ? null : Activator.CreateInstance(parentType), null), + paramsSourceMethod.Invoke(paramsSourceMethod.IsStatic ? null : Activator.CreateInstance(sourceType), null), paramsSourceMethod, - parentType)); + sourceType)); - var paramsSourceProperty = parentType.GetAllProperties().SingleOrDefault(property => property.Name == sourceName && property.GetMethod.IsPublic); + var paramsSourceProperty = sourceType.GetAllProperties().SingleOrDefault(property => property.Name == sourceName && property.GetMethod.IsPublic); if (paramsSourceProperty != default) return (paramsSourceProperty, ToArray( - paramsSourceProperty.GetValue(paramsSourceProperty.GetMethod.IsStatic ? null : Activator.CreateInstance(parentType)), + paramsSourceProperty.GetValue(paramsSourceProperty.GetMethod.IsStatic ? null : Activator.CreateInstance(sourceType)), paramsSourceProperty, - parentType)); + sourceType)); - throw new InvalidBenchmarkDeclarationException($"{parentType.Name} has no public, accessible method/property called {sourceName}, unable to read values for [ParamsSource]"); + throw new InvalidBenchmarkDeclarationException($"{sourceType.Name} has no public, accessible method/property called {sourceName}, unable to read values for [ParamsSource]"); } private static object[] ToArray(object sourceValue, MemberInfo memberInfo, Type type) diff --git a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs index ada1b2b78e..4892de7f0c 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs @@ -47,8 +47,10 @@ public bool Equals(BenchmarkCase x, BenchmarkCase y) return false; if (AreDifferent(jobX.Infrastructure.Arguments, jobY.Infrastructure.Arguments)) // arguments can be anything (Mono runtime settings or MsBuild parameters) return false; +#pragma warning disable CS0618 // Type or member is obsolete if (AreDifferent(jobX.Infrastructure.NuGetReferences, jobY.Infrastructure.NuGetReferences)) return false; +#pragma warning restore CS0618 // Type or member is obsolete if (!jobX.Environment.Gc.Equals(jobY.Environment.Gc)) // GC settings are per .config/.csproj return false; @@ -81,8 +83,10 @@ public int GetHashCode(BenchmarkCase obj) hashCode.Add(job.Infrastructure.BuildConfiguration); foreach (var arg in job.Infrastructure.Arguments ?? Array.Empty()) hashCode.Add(arg); +#pragma warning disable CS0618 // Type or member is obsolete foreach (var reference in job.Infrastructure.NuGetReferences ?? Array.Empty()) hashCode.Add(reference); +#pragma warning restore CS0618 // Type or member is obsolete return hashCode.ToHashCode(); } diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 4a1e85eee9..fefd88a906 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -8,6 +8,7 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; @@ -18,9 +19,9 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Mathematics; -using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.Parameters; using BenchmarkDotNet.Toolchains.Results; using BenchmarkDotNet.Validators; @@ -43,7 +44,7 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) var artifactsToCleanup = new List(); var rootArtifactsFolderPath = GetRootArtifactsFolderPath(benchmarkRunInfos); - var maxTitleLength = RuntimeInformation.IsWindows() + var maxTitleLength = OsDetector.IsWindows() ? 254 - rootArtifactsFolderPath.Length : int.MaxValue; var title = GetTitle(benchmarkRunInfos, maxTitleLength); @@ -54,6 +55,7 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) using (var streamLogger = new StreamLogger(GetLogFileStreamWriter(benchmarkRunInfos, logFilePath))) { var compositeLogger = CreateCompositeLogger(benchmarkRunInfos, streamLogger); + using var wakeLock = WakeLock.Request(WakeLock.GetWakeLockType(benchmarkRunInfos), "BenchmarkDotNet Running Benchmarks", streamLogger); var eventProcessor = new CompositeEventProcessor(benchmarkRunInfos); eventProcessor.OnStartValidationStage(); @@ -69,24 +71,48 @@ internal static Summary[] Run(BenchmarkRunInfo[] benchmarkRunInfos) PrintValidationErrors(compositeLogger, validationErrors); + eventProcessor.OnEndValidationStage(); // Ensure that OnEndValidationStage() is called when a critical validation error exists. + if (validationErrors.Any(validationError => validationError.IsCritical)) return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath, validationErrors.ToImmutableArray()) }; - if (!supportedBenchmarks.Any(benchmarks => benchmarks.BenchmarksCases.Any())) - return new[] { Summary.ValidationFailed(title, resultsFolderPath, logFilePath) }; - - eventProcessor.OnEndValidationStage(); - int totalBenchmarkCount = supportedBenchmarks.Sum(benchmarkInfo => benchmarkInfo.BenchmarksCases.Length); int benchmarksToRunCount = totalBenchmarkCount - (idToResume + 1); // ids are indexed from 0 compositeLogger.WriteLineHeader("// ***** BenchmarkRunner: Start *****"); compositeLogger.WriteLineHeader($"// ***** Found {totalBenchmarkCount} benchmark(s) in total *****"); var globalChronometer = Chronometer.Start(); - var buildPartitions = BenchmarkPartitioner.CreateForBuild(supportedBenchmarks, resolver); eventProcessor.OnStartBuildStage(buildPartitions); - var buildResults = BuildInParallel(compositeLogger, rootArtifactsFolderPath, buildPartitions, in globalChronometer, eventProcessor); + + var sequentialBuildPartitions = buildPartitions.Where(partition => + partition.Benchmarks.Any(x => x.Config.Options.IsSet(ConfigOptions.DisableParallelBuild)) + // .Net SDK 8+ supports ArtifactsPath for proper parallel builds. + // Older SDKs may produce builds with incorrect bindings if more than 1 partition is built in parallel. + || (partition.RepresentativeBenchmarkCase.GetToolchain().Generator is DotNetCliGenerator + && partition.RepresentativeBenchmarkCase.GetRuntime().RuntimeMoniker.GetRuntimeVersion().Major < 8) + ) + .ToArray(); + var parallelBuildPartitions = buildPartitions.Except(sequentialBuildPartitions).ToArray(); + + var buildResults = new Dictionary(); + if (parallelBuildPartitions.Length > 0) + { + var results = BuildInParallel(compositeLogger, rootArtifactsFolderPath, parallelBuildPartitions, in globalChronometer, eventProcessor); + foreach (var kvp in results) + { + buildResults.Add(kvp.Key, kvp.Value); + } + } + + if (sequentialBuildPartitions.Length > 0) + { + var results = BuildSequential(compositeLogger, rootArtifactsFolderPath, sequentialBuildPartitions, in globalChronometer, eventProcessor); + foreach (var kvp in results) + { + buildResults.Add(kvp.Key, kvp.Value); + } + } var allBuildsHaveFailed = buildResults.Values.All(buildResult => !buildResult.IsBuildSuccess); @@ -352,17 +378,6 @@ private static ImmutableArray Validate(params BenchmarkRunInfo[ { var validationErrors = new List(); - if (benchmarks.Any(b => b.Config.Options.IsSet(ConfigOptions.JoinSummary))) - { - var joinedCases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - validationErrors.AddRange( - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(joinedCases, null)) - ); - } - foreach (var benchmarkRunInfo in benchmarks) validationErrors.AddRange(benchmarkRunInfo.Config.GetCompositeValidator().Validate(new ValidationParameters(benchmarkRunInfo.BenchmarksCases, benchmarkRunInfo.Config))); @@ -417,11 +432,31 @@ private static Dictionary BuildInParallel(ILogger l logger.WriteLineHeader($"// ***** Done, took {GetFormattedDifference(afterParallelBuild, afterSequentialBuild)} *****"); return buildResults; + } - static string GetFormattedDifference(ClockSpan before, ClockSpan after) - => (after.GetTimeSpan() - before.GetTimeSpan()).ToFormattedTotalTime(DefaultCultureInfo.Instance); + private static Dictionary BuildSequential(ILogger logger, string rootArtifactsFolderPath, BuildPartition[] buildPartitions, in StartedClock globalChronometer, EventProcessor eventProcessor) + { + logger.WriteLineHeader($"// ***** Building {buildPartitions.Length} exe(s) in Sequential: Start *****"); + + var beforeBuild = globalChronometer.GetElapsed(); + + var buildResults = new Dictionary(); + foreach (var buildPartition in buildPartitions) + { + buildResults[buildPartition] = Build(buildPartition, rootArtifactsFolderPath, logger); + eventProcessor.OnBuildComplete(buildPartition, buildResults[buildPartition]); + } + + var afterBuild = globalChronometer.GetElapsed(); + + logger.WriteLineHeader($"// ***** Done, took {GetFormattedDifference(beforeBuild, afterBuild)} *****"); + + return buildResults; } + private static string GetFormattedDifference(ClockSpan before, ClockSpan after) + => (after.GetTimeSpan() - before.GetTimeSpan()).ToFormattedTotalTime(DefaultCultureInfo.Instance); + private static BuildResult Build(BuildPartition buildPartition, string rootArtifactsFolderPath, ILogger buildLogger) { var toolchain = buildPartition.RepresentativeBenchmarkCase.GetToolchain(); // it's guaranteed that all the benchmarks in single partition have same toolchain @@ -587,21 +622,48 @@ private static void LogTotalTime(ILogger logger, TimeSpan time, int executedBenc private static (BenchmarkRunInfo[], List) GetSupportedBenchmarks(BenchmarkRunInfo[] benchmarkRunInfos, IResolver resolver) { - List validationErrors = new (); + List validationErrors = new(); + List runInfos = new(benchmarkRunInfos.Length); + + if (benchmarkRunInfos.Length == 0) + { + validationErrors.Add(new ValidationError(true, $"No benchmarks were found.")); + return (Array.Empty(), validationErrors); + } + + foreach (var benchmarkRunInfo in benchmarkRunInfos) + { + if (benchmarkRunInfo.BenchmarksCases.Length == 0) + { + validationErrors.Add(new ValidationError(true, $"No [Benchmark] attribute found on '{benchmarkRunInfo.Type.Name}' benchmark case.")); + continue; + } - var runInfos = benchmarkRunInfos.Select(info => new BenchmarkRunInfo( - info.BenchmarksCases.Where(benchmark => + var validBenchmarks = benchmarkRunInfo.BenchmarksCases + .Where(benchmark => { - var errors = benchmark.GetToolchain().Validate(benchmark, resolver).ToArray(); + + var errors = benchmark.GetToolchain() + .Validate(benchmark, resolver) + .ToArray(); + validationErrors.AddRange(errors); - return !errors.Any(); - }).ToArray(), - info.Type, - info.Config)) - .Where(infos => infos.BenchmarksCases.Any()) - .ToArray(); - - return (runInfos, validationErrors); + + return !errors.Any(error => error.IsCritical); + }) + .ToArray(); + + runInfos.Add( + new BenchmarkRunInfo( + validBenchmarks, + benchmarkRunInfo.Type, + benchmarkRunInfo.Config + + )); + + + } + return (runInfos.ToArray(), validationErrors); } private static string GetRootArtifactsFolderPath(BenchmarkRunInfo[] benchmarkRunInfos) @@ -674,6 +736,9 @@ void AddLogger(ILogger logger) foreach (var logger in benchmarkRunInfo.Config.GetLoggers()) AddLogger(logger); + if (benchmarkRunInfos.Length == 0) + AddLogger(new ConsoleLogger()); + AddLogger(streamLogger); return new CompositeLogger(loggers.Values.ToImmutableHashSet()); diff --git a/src/BenchmarkDotNet/Running/BuildPartition.cs b/src/BenchmarkDotNet/Running/BuildPartition.cs index 4ffc48d6c8..2ebe5ee44c 100644 --- a/src/BenchmarkDotNet/Running/BuildPartition.cs +++ b/src/BenchmarkDotNet/Running/BuildPartition.cs @@ -1,8 +1,10 @@ using System; using System.IO; using System.Reflection; +using System.Threading; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; @@ -18,12 +20,16 @@ namespace BenchmarkDotNet.Running { public class BuildPartition { + // We use an auto-increment global counter instead of Guid to guarantee uniqueness per benchmark run (Guid has a small chance to collide), + // assuming there are fewer than 4 billion build partitions (a safe assumption). + internal static int s_partitionCounter; + public BuildPartition(BenchmarkBuildInfo[] benchmarks, IResolver resolver) { Resolver = resolver; RepresentativeBenchmarkCase = benchmarks[0].BenchmarkCase; Benchmarks = benchmarks; - ProgramName = benchmarks[0].Config.Options.IsSet(ConfigOptions.KeepBenchmarkFiles) ? RepresentativeBenchmarkCase.Job.FolderInfo : Guid.NewGuid().ToString(); + ProgramName = GetProgramName(RepresentativeBenchmarkCase, Interlocked.Increment(ref s_partitionCounter)); LogBuildOutput = benchmarks[0].Config.Options.IsSet(ConfigOptions.LogBuildOutput); GenerateMSBuildBinLog = benchmarks[0].Config.Options.IsSet(ConfigOptions.GenerateMSBuildBinLog); } @@ -76,6 +82,33 @@ private static string GetResolvedAssemblyLocation(Assembly assembly) => // manually construct the path. assembly.Location.Length == 0 ? Path.Combine(AppContext.BaseDirectory, assembly.GetName().Name) : assembly.Location; + internal static string GetProgramName(BenchmarkCase representativeBenchmarkCase, int id) + { + // Combine the benchmark's assembly name, folder info, and build partition id. + string benchmarkAssemblyName = representativeBenchmarkCase.Descriptor.Type.Assembly.GetName().Name; + string folderInfo = representativeBenchmarkCase.Job.FolderInfo; + var programName = $"{benchmarkAssemblyName}-{folderInfo}-{id}"; + // Very long program name can cause the path to exceed Windows' 260 character limit, + // for example BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks. + // 36 is an arbitrary limit, but it's the length of Guid strings which is what was used previously. + const int MaxLength = 36; + if (!OsDetector.IsWindows() || programName.Length <= MaxLength) + { + return programName; + } + programName = $"{benchmarkAssemblyName}-{id}"; + if (programName.Length <= MaxLength) + { + return programName; + } + programName = $"{folderInfo}-{id}"; + if (programName.Length <= MaxLength) + { + return programName; + } + return id.ToString(); + } + internal bool ForcedNoDependenciesForIntegrationTests { get @@ -91,4 +124,4 @@ internal bool ForcedNoDependenciesForIntegrationTests } } } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Running/ConsoleTitler.cs b/src/BenchmarkDotNet/Running/ConsoleTitler.cs index 9f8e21a1fe..d045e498aa 100644 --- a/src/BenchmarkDotNet/Running/ConsoleTitler.cs +++ b/src/BenchmarkDotNet/Running/ConsoleTitler.cs @@ -1,6 +1,7 @@ using System; using System.IO; -using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Helpers; namespace BenchmarkDotNet.Running { @@ -8,7 +9,7 @@ namespace BenchmarkDotNet.Running /// Updates Console.Title, subject to platform capabilities and Console availability. /// Restores the original (or fallback) title upon disposal. /// - internal class ConsoleTitler : IDisposable + internal class ConsoleTitler : DisposeAtProcessTermination { /// /// Whether this instance has any effect. This will be false if the platform doesn't support Console retitling, @@ -62,7 +63,7 @@ public ConsoleTitler(string initialTitle) #if NET6_0_OR_GREATER [System.Runtime.Versioning.SupportedOSPlatformGuard("windows")] #endif - private static bool PlatformSupportsTitleRead() => RuntimeInformation.IsWindows(); + private static bool PlatformSupportsTitleRead() => OsDetector.IsWindows(); /// /// Updates Console.Title if enabled. @@ -75,13 +76,14 @@ public void UpdateTitle(string title) } } - public void Dispose() + public override void Dispose() { if (IsEnabled) { Console.Title = oldConsoleTitle; IsEnabled = false; } + base.Dispose(); } } } diff --git a/src/BenchmarkDotNet/Running/PowerManagementApplier.cs b/src/BenchmarkDotNet/Running/PowerManagementApplier.cs index fdfa0239f5..f48d95d61e 100644 --- a/src/BenchmarkDotNet/Running/PowerManagementApplier.cs +++ b/src/BenchmarkDotNet/Running/PowerManagementApplier.cs @@ -1,13 +1,13 @@ using System; using System.Collections.Generic; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Running { - internal class PowerManagementApplier : IDisposable + internal class PowerManagementApplier : DisposeAtProcessTermination { private static readonly Guid UserPowerPlan = new Guid("67b4a053-3646-4532-affd-0535c9ea82a7"); @@ -27,13 +27,17 @@ internal class PowerManagementApplier : IDisposable internal PowerManagementApplier(ILogger logger) => this.logger = logger; - public void Dispose() => ApplyUserPowerPlan(); + public override void Dispose() + { + ApplyUserPowerPlan(); + base.Dispose(); + } internal static Guid Map(PowerPlan value) => PowerPlansDict[value]; internal void ApplyPerformancePlan(Guid id) { - if (!RuntimeInformation.IsWindows() || id == Guid.Empty) + if (!OsDetector.IsWindows() || id == Guid.Empty) return; if (id != UserPowerPlan) @@ -44,7 +48,7 @@ internal void ApplyPerformancePlan(Guid id) private void ApplyUserPowerPlan() { - if (powerPlanChanged && RuntimeInformation.IsWindows()) + if (powerPlanChanged && OsDetector.IsWindows()) { try { diff --git a/src/BenchmarkDotNet/Running/TypeFilter.cs b/src/BenchmarkDotNet/Running/TypeFilter.cs index 4e58ebdeaf..4d2188844f 100644 --- a/src/BenchmarkDotNet/Running/TypeFilter.cs +++ b/src/BenchmarkDotNet/Running/TypeFilter.cs @@ -15,6 +15,32 @@ public static (bool allTypesValid, IReadOnlyList runnable) GetTypesWithRun { var validRunnableTypes = new List(); + bool hasRunnableTypeBenchmarks = types.Any(type => type.ContainsRunnableBenchmarks()); + bool hasRunnableAssemblyBenchmarks = assemblies.Any(assembly => GenericBenchmarksBuilder.GetRunnableBenchmarks(assembly.GetRunnableBenchmarks()).Length > 0); + + if (!hasRunnableTypeBenchmarks && !hasRunnableAssemblyBenchmarks) + { + if (types.Any()) + { + foreach (var type in types) + { + logger.WriteLineError($"No [Benchmark] attribute found on '{type.Name}' benchmark case."); + } + } + else if (assemblies.Any()) + { + foreach (var assembly in assemblies) + { + logger.WriteLineError($"No [Benchmark] attribute found on '{assembly.GetName().Name}' assembly."); + } + } + else + { + logger.WriteLineError("No benchmarks were found."); + } + return (false, Array.Empty()); + } + foreach (var type in types) { if (type.ContainsRunnableBenchmarks()) diff --git a/src/BenchmarkDotNet/Running/WakeLock.PInvoke.cs b/src/BenchmarkDotNet/Running/WakeLock.PInvoke.cs new file mode 100644 index 0000000000..be002d5d20 --- /dev/null +++ b/src/BenchmarkDotNet/Running/WakeLock.PInvoke.cs @@ -0,0 +1,110 @@ +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Running; + +internal partial class WakeLock +{ + private static class PInvoke + { + public static SafePowerHandle PowerCreateRequest(string reason) + { + IntPtr reasonPtr = Marshal.StringToHGlobalAuto(reason); + try + { + REASON_CONTEXT context = new REASON_CONTEXT() + { + Version = POWER_REQUEST_CONTEXT_VERSION, + Flags = POWER_REQUEST_CONTEXT_FLAGS.POWER_REQUEST_CONTEXT_SIMPLE_STRING, + Reason = new REASON_CONTEXT.REASON_CONTEXT_UNION { SimpleReasonString = reasonPtr } + + }; + SafePowerHandle safePowerHandle = PowerCreateRequest(context); + if (safePowerHandle.IsInvalid) { throw new Win32Exception(); } + return safePowerHandle; + } + finally + { + if (reasonPtr != IntPtr.Zero) + { + Marshal.FreeHGlobal(reasonPtr); + } + } + } + + [DllImport("kernel32.dll", ExactSpelling = true, SetLastError = true)] + private static extern SafePowerHandle PowerCreateRequest(REASON_CONTEXT Context); + + public static void PowerSetRequest(SafePowerHandle safePowerHandle, POWER_REQUEST_TYPE requestType) + { + if (!InvokePowerSetRequest(safePowerHandle, requestType)) + { + throw new Win32Exception(); + } + } + + [DllImport("kernel32.dll", EntryPoint = "PowerSetRequest", ExactSpelling = true, SetLastError = true)] + private static extern bool InvokePowerSetRequest(SafePowerHandle PowerRequest, POWER_REQUEST_TYPE RequestType); + + public static void PowerClearRequest(SafePowerHandle safePowerHandle, POWER_REQUEST_TYPE requestType) + { + if (!InvokePowerClearRequest(safePowerHandle, requestType)) + { + throw new Win32Exception(); + } + } + + [DllImport("kernel32.dll", EntryPoint = "PowerClearRequest", ExactSpelling = true, SetLastError = true)] + private static extern bool InvokePowerClearRequest(SafePowerHandle PowerRequest, POWER_REQUEST_TYPE RequestType); + + [DllImport("kernel32.dll", ExactSpelling = true, SetLastError = true)] + public static extern bool CloseHandle(nint hObject); + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] + private struct REASON_CONTEXT + { + public uint Version; + + public POWER_REQUEST_CONTEXT_FLAGS Flags; + + public REASON_CONTEXT_UNION Reason; + + [StructLayout(LayoutKind.Explicit, CharSet = CharSet.Unicode)] + public struct REASON_CONTEXT_UNION + { + [FieldOffset(0)] + public nint SimpleReasonString; + + // The DETAILED structure is not (yet) used, but needed for ARM CPUs, otherwise PowerCreateRequest fails, see #2745 + [FieldOffset(0)] + public DETAILED Detailed; + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] + public struct DETAILED + { + public nint LocalizedReasonModule; + public uint LocalizedReasonId; + public uint ReasonStringCount; + public nint ReasonStrings; + } + } + } + + private const uint POWER_REQUEST_CONTEXT_VERSION = 0U; + + private enum POWER_REQUEST_CONTEXT_FLAGS : uint + { + POWER_REQUEST_CONTEXT_DETAILED_STRING = 2U, + POWER_REQUEST_CONTEXT_SIMPLE_STRING = 1U, + } + + public enum POWER_REQUEST_TYPE + { + PowerRequestDisplayRequired = 0, + PowerRequestSystemRequired = 1, + PowerRequestAwayModeRequired = 2, + PowerRequestExecutionRequired = 3, + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/WakeLock.SafePowerHandle.cs b/src/BenchmarkDotNet/Running/WakeLock.SafePowerHandle.cs new file mode 100644 index 0000000000..8e6203574d --- /dev/null +++ b/src/BenchmarkDotNet/Running/WakeLock.SafePowerHandle.cs @@ -0,0 +1,13 @@ +using Microsoft.Win32.SafeHandles; + +namespace BenchmarkDotNet.Running; + +internal partial class WakeLock +{ + private sealed class SafePowerHandle : SafeHandleZeroOrMinusOneIsInvalid + { + private SafePowerHandle() : base(true) { } + + protected override bool ReleaseHandle() => PInvoke.CloseHandle(handle); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Running/WakeLock.cs b/src/BenchmarkDotNet/Running/WakeLock.cs new file mode 100644 index 0000000000..124fbd5e22 --- /dev/null +++ b/src/BenchmarkDotNet/Running/WakeLock.cs @@ -0,0 +1,69 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Loggers; +using System; +using System.ComponentModel; +using System.Linq; + +namespace BenchmarkDotNet.Running; + +internal partial class WakeLock +{ + public static WakeLockType GetWakeLockType(BenchmarkRunInfo[] benchmarkRunInfos) => + benchmarkRunInfos.Length == 0 ? WakeLockType.None : benchmarkRunInfos.Select(static i => i.Config.WakeLock).Max(); + + private static readonly bool OsVersionIsSupported = + // Must be windows 7 or greater + OsDetector.IsWindows() && Environment.OSVersion.Version >= new Version(6, 1); + + public static IDisposable? Request(WakeLockType wakeLockType, string reason, ILogger logger) => + wakeLockType == WakeLockType.None || !OsVersionIsSupported ? null : new WakeLockSentinel(wakeLockType, reason, logger); + + private class WakeLockSentinel : DisposeAtProcessTermination + { + private readonly WakeLockType wakeLockType; + private readonly SafePowerHandle? safePowerHandle; + private readonly ILogger logger; + + public WakeLockSentinel(WakeLockType wakeLockType, string reason, ILogger logger) + { + this.wakeLockType = wakeLockType; + this.logger = logger; + try + { + safePowerHandle = PInvoke.PowerCreateRequest(reason); + PInvoke.PowerSetRequest(safePowerHandle, PInvoke.POWER_REQUEST_TYPE.PowerRequestSystemRequired); + if (wakeLockType == WakeLockType.Display) + { + PInvoke.PowerSetRequest(safePowerHandle, PInvoke.POWER_REQUEST_TYPE.PowerRequestDisplayRequired); + } + } + catch (Win32Exception ex) + { + logger.WriteLineError($"Unable to prevent the system from entering sleep or turning off the display (error message: {ex.Message})."); + } + } + + public override void Dispose() + { + if (safePowerHandle != null) + { + try + { + if (wakeLockType == WakeLockType.Display) + { + PInvoke.PowerClearRequest(safePowerHandle, PInvoke.POWER_REQUEST_TYPE.PowerRequestDisplayRequired); + } + PInvoke.PowerClearRequest(safePowerHandle, PInvoke.POWER_REQUEST_TYPE.PowerRequestSystemRequired); + } + catch (Win32Exception ex) + { + logger.WriteLineError($"Unable to allow the system from entering sleep or turning off the display (error message: {ex.Message})."); + } + safePowerHandle.Dispose(); + } + base.Dispose(); + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Templates/BenchmarkType.txt b/src/BenchmarkDotNet/Templates/BenchmarkType.txt index f17737d646..ea1bbaa858 100644 --- a/src/BenchmarkDotNet/Templates/BenchmarkType.txt +++ b/src/BenchmarkDotNet/Templates/BenchmarkType.txt @@ -3,7 +3,7 @@ { public static void Run(BenchmarkDotNet.Engines.IHost host, System.String benchmarkName) { - BenchmarkDotNet.Autogenerated.Runnable_$ID$ instance = new BenchmarkDotNet.Autogenerated.Runnable_$ID$(); // do NOT change name "instance" (used in SmartParamameter) + BenchmarkDotNet.Autogenerated.Runnable_$ID$ instance = new BenchmarkDotNet.Autogenerated.Runnable_$ID$ { $ParamsInitializer$ }; // do NOT change name "instance" (used in SmartParamameter) $ParamsContent$ host.WriteLine(); diff --git a/src/BenchmarkDotNet/Templates/BuildPlots.R b/src/BenchmarkDotNet/Templates/BuildPlots.R index da071d8d41..c5dd6118f6 100644 --- a/src/BenchmarkDotNet/Templates/BuildPlots.R +++ b/src/BenchmarkDotNet/Templates/BuildPlots.R @@ -105,7 +105,7 @@ for (file in files) { densityPlot <- ggplot(df, aes(x=Measurement_Value, fill=Job_Id)) + ggtitle(paste(title, "/", target)) + xlab(paste("Time,", timeUnit)) + - geom_density(alpha=.5, bw="SJ") + geom_density(alpha=.5, bw="sj") printNice(densityPlot) ggsaveNice(gsub("-measurements.csv", paste0("-", target, "-density.png"), file), densityPlot) @@ -118,7 +118,7 @@ for (file in files) { paramsDensityPlot <- ggplot(paramsDf, aes(x=Measurement_Value, fill=Job_Id)) + ggtitle(paste(title, "/", target, "/", params)) + xlab(paste("Time,", timeUnit)) + - geom_density(alpha=.5, bw="SJ") + geom_density(alpha=.5, bw="sj") printNice(paramsDensityPlot) prefix <- createPrefix(c(target,params)) ggsaveNice(gsub("-measurements.csv", paste0(prefix, "-density.png"), file), paramsDensityPlot) @@ -155,7 +155,7 @@ for (file in files) { densityPlotJob <- ggplot(jobDf, aes(x=Measurement_Value, fill="red")) + ggtitle(paste(title, "/", target, "/", job)) + xlab(paste("Time,", timeUnit)) + - geom_density(alpha=.5, bw="SJ") + geom_density(alpha=.5, bw="sj") printNice(densityPlotJob) ggsaveNice(gsub("-measurements.csv", paste0("-", target, "-", job, "-density.png"), file), densityPlotJob) } diff --git a/src/BenchmarkDotNet/Templates/CsProj.txt b/src/BenchmarkDotNet/Templates/CsProj.txt index 5cd08250dc..99e536b985 100644 --- a/src/BenchmarkDotNet/Templates/CsProj.txt +++ b/src/BenchmarkDotNet/Templates/CsProj.txt @@ -9,7 +9,6 @@ $PLATFORM$ $PROGRAMNAME$ Exe - bin\$CONFIGURATIONNAME$ False false @@ -23,9 +22,10 @@ true latest - true false + + true BenchmarkDotNet.Autogenerated.UniqueProgramName diff --git a/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt b/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt index 1db40072d1..adcb87389e 100644 --- a/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt +++ b/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt @@ -9,7 +9,6 @@ Exe - bin $TFM$ $RUNTIMEPACK$ $RUNTIMEIDENTIFIER$ @@ -20,6 +19,8 @@ true BenchmarkDotNet.Autogenerated.UniqueProgramName true + + true diff --git a/src/BenchmarkDotNet/Templates/WasmCsProj.txt b/src/BenchmarkDotNet/Templates/WasmCsProj.txt index e45f5c3789..3db39f5669 100644 --- a/src/BenchmarkDotNet/Templates/WasmCsProj.txt +++ b/src/BenchmarkDotNet/Templates/WasmCsProj.txt @@ -11,12 +11,11 @@ Exe - bin Release false $TFM$ true - $(MSBuildThisFileDirectory)\bin\$TFM$\browser-wasm\publish + $(PublishDir) $PROGRAMNAME$ browser-wasm true @@ -26,6 +25,8 @@ false false false + + true BenchmarkDotNet.Autogenerated.UniqueProgramName @@ -49,8 +50,8 @@ - - + + diff --git a/src/BenchmarkDotNet/Toolchains/AppConfigGenerator.cs b/src/BenchmarkDotNet/Toolchains/AppConfigGenerator.cs index c1db452d2b..fcaf3824b8 100644 --- a/src/BenchmarkDotNet/Toolchains/AppConfigGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/AppConfigGenerator.cs @@ -8,6 +8,8 @@ using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; +#nullable enable + namespace BenchmarkDotNet.Toolchains { internal static class AppConfigGenerator @@ -26,22 +28,32 @@ internal static class AppConfigGenerator internal static void Generate(Job job, TextReader source, TextWriter destination, IResolver resolver) { - using (var xmlReader = XmlReader.Create(source)) - { - var xmlDocument = new XmlDocument(); + var xmlDocument = new XmlDocument(); + + XmlNode configurationElement; - var configurationElement = GetOrCreateConfigurationElement(xmlDocument, xmlReader); + if (source == TextReader.Null) + { + // Create a new configuration node. + configurationElement = xmlDocument.CreateNode(XmlNodeType.Element, "configuration", string.Empty); + xmlDocument.AppendChild(configurationElement); + } + else + { + // Try to get configuration node from specified TextReader. + using var xmlReader = XmlReader.Create(source); + configurationElement = GetOrCreateConfigurationElement(xmlDocument, xmlReader); + } - var runtimeElement = GetOrCreateRuntimeElement(xmlDocument, configurationElement); + var runtimeElement = GetOrCreateRuntimeElement(xmlDocument, configurationElement); - ClearStartupSettingsForCustomClr(configurationElement, job.Environment.Runtime); - ClearAllRuntimeSettingsThatCanBeSetOnlyByJobConfiguration(runtimeElement); + ClearStartupSettingsForCustomClr(configurationElement, job.Environment.Runtime); + ClearAllRuntimeSettingsThatCanBeSetOnlyByJobConfiguration(runtimeElement); - GenerateJitSettings(xmlDocument, runtimeElement, job.Environment); - GenerateGCSettings(xmlDocument, runtimeElement, job.Environment.Gc, resolver); + GenerateJitSettings(xmlDocument, runtimeElement, job.Environment); + GenerateGCSettings(xmlDocument, runtimeElement, job.Environment.Gc, resolver); - xmlDocument.Save(destination); - } + xmlDocument.Save(destination); } private static XmlNode GetOrCreateConfigurationElement(XmlDocument xmlDocument, XmlReader xmlReader) @@ -49,19 +61,23 @@ private static XmlNode GetOrCreateConfigurationElement(XmlDocument xmlDocument, try { xmlDocument.Load(xmlReader); - - return xmlDocument.SelectSingleNode("/configuration"); + var configurationNode = xmlDocument.SelectSingleNode("/configuration"); + if (configurationNode != null) + return configurationNode; } - catch // empty document + catch (XmlException) { - return xmlDocument.AppendChild(xmlDocument.CreateNode(XmlNodeType.Element, "configuration", string.Empty)); + // Failed to load XML content. } + + // If the XML is invalid or configuration node is not exists. Create a new configuration element + return xmlDocument.AppendChild(xmlDocument.CreateNode(XmlNodeType.Element, "configuration", string.Empty))!; } private static XmlNode GetOrCreateRuntimeElement(XmlDocument xmlDocument, XmlNode configurationElement) { return configurationElement.SelectSingleNode("runtime") - ?? configurationElement.AppendChild(xmlDocument.CreateNode(XmlNodeType.Element, "runtime", string.Empty)); + ?? configurationElement.AppendChild(xmlDocument.CreateNode(XmlNodeType.Element, "runtime", string.Empty))!; } private static void ClearAllRuntimeSettingsThatCanBeSetOnlyByJobConfiguration(XmlNode runtimeElement) @@ -75,7 +91,7 @@ private static void ClearAllRuntimeSettingsThatCanBeSetOnlyByJobConfiguration(Xm } } - private static void ClearStartupSettingsForCustomClr(XmlNode configurationElement, Runtime runtime) + private static void ClearStartupSettingsForCustomClr(XmlNode configurationElement, Runtime? runtime) { if (!(runtime is ClrRuntime clrRuntime) || string.IsNullOrEmpty(clrRuntime.Version)) return; diff --git a/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs b/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs index 0c097f5bb0..a9467494fa 100644 --- a/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs +++ b/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs @@ -9,7 +9,7 @@ public class ArtifactsPaths [PublicAPI] public string RootArtifactsFolderPath { get; } [PublicAPI] public string BuildArtifactsDirectoryPath { get; } [PublicAPI] public string BinariesDirectoryPath { get; } - [PublicAPI] public string IntermediateDirectoryPath { get; } + [PublicAPI] public string PublishDirectoryPath { get; } [PublicAPI] public string ProgramCodePath { get; } [PublicAPI] public string AppConfigPath { get; } [PublicAPI] public string NuGetConfigPath { get; } @@ -23,7 +23,7 @@ public ArtifactsPaths( string rootArtifactsFolderPath, string buildArtifactsDirectoryPath, string binariesDirectoryPath, - string intermediateDirectoryPath, + string publishDirectoryPath, string programCodePath, string appConfigPath, string nuGetConfigPath, @@ -36,7 +36,7 @@ public ArtifactsPaths( RootArtifactsFolderPath = rootArtifactsFolderPath; BuildArtifactsDirectoryPath = buildArtifactsDirectoryPath; BinariesDirectoryPath = binariesDirectoryPath; - IntermediateDirectoryPath = intermediateDirectoryPath; + PublishDirectoryPath = publishDirectoryPath; ProgramCodePath = programCodePath; AppConfigPath = appConfigPath; NuGetConfigPath = nuGetConfigPath; diff --git a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs index 8bdacf37a4..3cbaa80f65 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs @@ -15,12 +15,12 @@ public class CoreRunToolchain : IToolchain /// /// the path to CoreRun /// /should a copy of CoreRun be performed? True by default. The toolchain replaces old dependencies in CoreRun folder with newer versions if used by the benchmarks. - /// TFM, netcoreapp2.1 is the default + /// TFM, net8.0 is the default /// path to dotnet cli, if not provided the one from PATH will be used /// display name, CoreRun is the default value /// the directory to restore packages to public CoreRunToolchain(FileInfo coreRun, bool createCopy = true, - string targetFrameworkMoniker = "netcoreapp2.1", + string targetFrameworkMoniker = "net8.0", FileInfo? customDotNetCliPath = null, DirectoryInfo? restorePath = null, string displayName = "CoreRun") { diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs index 187458be17..2761bd7c2c 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; @@ -44,7 +45,7 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas yield return validationError; } - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) { yield return new ValidationError(true, $"Classic .NET toolchain is supported only for Windows, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs index c6cf6248df..4aea1327f8 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs @@ -25,6 +25,7 @@ public class CsProjCoreToolchain : Toolchain, IEquatable [PublicAPI] public static readonly IToolchain NetCoreApp70 = From(NetCoreAppSettings.NetCoreApp70); [PublicAPI] public static readonly IToolchain NetCoreApp80 = From(NetCoreAppSettings.NetCoreApp80); [PublicAPI] public static readonly IToolchain NetCoreApp90 = From(NetCoreAppSettings.NetCoreApp90); + [PublicAPI] public static readonly IToolchain NetCoreApp10_0 = From(NetCoreAppSettings.NetCoreApp10_0); internal CsProjCoreToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) : base(name, generator, builder, executor) diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs index 7e91c36ff4..7b5a642744 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs @@ -64,9 +64,6 @@ protected override string GetProjectFilePath(string buildArtifactsDirectoryPath) protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, TargetFrameworkMoniker); - protected override string GetIntermediateDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => Path.Combine(buildArtifactsDirectoryPath, "obj", configuration, TargetFrameworkMoniker); - [SuppressMessage("ReSharper", "StringLiteralTypo")] // R# complains about $variables$ protected override void GenerateProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) { @@ -85,7 +82,6 @@ protected override void GenerateProject(BuildPartition buildPartition, Artifacts .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) .Replace("$RUNTIMESETTINGS$", GetRuntimeSettings(benchmark.Job.Environment.Gc, buildPartition.Resolver)) .Replace("$COPIEDSETTINGS$", customProperties) - .Replace("$CONFIGURATIONNAME$", buildPartition.BuildConfiguration) .Replace("$SDKNAME$", sdkName) .ToString(); diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs index dc48965f1e..6223f9ea3c 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Portability; using JetBrains.Annotations; @@ -49,7 +50,7 @@ public CustomDotNetCliToolchainBuilder UseNuGetClearTag(bool value) return this; } - /// TFM, example: netcoreapp2.1 + /// TFM, example: net8.0 [PublicAPI] [SuppressMessage("ReSharper", "ParameterHidesMember")] public CustomDotNetCliToolchainBuilder TargetFrameworkMoniker(string targetFrameworkMoniker) @@ -127,7 +128,7 @@ internal static string GetPortableRuntimeIdentifier() // Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment.GetRuntimeIdentifier() // returns win10-x64, we want the simpler form win-x64 // the values taken from https://docs.microsoft.com/en-us/dotnet/core/rid-catalog#macos-rids - string osPart = RuntimeInformation.IsWindows() ? "win" : (RuntimeInformation.IsMacOS() ? "osx" : "linux"); + string osPart = OsDetector.IsWindows() ? "win" : (OsDetector.IsMacOS() ? "osx" : "linux"); string architecture = #if NETSTANDARD diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs index e9078f293f..70f959121e 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs @@ -1,9 +1,11 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.IO; using System.Linq; using System.Text; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; @@ -46,17 +48,19 @@ public DotNetCliCommand(string cliPath, string arguments, GenerateResult generat } public DotNetCliCommand WithArguments(string arguments) - => new (CliPath, arguments, GenerateResult, Logger, BuildPartition, EnvironmentVariables, Timeout, logOutput: LogOutput); + => new(CliPath, arguments, GenerateResult, Logger, BuildPartition, EnvironmentVariables, Timeout, logOutput: LogOutput); public DotNetCliCommand WithCliPath(string cliPath) - => new (cliPath, Arguments, GenerateResult, Logger, BuildPartition, EnvironmentVariables, Timeout, logOutput: LogOutput); + => new(cliPath, Arguments, GenerateResult, Logger, BuildPartition, EnvironmentVariables, Timeout, logOutput: LogOutput); [PublicAPI] public BuildResult RestoreThenBuild() { DotNetCliCommandExecutor.LogEnvVars(WithArguments(null)); +#pragma warning disable CS0618 // Type or member is obsolete var packagesResult = AddPackages(); +#pragma warning restore CS0618 // Type or member is obsolete if (!packagesResult.IsSuccess) return BuildResult.Failure(GenerateResult, packagesResult.AllInformation); @@ -96,7 +100,9 @@ public BuildResult RestoreThenBuildThenPublish() { DotNetCliCommandExecutor.LogEnvVars(WithArguments(null)); +#pragma warning disable CS0618 // Type or member is obsolete var packagesResult = AddPackages(); +#pragma warning restore CS0618 // Type or member is obsolete if (!packagesResult.IsSuccess) return BuildResult.Failure(GenerateResult, packagesResult.AllInformation); @@ -114,6 +120,8 @@ public BuildResult RestoreThenBuildThenPublish() return PublishNoRestore().ToBuildResult(GenerateResult); } + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("This method will soon be removed")] public DotNetCliCommandResult AddPackages() { var executionTime = new TimeSpan(0); @@ -149,6 +157,7 @@ public DotNetCliCommandResult PublishNoRestore() => DotNetCliCommandExecutor.Execute(WithArguments( GetPublishCommand(GenerateResult.ArtifactsPaths, BuildPartition, $"{Arguments} --no-restore", "publish-no-restore"))); + [Obsolete] internal static IEnumerable GetAddPackagesCommands(BuildPartition buildPartition) => GetNuGetAddPackageCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver); @@ -203,6 +212,7 @@ private static string GetCustomMsBuildArguments(BenchmarkCase benchmarkCase, IRe return string.Join(" ", msBuildArguments.Select(arg => arg.TextRepresentation)); } + [Obsolete] private static IEnumerable GetNuGetAddPackageCommands(BenchmarkCase benchmarkCase, IResolver resolver) { if (!benchmarkCase.Job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic)) @@ -217,7 +227,7 @@ private static string GetMandatoryMsBuildSettings(string buildConfiguration) { // we use these settings to make sure that MSBuild does the job and simply quits without spawning any long living processes // we want to avoid "file in use" and "zombie processes" issues - const string NoMsBuildZombieProcesses = "/p:UseSharedCompilation=false /p:BuildInParallel=false /m:1 /p:Deterministic=true"; + const string NoMsBuildZombieProcesses = "--nodeReuse:false /p:UseSharedCompilation=false /p:Deterministic=true"; const string EnforceOptimizations = "/p:Optimize=true"; if (string.Equals(buildConfiguration, RuntimeInformation.DebugConfigurationName, StringComparison.OrdinalIgnoreCase)) @@ -228,6 +238,7 @@ private static string GetMandatoryMsBuildSettings(string buildConfiguration) return $"{NoMsBuildZombieProcesses} {EnforceOptimizations}"; } + [Obsolete] private static string BuildAddPackageCommand(NuGetReference reference) { var commandBuilder = new StringBuilder(); @@ -262,10 +273,11 @@ internal static StringBuilder MaybeAppendOutputPaths(this StringBuilder stringBu ? stringBuilder : stringBuilder // Use AltDirectorySeparatorChar so it's not interpreted as an escaped quote `\"`. - .AppendArgument($"/p:IntermediateOutputPath=\"{artifactsPaths.IntermediateDirectoryPath}{Path.AltDirectorySeparatorChar}\"") + .AppendArgument($"/p:ArtifactsPath=\"{artifactsPaths.BuildArtifactsDirectoryPath}{Path.AltDirectorySeparatorChar}\"") .AppendArgument($"/p:OutDir=\"{artifactsPaths.BinariesDirectoryPath}{Path.AltDirectorySeparatorChar}\"") // OutputPath is legacy, per-project version of OutDir. We set both just in case. https://github.com/dotnet/msbuild/issues/87 .AppendArgument($"/p:OutputPath=\"{artifactsPaths.BinariesDirectoryPath}{Path.AltDirectorySeparatorChar}\"") + .AppendArgument($"/p:PublishDir=\"{artifactsPaths.PublishDirectoryPath}{Path.AltDirectorySeparatorChar}\"") .AppendArgument(isRestore ? string.Empty : $"--output \"{artifactsPaths.BinariesDirectoryPath}{Path.AltDirectorySeparatorChar}\""); } } diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs index 4348685369..10df4af9b3 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; @@ -56,7 +57,7 @@ public static DotNetCliCommandResult Execute(DotNetCliCommand parameters) internal static string? GetDotNetSdkVersion() { - using (var process = new Process { StartInfo = BuildStartInfo(customDotNetCliPath: null, workingDirectory: string.Empty, arguments: "--version") }) + using (var process = new Process { StartInfo = BuildStartInfo(customDotNetCliPath: null, workingDirectory: string.Empty, arguments: "--version", redirectStandardError: false) }) using (new ConsoleExitHandler(process, NullLogger.Instance)) { try @@ -137,7 +138,7 @@ internal static ProcessStartInfo BuildStartInfo(string? customDotNetCliPath, str private static string GetDefaultDotNetCliPath() { - if (!Portability.RuntimeInformation.IsLinux()) + if (!OsDetector.IsLinux()) return "dotnet"; using (var parentProcess = Process.GetProcessById(libc.getppid())) @@ -156,7 +157,7 @@ private static string GetDefaultDotNetCliPath() internal static string GetSdkPath(string cliPath) { - DotNetCliCommand cliCommand = new ( + DotNetCliCommand cliCommand = new( cliPath: cliPath, arguments: "--info", generateResult: null, diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandResult.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandResult.cs index 3d65270621..4a5b00a1fd 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandResult.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandResult.cs @@ -35,7 +35,7 @@ public static DotNetCliCommandResult Failure(TimeSpan time, string standardError [PublicAPI] public BuildResult ToBuildResult(GenerateResult generateResult) - => IsSuccess || File.Exists(generateResult.ArtifactsPaths.ExecutablePath) // dotnet cli could have successfully built the program, but returned 1 as exit code because it had some warnings + => IsSuccess ? BuildResult.Success(generateResult) : BuildResult.Failure(generateResult, AllInformation); } diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs index 55815ed005..d0595fe166 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs @@ -75,17 +75,20 @@ private ExecuteResult Execute(BenchmarkCase benchmarkCase, startInfo.SetEnvironmentVariables(benchmarkCase, resolver); - using (Process process = new () { StartInfo = startInfo }) - using (ConsoleExitHandler consoleExitHandler = new (process, logger)) - using (AsyncProcessOutputReader processOutputReader = new (process, logOutput: true, logger, readStandardError: false)) + using (Process process = new() { StartInfo = startInfo }) + using (ConsoleExitHandler consoleExitHandler = new(process, logger)) + using (AsyncProcessOutputReader processOutputReader = new(process, logOutput: true, logger, readStandardError: false)) { - Broker broker = new (logger, process, diagnoser, benchmarkCase, benchmarkId, inputFromBenchmark, acknowledgments); + Broker broker = new(logger, process, diagnoser, benchmarkCase, benchmarkId, inputFromBenchmark, acknowledgments); logger.WriteLineInfo($"// Execute: {process.StartInfo.FileName} {process.StartInfo.Arguments} in {process.StartInfo.WorkingDirectory}"); - diagnoser?.Handle(HostSignal.BeforeProcessStart, new DiagnoserActionParameters(process, benchmarkCase, benchmarkId)); + diagnoser?.Handle(HostSignal.BeforeProcessStart, broker.DiagnoserActionParameters); process.Start(); + + diagnoser?.Handle(HostSignal.AfterProcessStart, broker.DiagnoserActionParameters); + processOutputReader.BeginRead(); process.EnsureHighPriority(logger); diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs index a432fbf881..aaa2cda154 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs @@ -12,6 +12,8 @@ public abstract class DotNetCliGenerator : GeneratorBase { private static readonly string[] ProjectExtensions = { ".csproj", ".fsproj", ".vbroj" }; + private static readonly string[] SolutionExtensions = { ".sln", ".slnx" }; + [PublicAPI] public string TargetFrameworkMoniker { get; } [PublicAPI] public string CliPath { get; } @@ -111,7 +113,7 @@ protected override void GenerateBuildScript(BuildPartition buildPartition, Artif private static bool IsRootSolutionFolder(DirectoryInfo directoryInfo) => directoryInfo .GetFileSystemInfos() - .Any(fileInfo => fileInfo.Extension == ".sln" || fileInfo.Name == "global.json"); + .Any(fileInfo => SolutionExtensions.Contains(fileInfo.Extension) || fileInfo.Name == "global.json"); private static bool IsRootProjectFolder(DirectoryInfo directoryInfo) => directoryInfo diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/MsBuildErrorMapper.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/MsBuildErrorMapper.cs index f8f159ff96..9c95be192f 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/MsBuildErrorMapper.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/MsBuildErrorMapper.cs @@ -91,6 +91,8 @@ private static string Map(Capture capture) return "net8.0"; case ".NETCoreApp,Version=v9.0": return "net9.0"; + case ".NETCoreApp,Version=v10.0": + return "net10.0"; default: return capture.Value; // we don't want to throw for future versions of .NET } diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs index 6f48698cb9..807e394682 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs @@ -9,20 +9,21 @@ namespace BenchmarkDotNet.Toolchains.DotNetCli [PublicAPI] public class NetCoreAppSettings { - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp20 = new ("netcoreapp2.0", null, ".NET Core 2.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp21 = new ("netcoreapp2.1", null, ".NET Core 2.1"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp22 = new ("netcoreapp2.2", null, ".NET Core 2.2"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp30 = new ("netcoreapp3.0", null, ".NET Core 3.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp31 = new ("netcoreapp3.1", null, ".NET Core 3.1"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp50 = new ("net5.0", null, ".NET 5.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp60 = new ("net6.0", null, ".NET 6.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp70 = new ("net7.0", null, ".NET 7.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp80 = new ("net8.0", null, ".NET 8.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp90 = new ("net9.0", null, ".NET 9.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp20 = new("netcoreapp2.0", null, ".NET Core 2.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp21 = new("netcoreapp2.1", null, ".NET Core 2.1"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp22 = new("netcoreapp2.2", null, ".NET Core 2.2"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp30 = new("netcoreapp3.0", null, ".NET Core 3.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp31 = new("netcoreapp3.1", null, ".NET Core 3.1"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp50 = new("net5.0", null, ".NET 5.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp60 = new("net6.0", null, ".NET 6.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp70 = new("net7.0", null, ".NET 7.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp80 = new("net8.0", null, ".NET 8.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp90 = new("net9.0", null, ".NET 9.0"); + [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp10_0 = new("net10.0", null, ".NET 10.0"); /// /// - /// sample values: netcoreapp2.0, netcoreapp2.1 + /// sample values: net6.0, net8.0 /// /// /// used in the auto-generated .csproj file @@ -64,7 +65,7 @@ public NetCoreAppSettings( } /// - /// sample values: netcoreapp2.0, netcoreapp2.1 + /// sample values: net6.0, net8.0 /// public string TargetFrameworkMoniker { get; } diff --git a/src/BenchmarkDotNet/Toolchains/Executor.cs b/src/BenchmarkDotNet/Toolchains/Executor.cs index 145e527439..adc05e12e6 100644 --- a/src/BenchmarkDotNet/Toolchains/Executor.cs +++ b/src/BenchmarkDotNet/Toolchains/Executor.cs @@ -41,16 +41,16 @@ private static ExecuteResult Execute(BenchmarkCase benchmarkCase, BenchmarkId be { try { - using AnonymousPipeServerStream inputFromBenchmark = new (PipeDirection.In, HandleInheritability.Inheritable); - using AnonymousPipeServerStream acknowledgments = new (PipeDirection.Out, HandleInheritability.Inheritable); + using AnonymousPipeServerStream inputFromBenchmark = new(PipeDirection.In, HandleInheritability.Inheritable); + using AnonymousPipeServerStream acknowledgments = new(PipeDirection.Out, HandleInheritability.Inheritable); string args = benchmarkId.ToArguments(inputFromBenchmark.GetClientHandleAsString(), acknowledgments.GetClientHandleAsString()); - using (Process process = new () { StartInfo = CreateStartInfo(benchmarkCase, artifactsPaths, args, resolver) }) - using (ConsoleExitHandler consoleExitHandler = new (process, logger)) - using (AsyncProcessOutputReader processOutputReader = new (process, logOutput: true, logger, readStandardError: false)) + using (Process process = new() { StartInfo = CreateStartInfo(benchmarkCase, artifactsPaths, args, resolver) }) + using (ConsoleExitHandler consoleExitHandler = new(process, logger)) + using (AsyncProcessOutputReader processOutputReader = new(process, logOutput: true, logger, readStandardError: false)) { - Broker broker = new (logger, process, diagnoser, benchmarkCase, benchmarkId, inputFromBenchmark, acknowledgments); + Broker broker = new(logger, process, diagnoser, benchmarkCase, benchmarkId, inputFromBenchmark, acknowledgments); diagnoser?.Handle(HostSignal.BeforeProcessStart, new DiagnoserActionParameters(process, benchmarkCase, benchmarkId)); @@ -79,6 +79,8 @@ private static ExecuteResult Execute(Process process, BenchmarkCase benchmarkCas return new ExecuteResult(true, null, null, Array.Empty(), Array.Empty(), Array.Empty(), launchIndex); } + broker.Diagnoser?.Handle(HostSignal.AfterProcessStart, broker.DiagnoserActionParameters); + processOutputReader.BeginRead(); process.EnsureHighPriority(logger); diff --git a/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs b/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs index cab9d811c4..165dedcbba 100644 --- a/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs +++ b/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs @@ -2,6 +2,7 @@ using System.IO; using System.Text; using BenchmarkDotNet.Code; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; @@ -50,17 +51,17 @@ [PublicAPI] protected virtual string GetBinariesDirectoryPath(string buildArtifa => buildArtifactsDirectoryPath; /// - /// returns a path where intermediate files should be found after the build (usually \obj) + /// returns a path where the publish directory should be found after the build (usually \publish) /// [PublicAPI] - protected virtual string GetIntermediateDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => string.Empty; + protected virtual string GetPublishDirectoryPath(string buildArtifactsDirectoryPath, string configuration) + => Path.Combine(buildArtifactsDirectoryPath, "publish"); /// /// returns OS-specific executable extension /// [PublicAPI] protected virtual string GetExecutableExtension() - => RuntimeInformation.ExecutableExtension; + => OsDetector.ExecutableExtension; /// /// returns a path to the auto-generated .csproj file @@ -137,12 +138,12 @@ private ArtifactsPaths GetArtifactsPaths(BuildPartition buildPartition, string r rootArtifactsFolderPath: rootArtifactsFolderPath, buildArtifactsDirectoryPath: buildArtifactsDirectoryPath, binariesDirectoryPath: binariesDirectoryPath, - intermediateDirectoryPath: GetIntermediateDirectoryPath(buildArtifactsDirectoryPath, buildPartition.BuildConfiguration), + publishDirectoryPath: GetPublishDirectoryPath(buildArtifactsDirectoryPath, buildPartition.BuildConfiguration), programCodePath: Path.Combine(buildArtifactsDirectoryPath, $"{programName}{codeFileExtension}"), appConfigPath: $"{executablePath}.config", nuGetConfigPath: Path.Combine(buildArtifactsDirectoryPath, "NuGet.config"), projectFilePath: GetProjectFilePath(buildArtifactsDirectoryPath), - buildScriptFilePath: Path.Combine(buildArtifactsDirectoryPath, $"{programName}{RuntimeInformation.ScriptFileExtension}"), + buildScriptFilePath: Path.Combine(buildArtifactsDirectoryPath, $"{programName}{OsDetector.ScriptFileExtension}"), executablePath: executablePath, programName: programName, packagesDirectoryName: GetPackagesDirectoryPath(buildArtifactsDirectoryPath)); diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs index e0618cb642..4796c84a77 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs @@ -247,6 +247,10 @@ private static void EmitNoArgsMethodCallPopReturn( private TypeBuilder runnableBuilder; private ConsumableTypeInfo consumableInfo; private ConsumeEmitter consumeEmitter; + private ConsumableTypeInfo globalSetupReturnInfo; + private ConsumableTypeInfo globalCleanupReturnInfo; + private ConsumableTypeInfo iterationSetupReturnInfo; + private ConsumableTypeInfo iterationCleanupReturnInfo; private FieldBuilder globalSetupActionField; private FieldBuilder globalCleanupActionField; @@ -358,6 +362,10 @@ private void InitForEmitRunnable(BenchmarkBuildInfo newBenchmark) consumableInfo = new ConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.WorkloadMethod.ReturnType); consumeEmitter = ConsumeEmitter.GetConsumeEmitter(consumableInfo); + globalSetupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.GlobalSetupMethod?.ReturnType); + globalCleanupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.GlobalCleanupMethod?.ReturnType); + iterationSetupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.IterationSetupMethod?.ReturnType); + iterationCleanupReturnInfo = GetConsumableTypeInfo(benchmark.BenchmarkCase.Descriptor.IterationCleanupMethod?.ReturnType); // Init types runnableBuilder = DefineRunnableTypeBuilder(benchmark, moduleBuilder); @@ -365,6 +373,11 @@ private void InitForEmitRunnable(BenchmarkBuildInfo newBenchmark) workloadDelegateType = EmitWorkloadDelegateType(); } + private static ConsumableTypeInfo GetConsumableTypeInfo(Type methodReturnType) + { + return methodReturnType == null ? null : new ConsumableTypeInfo(methodReturnType); + } + private Type EmitOverheadDelegateType() { // .class public auto ansi sealed BenchmarkDotNet.Autogenerated.Runnable_0OverheadDelegate @@ -890,34 +903,84 @@ private void EmitSetupCleanupMethods() { // Emit Setup/Cleanup methods // We emit empty method instead of EmptyAction = "() => { }" - globalSetupMethod = EmitWrapperMethod( - GlobalSetupMethodName, - Descriptor.GlobalSetupMethod); - globalCleanupMethod = EmitWrapperMethod( - GlobalCleanupMethodName, - Descriptor.GlobalCleanupMethod); - iterationSetupMethod = EmitWrapperMethod( - IterationSetupMethodName, - Descriptor.IterationSetupMethod); - iterationCleanupMethod = EmitWrapperMethod( - IterationCleanupMethodName, - Descriptor.IterationCleanupMethod); + globalSetupMethod = EmitWrapperMethod(GlobalSetupMethodName, Descriptor.GlobalSetupMethod, globalSetupReturnInfo); + globalCleanupMethod = EmitWrapperMethod(GlobalCleanupMethodName, Descriptor.GlobalCleanupMethod, globalCleanupReturnInfo); + iterationSetupMethod = EmitWrapperMethod(IterationSetupMethodName, Descriptor.IterationSetupMethod, iterationSetupReturnInfo); + iterationCleanupMethod = EmitWrapperMethod(IterationCleanupMethodName, Descriptor.IterationCleanupMethod, iterationCleanupReturnInfo); } - private MethodBuilder EmitWrapperMethod(string methodName, MethodInfo optionalTargetMethod) + private MethodBuilder EmitWrapperMethod(string methodName, MethodInfo optionalTargetMethod, ConsumableTypeInfo returnTypeInfo) { var methodBuilder = runnableBuilder.DefinePrivateVoidInstanceMethod(methodName); var ilBuilder = methodBuilder.GetILGenerator(); if (optionalTargetMethod != null) - EmitNoArgsMethodCallPopReturn(methodBuilder, optionalTargetMethod, ilBuilder, forceDirectCall: true); + { + if (returnTypeInfo?.IsAwaitable == true) + { + EmitAwaitableSetupTeardown(methodBuilder, optionalTargetMethod, ilBuilder, returnTypeInfo); + } + else + { + EmitNoArgsMethodCallPopReturn(methodBuilder, optionalTargetMethod, ilBuilder, forceDirectCall: true); + } + } ilBuilder.EmitVoidReturn(methodBuilder); return methodBuilder; } + private void EmitAwaitableSetupTeardown( + MethodBuilder methodBuilder, + MethodInfo targetMethod, + ILGenerator ilBuilder, + ConsumableTypeInfo returnTypeInfo) + { + if (targetMethod == null) + throw new ArgumentNullException(nameof(targetMethod)); + + if (returnTypeInfo.WorkloadMethodReturnType == typeof(void)) + { + ilBuilder.Emit(OpCodes.Ldarg_0); + } + /* + // call for instance + // GlobalSetup(); + IL_0006: ldarg.0 + IL_0007: call instance void [BenchmarkDotNet]BenchmarkDotNet.Samples.SampleBenchmark::GlobalSetup() + */ + /* + // call for static + // GlobalSetup(); + IL_0006: call string [BenchmarkDotNet]BenchmarkDotNet.Samples.SampleBenchmark::GlobalCleanup() + */ + if (targetMethod.IsStatic) + { + ilBuilder.Emit(OpCodes.Call, targetMethod); + + } + else if (methodBuilder.IsStatic) + { + throw new InvalidOperationException( + $"[BUG] Static method {methodBuilder.Name} tries to call instance member {targetMethod.Name}"); + } + else + { + ilBuilder.Emit(OpCodes.Ldarg_0); + ilBuilder.Emit(OpCodes.Call, targetMethod); + } + + /* + // BenchmarkDotNet.Helpers.AwaitHelper.GetResult(...); + IL_000e: call !!0 BenchmarkDotNet.Helpers.AwaitHelper::GetResult(valuetype [System.Runtime]System.Threading.Tasks.ValueTask`1) + */ + + ilBuilder.Emit(OpCodes.Call, returnTypeInfo.GetResultMethod); + ilBuilder.Emit(OpCodes.Pop); + } + private void EmitCtorBody() { var ilBuilder = ctorMethod.GetILGenerator(); @@ -983,7 +1046,7 @@ private MethodBuilder EmitRunMethod() { var prepareForRunMethodTemplate = typeof(RunnableReuse).GetMethod(nameof(RunnableReuse.PrepareForRun)) ?? throw new MissingMemberException(nameof(RunnableReuse.PrepareForRun)); - var resultTuple = new ValueTuple(); + (Job, EngineParameters, IEngineFactory) resultTuple = new(); /* .method public hidebysig static diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitArtifactsPath.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitArtifactsPath.cs index 0686a322ac..ca8ca60974 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitArtifactsPath.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitArtifactsPath.cs @@ -12,7 +12,7 @@ public InProcessEmitArtifactsPath( baseArtifacts.RootArtifactsFolderPath, baseArtifacts.BuildArtifactsDirectoryPath, baseArtifacts.BinariesDirectoryPath, - baseArtifacts.IntermediateDirectoryPath, + baseArtifacts.PublishDirectoryPath, baseArtifacts.ProgramCodePath, baseArtifacts.AppConfigPath, baseArtifacts.NuGetConfigPath, diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs index 64d2019a75..60d6f8a6ae 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitExecutor.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Reflection; using System.Threading; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; @@ -59,7 +60,7 @@ public ExecuteResult Execute(ExecuteParameters executeParameters) if (executeParameters.BenchmarkCase.Descriptor.WorkloadMethod .GetCustomAttributes(false) .Any() && - Portability.RuntimeInformation.IsWindows()) + OsDetector.IsWindows()) { runThread.SetApartmentState(ApartmentState.STA); } diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitGenerator.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitGenerator.cs index 18734fcf46..5e74d65f12 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitGenerator.cs @@ -47,7 +47,7 @@ private ArtifactsPaths GetArtifactsPaths(BuildPartition buildPartition, string r rootArtifactsFolderPath: rootArtifactsFolderPath, buildArtifactsDirectoryPath: buildArtifactsDirectoryPath, binariesDirectoryPath: binariesDirectoryPath, - intermediateDirectoryPath: null, + publishDirectoryPath: null, programCodePath: null, appConfigPath: null, nuGetConfigPath: null, diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs index 49cd24e3d4..b50ba77c1c 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitExecutor.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Reflection; using System.Threading; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; @@ -59,7 +60,7 @@ public ExecuteResult Execute(ExecuteParameters executeParameters) var runThread = new Thread(() => exitCode = ExecuteCore(host, executeParameters)); if (executeParameters.BenchmarkCase.Descriptor.WorkloadMethod.GetCustomAttributes(false).Any() && - Portability.RuntimeInformation.IsWindows()) + OsDetector.IsWindows()) { runThread.SetApartmentState(ApartmentState.STA); } diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs index a52e7ebcf8..18ee40c30a 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs @@ -1,4 +1,5 @@ -using System; +using System; +using System.Diagnostics.CodeAnalysis; using System.Reflection; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; @@ -15,6 +16,9 @@ namespace BenchmarkDotNet.Toolchains.InProcess.NoEmit /// internal class InProcessNoEmitRunner { +#if NET6_0_OR_GREATER + [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Runnable))] +#endif public static int Run(IHost host, BenchmarkCase benchmarkCase) { // the first thing to do is to let diagnosers hook in before anything happens @@ -157,4 +161,4 @@ public static void RunCore(IHost host, BenchmarkCase benchmarkCase) } } } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs index acbeeffe38..30135c15cd 100644 --- a/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs @@ -48,12 +48,14 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas benchmarkCase); } +#pragma warning disable CS0618 // Type or member is obsolete if (benchmarkCase.Job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic)) { yield return new ValidationError(true, "The MonoAOT toolchain does not allow specifying NuGet package dependencies", benchmarkCase); } +#pragma warning restore CS0618 // Type or member is obsolete } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs index 56f2ebaa56..222eb2111b 100644 --- a/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs @@ -12,6 +12,7 @@ public class MonoToolchain : CsProjCoreToolchain, IEquatable [PublicAPI] public static readonly IToolchain Mono70 = From(new NetCoreAppSettings("net7.0", null, "mono70")); [PublicAPI] public static readonly IToolchain Mono80 = From(new NetCoreAppSettings("net8.0", null, "mono80")); [PublicAPI] public static readonly IToolchain Mono90 = From(new NetCoreAppSettings("net9.0", null, "mono90")); + [PublicAPI] public static readonly IToolchain Mono10_0 = From(new NetCoreAppSettings("net10.0", null, "mono10_0")); private MonoToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) : base(name, generator, builder, executor, customDotNetCliPath) diff --git a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs index 833c113d84..6ba5968617 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs @@ -1,6 +1,7 @@ using System.IO; using System.Text; using System.Xml; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; @@ -42,7 +43,6 @@ protected override void GenerateProject(BuildPartition buildPartition, Artifacts .Replace("$TFM$", TargetFrameworkMoniker) .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) .Replace("$COPIEDSETTINGS$", customProperties) - .Replace("$CONFIGURATIONNAME$", buildPartition.BuildConfiguration) .Replace("$SDKNAME$", sdkName) .Replace("$RUNTIMEPACK$", CustomRuntimePack ?? "") .Replace("$COMPILERBINARYPATH$", AotCompilerPath) @@ -53,8 +53,11 @@ protected override void GenerateProject(BuildPartition buildPartition, Artifacts File.WriteAllText(artifactsPaths.ProjectFilePath, content); } + protected override string GetPublishDirectoryPath(string buildArtifactsDirectoryPath, string configuration) + => Path.Combine(GetBinariesDirectoryPath(buildArtifactsDirectoryPath, configuration), "publish"); + protected override string GetExecutablePath(string binariesDirectoryPath, string programName) - => Portability.RuntimeInformation.IsWindows() + => OsDetector.IsWindows() ? Path.Combine(binariesDirectoryPath, "publish", $"{programName}.exe") : Path.Combine(binariesDirectoryPath, "publish", programName); diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs index 7e857ff364..e6756ebcbe 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs @@ -38,8 +38,8 @@ private static ExecuteResult Execute(BenchmarkCase benchmarkCase, BenchmarkId be try { using (Process process = CreateProcess(benchmarkCase, artifactsPaths, benchmarkId.ToArguments(), resolver)) - using (ConsoleExitHandler consoleExitHandler = new (process, logger)) - using (AsyncProcessOutputReader processOutputReader = new (process, logOutput: true, logger, readStandardError: false)) + using (ConsoleExitHandler consoleExitHandler = new(process, logger)) + using (AsyncProcessOutputReader processOutputReader = new(process, logOutput: true, logger, readStandardError: false)) { diagnoser?.Handle(HostSignal.BeforeProcessStart, new DiagnoserActionParameters(process, benchmarkCase, benchmarkId)); diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs index 7c9aa8826f..c33642f6a9 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs @@ -57,7 +57,6 @@ protected void GenerateProjectFile(BuildPartition buildPartition, ArtifactsPaths .Replace("$TFM$", TargetFrameworkMoniker) .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) .Replace("$COPIEDSETTINGS$", customProperties) - .Replace("$CONFIGURATIONNAME$", buildPartition.BuildConfiguration) .Replace("$SDKNAME$", sdkName) .Replace("$WASMDATADIR$", runtime.WasmDataDir) .Replace("$TARGET$", CustomRuntimePack != null ? "PublishWithCustomRuntimePack" : "Publish") diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs index 42e5c428d2..71ad2ade2b 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; @@ -26,7 +27,7 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas yield return validationError; } - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) { yield return new ValidationError(true, $"{nameof(WasmToolchain)} is supported only on Unix, benchmark '{benchmarkCase.DisplayInfo}' might not work correctly", diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs index 2f2a6d80f3..e9c64a82f5 100644 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs @@ -3,13 +3,15 @@ using System.IO; using System.Linq; using System.Text; +using System.Xml; using BenchmarkDotNet.ConsoleArguments; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Detectors.Cpu; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Portability.Cpu; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.DotNetCli; @@ -57,7 +59,7 @@ internal Generator(string ilCompilerVersion, private readonly string ilcOptimizationPreference; private readonly string ilcInstructionSet; - protected override string GetExecutableExtension() => RuntimeInformation.ExecutableExtension; + protected override string GetExecutableExtension() => OsDetector.ExecutableExtension; protected override string GetBuildArtifactsDirectoryPath(BuildPartition buildPartition, string programName) => useTempFolderForRestore @@ -132,14 +134,16 @@ private string GenerateProjectForNuGetBuild(BuildPartition buildPartition, Artif false true false - true + true {ilcOptimizationPreference} + {ilcOptimizationPreference} {GetTrimmingSettings()} - {ilcGenerateCompleteTypeMetadata} {ilcGenerateStackTraceData} + {ilcGenerateStackTraceData} false false false + true {GetInstructionSetSettings(buildPartition)} {GetRuntimeSettings(buildPartition.RepresentativeBenchmarkCase.Job.Environment.Gc, buildPartition.Resolver)} @@ -153,8 +157,20 @@ private string GenerateProjectForNuGetBuild(BuildPartition buildPartition, Artif {string.Join(Environment.NewLine, GetRdXmlFiles(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger).Select(file => $""))} +{GetCustomProperties(buildPartition, logger)} "; + private string GetCustomProperties(BuildPartition buildPartition, ILogger logger) + { + var projectFile = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger); + var xmlDoc = new XmlDocument(); + xmlDoc.Load(projectFile.FullName); + + (string customProperties, _) = GetSettingsThatNeedToBeCopied(xmlDoc, projectFile); + return customProperties; + } + + private string GetILCompilerPackageReference() => string.IsNullOrEmpty(ilCompilerVersion) ? "" : $@""; @@ -220,7 +236,7 @@ private void GenerateReflectionFile(ArtifactsPaths artifactsPaths) private string GetCurrentInstructionSet(Platform platform) => string.Join(",", GetCurrentProcessInstructionSets(platform)); - // based on https://github.com/dotnet/runtime/blob/ce61c09a5f6fc71d8f717d3fc4562f42171869a0/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs#L727 + // based on https://github.com/dotnet/runtime/tree/v10.0.0-rc.1.25451.107/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt private IEnumerable GetCurrentProcessInstructionSets(Platform platform) { if (!ConfigParser.TryParse(TargetFrameworkMoniker, out RuntimeMoniker runtimeMoniker)) @@ -240,39 +256,77 @@ private IEnumerable GetCurrentProcessInstructionSets(Platform platform) case Platform.X86: case Platform.X64: if (HardwareIntrinsics.IsX86BaseSupported) yield return "base"; - if (HardwareIntrinsics.IsX86SseSupported) yield return "sse"; - if (HardwareIntrinsics.IsX86Sse2Supported) yield return "sse2"; - if (HardwareIntrinsics.IsX86Sse3Supported) yield return "sse3"; - if (HardwareIntrinsics.IsX86Ssse3Supported) yield return "ssse3"; - if (HardwareIntrinsics.IsX86Sse41Supported) yield return "sse4.1"; - if (HardwareIntrinsics.IsX86Sse42Supported) yield return "sse4.2"; + if (HardwareIntrinsics.IsX86Sse42Supported) + { + if (runtimeMoniker <= RuntimeMoniker.NativeAot10_0) yield return "sse4.2"; + if (runtimeMoniker <= RuntimeMoniker.NativeAot90) yield return "popcnt"; + } if (HardwareIntrinsics.IsX86AvxSupported) yield return "avx"; - if (HardwareIntrinsics.IsX86Avx2Supported) yield return "avx2"; - if (HardwareIntrinsics.IsX86Avx512FSupported) yield return "avx512f"; - if (HardwareIntrinsics.IsX86Avx512BWSupported) yield return "avx512bw"; - if (HardwareIntrinsics.IsX86Avx512CDSupported) yield return "avx512cd"; - if (HardwareIntrinsics.IsX86Avx512DQSupported) yield return "avx512dq"; - if (HardwareIntrinsics.IsX86Avx512VbmiSupported) yield return "avx512vbmi"; - if (HardwareIntrinsics.IsX86AesSupported) yield return "aes"; - if (HardwareIntrinsics.IsX86Bmi1Supported) yield return "bmi"; - if (HardwareIntrinsics.IsX86Bmi2Supported) yield return "bmi2"; - if (HardwareIntrinsics.IsX86FmaSupported) yield return "fma"; - if (HardwareIntrinsics.IsX86LzcntSupported) yield return "lzcnt"; - if (HardwareIntrinsics.IsX86PclmulqdqSupported) yield return "pclmul"; - if (HardwareIntrinsics.IsX86PopcntSupported) yield return "popcnt"; + if (HardwareIntrinsics.IsX86Avx2Supported) + { + yield return "avx2"; + + if (runtimeMoniker <= RuntimeMoniker.NativeAot90) + { + yield return "bmi"; + yield return "bmi2"; + yield return "fma"; + yield return "lzcnt"; + } + } + if (HardwareIntrinsics.IsX86Avx512Supported && (runtimeMoniker > RuntimeMoniker.NativeAot80)) + { + if (runtimeMoniker >= RuntimeMoniker.NativeAot10_0) + { + yield return "avx512"; + } + else + { + yield return "avx512f"; + yield return "avx512f_vl"; + yield return "avx512bw"; + yield return "avx512bw_vl"; + yield return "avx512cd"; + yield return "avx512cd_vl"; + yield return "avx512dq"; + yield return "avx512dq_vl"; + } + } + if (HardwareIntrinsics.IsX86Avx512v2Supported && (runtimeMoniker > RuntimeMoniker.NativeAot80)) + { + if (runtimeMoniker >= RuntimeMoniker.NativeAot10_0) + { + yield return "avx512v2"; + } + else + { + yield return "avx512vbmi"; + yield return "avx512vbmi_vl"; + } + } + if (HardwareIntrinsics.IsX86Avx512v3Supported && (runtimeMoniker >= RuntimeMoniker.NativeAot10_0)) yield return "avx512v3"; + if (HardwareIntrinsics.IsX86Avx10v1Supported && (runtimeMoniker >= RuntimeMoniker.NativeAot90)) yield return "avx10v1"; + if (HardwareIntrinsics.IsX86Avx10v2Supported && (runtimeMoniker >= RuntimeMoniker.NativeAot10_0)) yield return "avx10v2"; + if (HardwareIntrinsics.IsX86AesSupported) + { + yield return "aes"; + if (runtimeMoniker <= RuntimeMoniker.NativeAot90) yield return "pclmul"; + } if (HardwareIntrinsics.IsX86AvxVnniSupported) yield return "avxvnni"; if (HardwareIntrinsics.IsX86SerializeSupported && runtimeMoniker > RuntimeMoniker.NativeAot70) yield return "serialize"; // https://github.com/dotnet/BenchmarkDotNet/issues/2463#issuecomment-1809625008 break; case Platform.Arm64: - if (HardwareIntrinsics.IsArmBaseSupported) yield return "base"; - if (HardwareIntrinsics.IsArmAdvSimdSupported) yield return "neon"; + if (HardwareIntrinsics.IsArmBaseSupported) + { + yield return "base"; + yield return "neon"; + } if (HardwareIntrinsics.IsArmAesSupported) yield return "aes"; if (HardwareIntrinsics.IsArmCrc32Supported) yield return "crc"; if (HardwareIntrinsics.IsArmDpSupported) yield return "dotprod"; if (HardwareIntrinsics.IsArmRdmSupported) yield return "rdma"; if (HardwareIntrinsics.IsArmSha1Supported) yield return "sha1"; if (HardwareIntrinsics.IsArmSha256Supported) yield return "sha2"; - // todo: handle "lse" break; default: yield break; diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs index 2ef6dd8a23..9d476bf0e0 100644 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs @@ -40,6 +40,14 @@ public class NativeAotToolchain : Toolchain .TargetFrameworkMoniker("net9.0") .ToToolchain(); + /// + /// compiled as net10.0, targets latest NativeAOT build from the .NET 10 feed: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json + /// + public static readonly IToolchain Net10_0 = CreateBuilder() + .UseNuGet("", "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json") + .TargetFrameworkMoniker("net10.0") + .ToToolchain(); + internal NativeAotToolchain(string displayName, string ilCompilerVersion, string runtimeFrameworkVersion, string targetFrameworkMoniker, string runtimeIdentifier, diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs index fc352ea50d..1bd248e5eb 100644 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs @@ -53,8 +53,8 @@ public NativeAotToolchainBuilder UseLocalBuild(DirectoryInfo ilcPackages) if (!ilcPackages.Exists) throw new DirectoryNotFoundException($"{ilcPackages} provided as {nameof(ilcPackages)} does NOT exist"); Feeds["local"] = ilcPackages.FullName; - ilCompilerVersion = "9.0.0-dev"; - Feeds["dotnet9"] = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v3/index.json"; + ilCompilerVersion = "10.0.0-dev"; + Feeds["dotnet10"] = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10/nuget/v3/index.json"; useTempFolderForRestore = true; DisplayName("local ILCompiler build"); diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs index 0718d6b8cb..3ba7054f7a 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using System.Reflection; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; @@ -27,9 +28,9 @@ protected override string[] GetArtifactsToCleanup(ArtifactsPaths artifactsPaths) protected override void GenerateBuildScript(BuildPartition buildPartition, ArtifactsPaths artifactsPaths) { - string prefix = RuntimeInformation.IsWindows() ? "" : "#!/bin/bash\n"; + string prefix = OsDetector.IsWindows() ? "" : "#!/bin/bash\n"; var list = new List(); - if (!RuntimeInformation.IsWindows()) + if (!OsDetector.IsWindows()) list.Add("mono"); list.Add("csc"); list.Add("/noconfig"); diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs index 31c4d68c29..6774268dcc 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs @@ -51,12 +51,14 @@ public override IEnumerable Validate(BenchmarkCase benchmarkCas benchmarkCase); } +#pragma warning disable CS0618 // Type or member is obsolete if (benchmarkCase.Job.HasValue(InfrastructureMode.NuGetReferencesCharacteristic)) { yield return new ValidationError(true, "The Roslyn toolchain does not allow specifying NuGet package dependencies", benchmarkCase); } +#pragma warning restore CS0618 // Type or member is obsolete } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs index c2872810fc..2beab5018c 100644 --- a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs +++ b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs @@ -1,4 +1,5 @@ using System; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; @@ -46,9 +47,9 @@ internal static IToolchain GetToolchain(this Runtime runtime, Descriptor? descri : CsProjClassicNetToolchain.From(clrRuntime.MsBuildMoniker); case MonoRuntime mono: - if (RuntimeInformation.IsAndroid()) + if (OsDetector.IsAndroid()) return InProcessEmitToolchain.Instance; - if (RuntimeInformation.IsIOS()) + if (OsDetector.IsIOS()) return InProcessNoEmitToolchain.Instance; if (!string.IsNullOrEmpty(mono.AotArgs)) return MonoAotToolchain.Instance; @@ -65,6 +66,7 @@ internal static IToolchain GetToolchain(this Runtime runtime, Descriptor? descri RuntimeMoniker.Mono70 => GetToolchain(RuntimeMoniker.Net70), RuntimeMoniker.Mono80 => GetToolchain(RuntimeMoniker.Net80), RuntimeMoniker.Mono90 => GetToolchain(RuntimeMoniker.Net90), + RuntimeMoniker.Mono10_0 => GetToolchain(RuntimeMoniker.Net10_0), _ => CsProjCoreToolchain.From(new NetCoreAppSettings(mono.MsBuildMoniker, null, mono.Name)) }; } @@ -154,6 +156,9 @@ private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker) case RuntimeMoniker.Net90: return CsProjCoreToolchain.NetCoreApp90; + case RuntimeMoniker.Net10_0: + return CsProjCoreToolchain.NetCoreApp10_0; + case RuntimeMoniker.NativeAot60: return NativeAotToolchain.Net60; @@ -166,6 +171,9 @@ private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker) case RuntimeMoniker.NativeAot90: return NativeAotToolchain.Net90; + case RuntimeMoniker.NativeAot10_0: + return NativeAotToolchain.Net10_0; + case RuntimeMoniker.Mono60: return MonoToolchain.Mono60; @@ -178,6 +186,9 @@ private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker) case RuntimeMoniker.Mono90: return MonoToolchain.Mono90; + case RuntimeMoniker.Mono10_0: + return MonoToolchain.Mono10_0; + default: throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "RuntimeMoniker not supported"); } diff --git a/src/BenchmarkDotNet/Validators/CompilationValidator.cs b/src/BenchmarkDotNet/Validators/CompilationValidator.cs index d8c2ec6c5b..b0bb87681d 100644 --- a/src/BenchmarkDotNet/Validators/CompilationValidator.cs +++ b/src/BenchmarkDotNet/Validators/CompilationValidator.cs @@ -7,6 +7,7 @@ using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains; using Microsoft.CodeAnalysis.CSharp; +using BenchmarkDotNet.Attributes; namespace BenchmarkDotNet.Validators { @@ -25,8 +26,38 @@ private CompilationValidator() { } public IEnumerable Validate(ValidationParameters validationParameters) => ValidateCSharpNaming(validationParameters.Benchmarks) .Union(ValidateNamingConflicts(validationParameters.Benchmarks)) + .Union(ValidateClassModifiers((validationParameters.Benchmarks)) .Union(ValidateAccessModifiers(validationParameters.Benchmarks)) - .Union(ValidateBindingModifiers(validationParameters.Benchmarks)); + .Union(ValidateBindingModifiers(validationParameters.Benchmarks)) + ); + + private static IEnumerable ValidateClassModifiers(IEnumerable benchmarks) + { + return benchmarks + .Distinct(BenchmarkMethodEqualityComparer.Instance) + .SelectMany(benchmark => + { + var type = benchmark.Descriptor.Type; + var errors = new List(); + + if (type.IsSealed) + { + errors.Add(new ValidationError( + true, + $"Benchmarked method `{benchmark.Descriptor.WorkloadMethod.Name}` is within a sealed class, Declaring type must be unsealed.", + benchmark)); + } + if (!type.IsVisible) + { + errors.Add(new ValidationError( + true, + $"Benchmarked method `{benchmark.Descriptor.WorkloadMethod.Name}` is within a non-visible class, all declaring types must be public.", + benchmark)); + } + // TODO: Generics validation + return errors; + }); + } private static IEnumerable ValidateCSharpNaming(IEnumerable benchmarks) => benchmarks diff --git a/src/BenchmarkDotNet/Validators/ConfigCompatibilityValidator.cs b/src/BenchmarkDotNet/Validators/ConfigCompatibilityValidator.cs deleted file mode 100644 index 9732f40383..0000000000 --- a/src/BenchmarkDotNet/Validators/ConfigCompatibilityValidator.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System.Linq; -using System.Collections.Generic; -using BenchmarkDotNet.Reports; - -namespace BenchmarkDotNet.Validators -{ - public class ConfigCompatibilityValidator : IValidator - { - public static readonly ConfigCompatibilityValidator FailOnError = new ConfigCompatibilityValidator(); - - public bool TreatsWarningsAsErrors => true; - - public IEnumerable Validate(ValidationParameters validationParameters) - { - var orderers = - validationParameters - .Benchmarks - .Where(benchmark => benchmark.Config.Orderer != Order.DefaultOrderer.Instance) - .Select(benchmark => benchmark.Config.Orderer) - .Distinct(); - - if (orderers.Count() > 1) - yield return new ValidationError(true, "You use JoinSummary options, but provided configurations cannot be joined. Only one Orderer per benchmark cases is allowed."); - - var styles = - validationParameters - .Benchmarks - .Where(benchmark => benchmark.Config.SummaryStyle != SummaryStyle.Default - && benchmark.Config.SummaryStyle != null) // Paranoid - .Select(benchmark => benchmark.Config.SummaryStyle) - .Distinct(); - - if (styles.Count() > 1) - yield return new ValidationError(true, "You use JoinSummary options, but provided configurations cannot be joined. Only one SummaryStyle per benchmark cases is allowed."); - } - } -} diff --git a/src/BenchmarkDotNet/Validators/DotNetSdkVersionValidator.cs b/src/BenchmarkDotNet/Validators/DotNetSdkValidator.cs similarity index 57% rename from src/BenchmarkDotNet/Validators/DotNetSdkVersionValidator.cs rename to src/BenchmarkDotNet/Validators/DotNetSdkValidator.cs index 8b61a0d51c..66b6a35aca 100644 --- a/src/BenchmarkDotNet/Validators/DotNetSdkVersionValidator.cs +++ b/src/BenchmarkDotNet/Validators/DotNetSdkValidator.cs @@ -1,4 +1,5 @@ using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; using System; @@ -20,35 +21,22 @@ public static IEnumerable ValidateCoreSdks(string? customDotNet if (IsCliPathInvalid(customDotNetCliPath, benchmark, out ValidationError? cliPathError)) { yield return cliPathError; + yield break; } - else if (TryGetSdkVersion(benchmark, out string requiredSdkVersion)) + var requiredSdkVersion = benchmark.GetRuntime().RuntimeMoniker.GetRuntimeVersion(); + if (!GetInstalledDotNetSdks(customDotNetCliPath).Any(sdk => sdk >= requiredSdkVersion)) { - var installedSdks = GetInstalledDotNetSdks(customDotNetCliPath); - if (!installedSdks.Any(sdk => sdk.StartsWith(requiredSdkVersion))) - { - yield return new ValidationError(true, $"The required .NET Core SDK version {requiredSdkVersion} for runtime moniker {benchmark.Job.Environment.Runtime.RuntimeMoniker} is not installed.", benchmark); - } + yield return new ValidationError(true, $"The required .NET Core SDK version {requiredSdkVersion} or higher for runtime moniker {benchmark.Job.Environment.Runtime.RuntimeMoniker} is not installed.", benchmark); } } public static IEnumerable ValidateFrameworkSdks(BenchmarkCase benchmark) { - if (!TryGetSdkVersion(benchmark, out string requiredSdkVersionString)) - { - yield break; - } - - if (!Version.TryParse(requiredSdkVersionString, out var requiredSdkVersion)) - { - yield return new ValidationError(true, $"Invalid .NET Framework SDK version format: {requiredSdkVersionString}", benchmark); - yield break; - } - + var requiredSdkVersion = benchmark.GetRuntime().RuntimeMoniker.GetRuntimeVersion(); var installedVersionString = cachedFrameworkSdks.Value.FirstOrDefault(); - if (installedVersionString == null || Version.TryParse(installedVersionString, out var installedVersion) && installedVersion < requiredSdkVersion) { - yield return new ValidationError(true, $"The required .NET Framework SDK version {requiredSdkVersionString} or higher is not installed.", benchmark); + yield return new ValidationError(true, $"The required .NET Framework SDK version {requiredSdkVersion} or higher is not installed.", benchmark); } } @@ -77,18 +65,7 @@ public static bool IsCliPathInvalid(string customDotNetCliPath, BenchmarkCase be return false; } - private static bool TryGetSdkVersion(BenchmarkCase benchmark, out string sdkVersion) - { - sdkVersion = string.Empty; - if (benchmark?.Job?.Environment?.Runtime?.RuntimeMoniker != null) - { - sdkVersion = GetSdkVersionFromMoniker(benchmark.Job.Environment.Runtime.RuntimeMoniker); - return true; - } - return false; - } - - private static IEnumerable GetInstalledDotNetSdks(string? customDotNetCliPath) + private static IEnumerable GetInstalledDotNetSdks(string? customDotNetCliPath) { string dotnetExecutable = string.IsNullOrEmpty(customDotNetCliPath) ? "dotnet" : customDotNetCliPath; var startInfo = new ProcessStartInfo(dotnetExecutable, "--list-sdks") @@ -104,7 +81,7 @@ private static IEnumerable GetInstalledDotNetSdks(string? customDotNetCl { if (process == null) { - return Enumerable.Empty(); + return Enumerable.Empty(); } process.WaitForExit(); @@ -113,17 +90,29 @@ private static IEnumerable GetInstalledDotNetSdks(string? customDotNetCl { var output = process.StandardOutput.ReadToEnd(); var lines = output.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); - return lines.Select(line => line.Split(' ')[0]); // The SDK version is the first part of each line. + + var versions = new List(lines.Count()); + foreach (var line in lines) + { + // Version.TryParse does not handle things like 3.0.0-WORD, so this will get just the 3.0.0 part + var parsableVersionPart = CoreRuntime.GetParsableVersionPart(line); + if (Version.TryParse(parsableVersionPart, out var version)) + { + versions.Add(version); + } + } + + return versions; } else { - return Enumerable.Empty(); + return Enumerable.Empty(); } } } catch (Win32Exception) // dotnet CLI is not installed or not found in the path. { - return Enumerable.Empty(); + return Enumerable.Empty(); } } @@ -192,49 +181,5 @@ private static string CheckFor45PlusVersion(int releaseKey) return ""; } - - private static string GetSdkVersionFromMoniker(RuntimeMoniker runtimeMoniker) - { - return runtimeMoniker switch - { - RuntimeMoniker.Net461 => "4.6.1", - RuntimeMoniker.Net462 => "4.6.2", - RuntimeMoniker.Net47 => "4.7", - RuntimeMoniker.Net471 => "4.7.1", - RuntimeMoniker.Net472 => "4.7.2", - RuntimeMoniker.Net48 => "4.8", - RuntimeMoniker.Net481 => "4.8.1", - RuntimeMoniker.NetCoreApp20 => "2.0", - RuntimeMoniker.NetCoreApp21 => "2.1", - RuntimeMoniker.NetCoreApp22 => "2.2", - RuntimeMoniker.NetCoreApp30 => "3.0", - RuntimeMoniker.NetCoreApp31 => "3.1", - RuntimeMoniker.Net50 => "5.0", - RuntimeMoniker.Net60 => "6.0", - RuntimeMoniker.Net70 => "7.0", - RuntimeMoniker.Net80 => "8.0", - RuntimeMoniker.Net90 => "9.0", - RuntimeMoniker.NativeAot60 => "6.0", - RuntimeMoniker.NativeAot70 => "7.0", - RuntimeMoniker.NativeAot80 => "8.0", - RuntimeMoniker.NativeAot90 => "9.0", - RuntimeMoniker.Mono60 => "6.0", - RuntimeMoniker.Mono70 => "7.0", - RuntimeMoniker.Mono80 => "8.0", - RuntimeMoniker.Mono90 => "9.0", - RuntimeMoniker.Wasm => Portability.RuntimeInformation.IsNetCore && CoreRuntime.TryGetVersion(out var version) ? $"{version.Major}.{version.Minor}" : "5.0", - RuntimeMoniker.WasmNet50 => "5.0", - RuntimeMoniker.WasmNet60 => "6.0", - RuntimeMoniker.WasmNet70 => "7.0", - RuntimeMoniker.WasmNet80 => "8.0", - RuntimeMoniker.WasmNet90 => "9.0", - RuntimeMoniker.MonoAOTLLVM => Portability.RuntimeInformation.IsNetCore && CoreRuntime.TryGetVersion(out var version) ? $"{version.Major}.{version.Minor}" : "6.0", - RuntimeMoniker.MonoAOTLLVMNet60 => "6.0", - RuntimeMoniker.MonoAOTLLVMNet70 => "7.0", - RuntimeMoniker.MonoAOTLLVMNet80 => "8.0", - RuntimeMoniker.MonoAOTLLVMNet90 => "9.0", - _ => throw new NotImplementedException($"SDK version check not implemented for {runtimeMoniker}") - }; - } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Validators/ParamsValidator.cs b/src/BenchmarkDotNet/Validators/ParamsValidator.cs index 5a503148d5..37dc645af5 100644 --- a/src/BenchmarkDotNet/Validators/ParamsValidator.cs +++ b/src/BenchmarkDotNet/Validators/ParamsValidator.cs @@ -9,7 +9,7 @@ namespace BenchmarkDotNet.Validators { public class ParamsValidator : IValidator { - public static readonly ParamsValidator FailOnError = new (); + public static readonly ParamsValidator FailOnError = new(); public bool TreatsWarningsAsErrors => true; diff --git a/src/BenchmarkDotNet/Validators/RuntimeValidator.cs b/src/BenchmarkDotNet/Validators/RuntimeValidator.cs new file mode 100644 index 0000000000..debadf880e --- /dev/null +++ b/src/BenchmarkDotNet/Validators/RuntimeValidator.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Toolchains; + +namespace BenchmarkDotNet.Validators; + +/// +/// Validator for runtime characteristic. +/// +/// +public class RuntimeValidator : IValidator +{ + public static readonly IValidator DontFailOnError = new RuntimeValidator(); + + private RuntimeValidator() { } + + public bool TreatsWarningsAsErrors => false; + + public IEnumerable Validate(ValidationParameters input) + { + var allBenchmarks = input.Benchmarks.ToArray(); + var nullRuntimeBenchmarks = allBenchmarks.Where(x => x.Job.Environment.Runtime == null).ToArray(); + + // There is no validation error if all the runtimes are set or if all the runtimes are null. + if (allBenchmarks.Length == nullRuntimeBenchmarks.Length) + { + return []; + } + + var errors = new List(); + foreach (var benchmark in nullRuntimeBenchmarks.Where(x=> !x.GetToolchain().IsInProcess)) + { + var job = benchmark.Job; + var jobText = job.HasValue(CharacteristicObject.IdCharacteristic) + ? job.Id + : CharacteristicSetPresenter.Display.ToPresentation(job); // Use job text representation instead for auto generated JobId. + + var message = $"Job({jobText}) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; + errors.Add(new ValidationError(false, message)); + } + return errors; + } +} diff --git a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json index 05090c29d1..1d1a4bce6f 100644 --- a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json +++ b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json @@ -34,6 +34,14 @@ "description": "The target framework for the project.", "datatype": "choice", "choices": [ + { + "choice": "net10.0", + "description": ".NET 10" + }, + { + "choice": "net9.0", + "description": ".NET 9" + }, { "choice": "net8.0", "description": ".NET 8" @@ -131,7 +139,7 @@ "type": "parameter", "datatype": "string", "description": "Version of BenchmarkDotNet that will be referenced.", - "defaultValue": "0.14.0", + "defaultValue": "0.15.3", "replaces": "$(BenchmarkDotNetVersion)" } }, @@ -176,4 +184,4 @@ "continueOnError": true } ] -} \ No newline at end of file +} diff --git a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json index b0731534ea..3a242758c0 100644 --- a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json +++ b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json @@ -34,6 +34,14 @@ "description": "The target framework for the project.", "datatype": "choice", "choices": [ + { + "choice": "net10.0", + "description": ".NET 10" + }, + { + "choice": "net9.0", + "description": ".NET 9" + }, { "choice": "net8.0", "description": ".NET 8" @@ -131,7 +139,7 @@ "type": "parameter", "datatype": "string", "description": "Version of BenchmarkDotNet that will be referenced.", - "defaultValue": "0.14.0", + "defaultValue": "0.15.3", "replaces": "$(BenchmarkDotNetVersion)" } }, @@ -176,4 +184,4 @@ "continueOnError": true } ] -} \ No newline at end of file +} diff --git a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json index bf4fc78b1a..42929a3e01 100644 --- a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json +++ b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json @@ -34,6 +34,14 @@ "description": "The target framework for the project.", "datatype": "choice", "choices": [ + { + "choice": "net10.0", + "description": ".NET 10" + }, + { + "choice": "net9.0", + "description": ".NET 9" + }, { "choice": "net8.0", "description": ".NET 8" @@ -131,7 +139,7 @@ "type": "parameter", "datatype": "string", "description": "Version of BenchmarkDotNet that will be referenced.", - "defaultValue": "0.14.0", + "defaultValue": "0.15.3", "replaces": "$(BenchmarkDotNetVersion)" } }, @@ -176,4 +184,4 @@ "continueOnError": true } ] -} \ No newline at end of file +} diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj index 0c61fd72af..6440cd9059 100644 --- a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/BenchmarkDotNet.Exporters.Plotting.Tests.csproj @@ -1,26 +1,26 @@ - - - - net8.0;net462 - false - true - true - false - - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + + net8.0;net462 + false + true + true + false + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs index 32f69061f6..8dd4a4443c 100644 --- a/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs +++ b/tests/BenchmarkDotNet.Exporters.Plotting.Tests/ScottPlotExporterTests.cs @@ -1,246 +1,297 @@ -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Tests.Mocks; -using System; -using System.Diagnostics.CodeAnalysis; -using System.IO; -using System.Linq; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Exporters.Plotting.Tests -{ - public class ScottPlotExporterTests(ITestOutputHelper output) - { - public static TheoryData GetGroupBenchmarkTypes() - { - var data = new TheoryData(); - foreach (var type in typeof(BaselinesBenchmarks).GetNestedTypes()) - data.Add(type); - return data; - } - - [Theory] - [MemberData(nameof(GetGroupBenchmarkTypes))] - public void BarPlots(Type benchmarkType) - { - var logger = new AccumulationLogger(); - logger.WriteLine("=== " + benchmarkType.Name + " ==="); - - var exporter = new ScottPlotExporter(); - var summary = MockFactory.CreateSummary(benchmarkType); - var filePaths = exporter.ExportToFiles(summary, logger).ToList(); - Assert.NotEmpty(filePaths); - Assert.All(filePaths, f => File.Exists(f)); - - foreach (string filePath in filePaths) - logger.WriteLine($"* {filePath}"); - output.WriteLine(logger.GetLog()); - } - - [SuppressMessage("ReSharper", "InconsistentNaming")] - public static class BaselinesBenchmarks - { - /* NoBaseline */ - [RankColumn, LogicalGroupColumn, BaselineColumn] - [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] - public class NoBaseline_MethodsParamsJobs - { - [Params(2, 10)] public int Param; - - [Benchmark] public void Base() { } - [Benchmark] public void Foo() { } - [Benchmark] public void Bar() { } - } - - [RankColumn, LogicalGroupColumn, BaselineColumn] - [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] - [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByMethod)] - public class NoBaseline_MethodsParamsJobs_GroupByMethod - { - [Params(2, 10)] public int Param; - - [Benchmark, BenchmarkCategory("CatA")] public void Base() { } - [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } - [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } - } - - [RankColumn, LogicalGroupColumn, BaselineColumn] - [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] - [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByJob)] - public class NoBaseline_MethodsParamsJobs_GroupByJob - { - [Params(2, 10)] public int Param; - - [Benchmark, BenchmarkCategory("CatA")] public void Base() { } - [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } - [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } - } - - [RankColumn, LogicalGroupColumn, BaselineColumn] - [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] - [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByParams)] - public class NoBaseline_MethodsParamsJobs_GroupByParams - { - [Params(2, 10)] public int Param; - - [Benchmark, BenchmarkCategory("CatA")] public void Base() { } - [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } - [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } - } - - [RankColumn, LogicalGroupColumn, BaselineColumn] - [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] - [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] - public class NoBaseline_MethodsParamsJobs_GroupByCategory - { - [Params(2, 10)] public int Param; - - [Benchmark(Baseline = true), BenchmarkCategory("CatA")] - public void A1() { } - - [Benchmark, BenchmarkCategory("CatA")] public void A2() { } - - [Benchmark(Baseline = true), BenchmarkCategory("CatB")] - public void B1() { } - - [Benchmark, BenchmarkCategory("CatB")] public void B2() { } - } - - [RankColumn, LogicalGroupColumn, BaselineColumn] - [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] - [GroupBenchmarksBy( - BenchmarkLogicalGroupRule.ByMethod, - BenchmarkLogicalGroupRule.ByJob, - BenchmarkLogicalGroupRule.ByParams, - BenchmarkLogicalGroupRule.ByCategory)] - public class NoBaseline_MethodsParamsJobs_GroupByAll - { - [Params(2, 10)] public int Param; - - [Benchmark(Baseline = true), BenchmarkCategory("CatA")] - public void A1() { } - - [Benchmark, BenchmarkCategory("CatA")] public void A2() { } - - [Benchmark(Baseline = true), BenchmarkCategory("CatB")] - public void B1() { } - - [Benchmark, BenchmarkCategory("CatB")] public void B2() { } - } - - /* MethodBaseline */ - - [RankColumn, LogicalGroupColumn, BaselineColumn] - public class MethodBaseline_Methods - { - [Benchmark(Baseline = true)] public void Base() { } - [Benchmark] public void Foo() { } - [Benchmark] public void Bar() { } - } - - [RankColumn, LogicalGroupColumn, BaselineColumn] - public class MethodBaseline_MethodsParams - { - [Params(2, 10)] public int Param; - - [Benchmark(Baseline = true)] public void Base() { } - [Benchmark] public void Foo() { } - [Benchmark] public void Bar() { } - } - - [RankColumn, LogicalGroupColumn, BaselineColumn] - [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] - public class MethodBaseline_MethodsJobs - { - [Benchmark(Baseline = true)] public void Base() { } - [Benchmark] public void Foo() { } - [Benchmark] public void Bar() { } - } - - [RankColumn, LogicalGroupColumn, BaselineColumn] - [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] - public class MethodBaseline_MethodsParamsJobs - { - [Params(2, 10)] public int Param; - - [Benchmark(Baseline = true)] public void Base() { } - [Benchmark] public void Foo() { } - [Benchmark] public void Bar() { } - } - - /* JobBaseline */ - - [RankColumn, LogicalGroupColumn, BaselineColumn] - [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] - public class JobBaseline_MethodsJobs - { - [Benchmark] public void Base() { } - [Benchmark] public void Foo() { } - [Benchmark] public void Bar() { } - } - - [RankColumn, LogicalGroupColumn, BaselineColumn] - [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] - public class JobBaseline_MethodsParamsJobs - { - [Params(2, 10)] public int Param; - - [Benchmark] public void Base() { } - [Benchmark] public void Foo() { } - [Benchmark] public void Bar() { } - } - - /* MethodJobBaseline */ - - [RankColumn, LogicalGroupColumn, BaselineColumn] - [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] - public class MethodJobBaseline_MethodsJobs - { - [Benchmark(Baseline = true)] public void Foo() { } - [Benchmark] public void Bar() { } - } - - [RankColumn, LogicalGroupColumn, BaselineColumn] - [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] - public class MethodJobBaseline_MethodsJobsParams - { - [Params(2, 10)] public int Param; - - [Benchmark(Baseline = true)] public void Foo() { } - [Benchmark] public void Bar() { } - } - - /* Invalid */ - - [RankColumn, LogicalGroupColumn, BaselineColumn] - public class Invalid_TwoMethodBaselines - { - [Benchmark(Baseline = true)] public void Foo() { } - [Benchmark(Baseline = true)] public void Bar() { } - } - - [RankColumn, LogicalGroupColumn, BaselineColumn] - [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2", baseline: true)] - public class Invalid_TwoJobBaselines - { - [Benchmark] public void Foo() { } - [Benchmark] public void Bar() { } - } - - /* Escape Params */ - - public class Escape_ParamsAndArguments - { - [Params("\t", "\n")] public string StringParam; - - [Arguments('\t')] - [Arguments('\n')] - [Benchmark] public void Foo(char charArg) { } - - [Benchmark] public void Bar() { } - } - } - } +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Mocks; +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Exporters.Plotting.Tests +{ + public class ScottPlotExporterTests(ITestOutputHelper output) + { + public static TheoryData GetGroupBenchmarkTypes() + { + var data = new TheoryData(); + foreach (var type in typeof(BaselinesBenchmarks).GetNestedTypes()) + data.Add(type); + return data; + } + + [Theory] + [MemberData(nameof(GetGroupBenchmarkTypes))] + public void BarPlots(Type benchmarkType) + { + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + benchmarkType.Name + " ==="); + + var exporter = new ScottPlotExporter() + { + IncludeBarPlot = true, + IncludeBoxPlot = false, + }; + var summary = MockFactory.CreateSummary(benchmarkType); + var filePaths = exporter.ExportToFiles(summary, logger).ToList(); + Assert.NotEmpty(filePaths); + Assert.All(filePaths, f => File.Exists(f)); + + foreach (string filePath in filePaths) + logger.WriteLine($"* {filePath}"); + output.WriteLine(logger.GetLog()); + } + + [Theory] + [MemberData(nameof(GetGroupBenchmarkTypes))] + public void BoxPlots(Type benchmarkType) + { + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + benchmarkType.Name + " ==="); + + var exporter = new ScottPlotExporter() + { + IncludeBarPlot = false, + IncludeBoxPlot = true, + }; + var summary = MockFactory.CreateSummaryWithBiasedDistribution(benchmarkType, 1, 4, 10, 9); + var filePaths = exporter.ExportToFiles(summary, logger).ToList(); + Assert.NotEmpty(filePaths); + Assert.All(filePaths, f => File.Exists(f)); + + foreach (string filePath in filePaths) + logger.WriteLine($"* {filePath}"); + output.WriteLine(logger.GetLog()); + } + + [Theory] + [MemberData(nameof(GetGroupBenchmarkTypes))] + public void BoxPlotsWithOneMeasurement(Type benchmarkType) + { + var logger = new AccumulationLogger(); + logger.WriteLine("=== " + benchmarkType.Name + " ==="); + + var exporter = new ScottPlotExporter() + { + IncludeBarPlot = false, + IncludeBoxPlot = true, + }; + var summary = MockFactory.CreateSummaryWithBiasedDistribution(benchmarkType, 1, 4, 10, 1); + var filePaths = exporter.ExportToFiles(summary, logger).ToList(); + Assert.NotEmpty(filePaths); + Assert.All(filePaths, f => File.Exists(f)); + + foreach (string filePath in filePaths) + logger.WriteLine($"* {filePath}"); + output.WriteLine(logger.GetLog()); + } + + [SuppressMessage("ReSharper", "InconsistentNaming")] + public static class BaselinesBenchmarks + { + /* NoBaseline */ + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + public class NoBaseline_MethodsParamsJobs + { + [Params(2, 10)] public int Param; + + [Benchmark] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByMethod)] + public class NoBaseline_MethodsParamsJobs_GroupByMethod + { + [Params(2, 10)] public int Param; + + [Benchmark, BenchmarkCategory("CatA")] public void Base() { } + [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } + [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByJob)] + public class NoBaseline_MethodsParamsJobs_GroupByJob + { + [Params(2, 10)] public int Param; + + [Benchmark, BenchmarkCategory("CatA")] public void Base() { } + [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } + [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByParams)] + public class NoBaseline_MethodsParamsJobs_GroupByParams + { + [Params(2, 10)] public int Param; + + [Benchmark, BenchmarkCategory("CatA")] public void Base() { } + [Benchmark, BenchmarkCategory("CatB")] public void Foo() { } + [Benchmark, BenchmarkCategory("CatB")] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] + public class NoBaseline_MethodsParamsJobs_GroupByCategory + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true), BenchmarkCategory("CatA")] + public void A1() { } + + [Benchmark, BenchmarkCategory("CatA")] public void A2() { } + + [Benchmark(Baseline = true), BenchmarkCategory("CatB")] + public void B1() { } + + [Benchmark, BenchmarkCategory("CatB")] public void B2() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + [GroupBenchmarksBy( + BenchmarkLogicalGroupRule.ByMethod, + BenchmarkLogicalGroupRule.ByJob, + BenchmarkLogicalGroupRule.ByParams, + BenchmarkLogicalGroupRule.ByCategory)] + public class NoBaseline_MethodsParamsJobs_GroupByAll + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true), BenchmarkCategory("CatA")] + public void A1() { } + + [Benchmark, BenchmarkCategory("CatA")] public void A2() { } + + [Benchmark(Baseline = true), BenchmarkCategory("CatB")] + public void B1() { } + + [Benchmark, BenchmarkCategory("CatB")] public void B2() { } + } + + /* MethodBaseline */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + public class MethodBaseline_Methods + { + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + public class MethodBaseline_MethodsParams + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + public class MethodBaseline_MethodsJobs + { + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1"), SimpleJob(id: "Job2")] + public class MethodBaseline_MethodsParamsJobs + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true)] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* JobBaseline */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class JobBaseline_MethodsJobs + { + [Benchmark] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class JobBaseline_MethodsParamsJobs + { + [Params(2, 10)] public int Param; + + [Benchmark] public void Base() { } + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* MethodJobBaseline */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class MethodJobBaseline_MethodsJobs + { + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] + public class MethodJobBaseline_MethodsJobsParams + { + [Params(2, 10)] public int Param; + + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* Invalid */ + + [RankColumn, LogicalGroupColumn, BaselineColumn] + public class Invalid_TwoMethodBaselines + { + [Benchmark(Baseline = true)] public void Foo() { } + [Benchmark(Baseline = true)] public void Bar() { } + } + + [RankColumn, LogicalGroupColumn, BaselineColumn] + [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2", baseline: true)] + public class Invalid_TwoJobBaselines + { + [Benchmark] public void Foo() { } + [Benchmark] public void Bar() { } + } + + /* Escape Params */ + + public class Escape_ParamsAndArguments + { + [Params("\t", "\n")] public string StringParam; + + [Arguments('\t')] + [Arguments('\n')] + [Benchmark] public void Foo(char charArg) { } + + [Benchmark] public void Bar() { } + } + } + } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj b/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj index 1f1c48deec..e78f71c447 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly/BenchmarkDotNet.IntegrationTests.ConfigPerAssembly.csproj @@ -14,7 +14,7 @@ - + \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.CustomPaths/BenchmarkDotNet.IntegrationTests.CustomPaths.csproj b/tests/BenchmarkDotNet.IntegrationTests.CustomPaths/BenchmarkDotNet.IntegrationTests.CustomPaths.csproj index c7f0cb3ae1..326690a5b6 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.CustomPaths/BenchmarkDotNet.IntegrationTests.CustomPaths.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.CustomPaths/BenchmarkDotNet.IntegrationTests.CustomPaths.csproj @@ -14,7 +14,7 @@ - + diff --git a/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj b/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj index 754b32efcf..6ceec4010a 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.DisabledOptimizations/BenchmarkDotNet.IntegrationTests.DisabledOptimizations.csproj @@ -16,7 +16,7 @@ - + \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj b/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj index 6a12d2c02c..28bf3ef4f9 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.EnabledOptimizations/BenchmarkDotNet.IntegrationTests.EnabledOptimizations.csproj @@ -16,7 +16,7 @@ - + \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.FSharp/BenchmarkDotNet.IntegrationTests.FSharp.fsproj b/tests/BenchmarkDotNet.IntegrationTests.FSharp/BenchmarkDotNet.IntegrationTests.FSharp.fsproj index f3a86d5579..505a549daf 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.FSharp/BenchmarkDotNet.IntegrationTests.FSharp.fsproj +++ b/tests/BenchmarkDotNet.IntegrationTests.FSharp/BenchmarkDotNet.IntegrationTests.FSharp.fsproj @@ -15,7 +15,7 @@ - - + + \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj index 8c8374eaf0..37dfe950f7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj @@ -3,17 +3,12 @@ BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks + net461;net48;netcoreapp2.0;net8.0 + + $(NoWarn);NETSDK1138;NU1903 + + $(NoWarn);NU1510 false - - - - - - - - - - net462;net48;net8.0 true BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks @@ -22,6 +17,8 @@ AnyCPU true true + + true @@ -32,8 +29,7 @@ - - + @@ -46,9 +42,10 @@ - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs index 0e9eb6f80d..b9cf360c8f 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs @@ -3,14 +3,10 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Portability; using Xunit; namespace BenchmarkDotNet.IntegrationTests.ManualRunning { - // Note: To properly test this locally, modify - // BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj, - // following the comments in that file. public class MultipleFrameworksTest : BenchmarkTestExecutor { private const string TfmEnvVarName = "TfmEnvVarName"; @@ -60,7 +56,7 @@ public void ThrowWhenWrong() { if (Environment.GetEnvironmentVariable(TfmEnvVarName) != moniker.ToString()) { - throw new InvalidOperationException($"Has not been recompiled, the value was {Environment.GetEnvironmentVariable(TfmEnvVarName)}"); + throw new InvalidOperationException($"Has not been recompiled, the value was {moniker}, expected {Environment.GetEnvironmentVariable(TfmEnvVarName)}"); } } } diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj index b3fef215b4..3e1859a72c 100755 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj @@ -1,4 +1,4 @@ - + BenchmarkDotNet.IntegrationTests.ManualRunning @@ -34,9 +34,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs index 7f950136fb..29bf8275dd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Extensions; @@ -110,7 +111,7 @@ private void AssertZeroResults(Type benchmarkType, IConfig config) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); - var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; + var cpuResolution = CpuDetector.Cpu?.MaxFrequency()?.ToResolution() ?? FallbackCpuResolutionValue; var threshold = new NumberValue(cpuResolution.Nanoseconds).ToThreshold(); foreach (var report in summary.Reports) @@ -163,7 +164,7 @@ private void AssertDifferentSizedStructsResults(IConfig config) .AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false))) ); - var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue; + var cpuResolution = CpuDetector.Cpu?.MaxFrequency()?.ToResolution() ?? FallbackCpuResolutionValue; var threshold = (cpuResolution / 2).ToThreshold(); foreach (var report in summary.Reports) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalNativeAotToolchainTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalNativeAotToolchainTests.cs index c8740a3829..64f469efc7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalNativeAotToolchainTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/LocalNativeAotToolchainTests.cs @@ -13,7 +13,7 @@ namespace BenchmarkDotNet.IntegrationTests.ManualRunning /// to run these tests please clone and build NativeAOT first, /// then update the hardcoded path /// and run following command from console: - /// dotnet test -c Release -f netcoreapp2.1 --filter "FullyQualifiedName~BenchmarkDotNet.IntegrationTests.ManualRunning.LocalNativeAotToolchainTests" + /// dotnet test -c Release -f net8.0 --filter "FullyQualifiedName~BenchmarkDotNet.IntegrationTests.ManualRunning.LocalNativeAotToolchainTests" /// /// in perfect world we would do this OOB for you, but building NativeAOT /// so it's not part of our CI jobs @@ -29,7 +29,7 @@ public void CanBenchmarkLocalBuildUsingRyuJit() { var config = ManualConfig.CreateEmpty() .AddJob(Job.Dry - .WithRuntime(NativeAotRuntime.Net60) + .WithRuntime(NativeAotRuntime.Net80) .WithToolchain( NativeAotToolchain.CreateBuilder() .UseLocalBuild(new System.IO.DirectoryInfo(IlcPath)) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeEmpty.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeEmpty.cs deleted file mode 100644 index f8607e4263..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeEmpty.cs +++ /dev/null @@ -1,85 +0,0 @@ -using BenchmarkDotNet.Attributes; - -namespace BenchmarkDotNet.IntegrationTests.ManualRunning.Smoke -{ - [MedianColumn, Q3Column, MaxColumn] - [LegacyJitX64Job, RyuJitX64Job, MonoJob] - [KeepBenchmarkFiles] - public class SmokeEmpty - { - [Benchmark] public void Void1() {} - [Benchmark] public void Void2() {} - [Benchmark] public void Void3() {} - [Benchmark] public void Void4() {} - - [Benchmark] public byte Byte1() => 0; - [Benchmark] public byte Byte2() => 0; - [Benchmark] public byte Byte3() => 0; - [Benchmark] public byte Byte4() => 0; - - [Benchmark] public sbyte Sbyte1() => 0; - [Benchmark] public sbyte Sbyte2() => 0; - [Benchmark] public sbyte Sbyte3() => 0; - [Benchmark] public sbyte Sbyte4() => 0; - - [Benchmark] public short Short1() => 0; - [Benchmark] public short Short2() => 0; - [Benchmark] public short Short3() => 0; - [Benchmark] public short Short4() => 0; - - [Benchmark] public ushort Ushort1() => 0; - [Benchmark] public ushort Ushort2() => 0; - [Benchmark] public ushort Ushort3() => 0; - [Benchmark] public ushort Ushort4() => 0; - - [Benchmark] public int Int1() => 0; - [Benchmark] public int Int2() => 0; - [Benchmark] public int Int3() => 0; - [Benchmark] public int Int4() => 0; - - [Benchmark] public uint Uint1() => 0u; - [Benchmark] public uint Uint2() => 0u; - [Benchmark] public uint Uint3() => 0u; - [Benchmark] public uint Uint4() => 0u; - - [Benchmark] public bool Bool1() => false; - [Benchmark] public bool Bool2() => false; - [Benchmark] public bool Bool3() => false; - [Benchmark] public bool Bool4() => false; - - [Benchmark] public char Char1() => 'a'; - [Benchmark] public char Char2() => 'a'; - [Benchmark] public char Char3() => 'a'; - [Benchmark] public char Char4() => 'a'; - - [Benchmark] public float Float1() => 0f; - [Benchmark] public float Float2() => 0f; - [Benchmark] public float Float3() => 0f; - [Benchmark] public float Float4() => 0f; - - [Benchmark] public double Double1() => 0d; - [Benchmark] public double Double2() => 0d; - [Benchmark] public double Double3() => 0d; - [Benchmark] public double Double4() => 0d; - - [Benchmark] public long Long1() => 0L; - [Benchmark] public long Long2() => 0L; - [Benchmark] public long Long3() => 0L; - [Benchmark] public long Long4() => 0L; - - [Benchmark] public ulong Ulong1() => 0uL; - [Benchmark] public ulong Ulong2() => 0uL; - [Benchmark] public ulong Ulong3() => 0uL; - [Benchmark] public ulong Ulong4() => 0uL; - - [Benchmark] public string String1() => ""; - [Benchmark] public string String2() => ""; - [Benchmark] public string String3() => ""; - [Benchmark] public string String4() => ""; - - [Benchmark] public object Object1() => null; - [Benchmark] public object Object2() => null; - [Benchmark] public object Object3() => null; - [Benchmark] public object Object4() => null; - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeValueTypes.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeValueTypes.cs deleted file mode 100644 index 34b93a6d75..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/Smoke/SmokeValueTypes.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System; -using System.Threading.Tasks; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Environments; - -namespace BenchmarkDotNet.IntegrationTests.ManualRunning.Smoke -{ - // ReSharper disable InconsistentNaming - - [RyuJitX64Job, LegacyJitX64Job, LegacyJitX86Job] - [MemoryDiagnoser] - public class SmokeValueTypes - { - [Benchmark] public Jit ReturnEnum() => Jit.RyuJit; - - [Benchmark] public DateTime ReturnDateTime() => new DateTime(); - - [Benchmark] public DateTime? ReturnNullableDateTime() => new DateTime(); - [Benchmark] public int? ReturnNullableInt() => 0; - - public struct StructWithReferencesOnly { public object _ref; } - [Benchmark] public StructWithReferencesOnly ReturnStructWithReferencesOnly() => new StructWithReferencesOnly(); - - public struct EmptyStruct { } - [Benchmark] public EmptyStruct ReturnEmptyStruct() => new EmptyStruct(); - - [Benchmark] public ValueTuple ReturnGenericStructOfValueType() => new ValueTuple(0); - [Benchmark] public ValueTuple ReturnGenericStructOfReferenceType() => new ValueTuple(null); - - [Benchmark] public ValueTask ReturnValueTaskOfValueType() => new ValueTask(0); - [Benchmark] public ValueTask ReturnValueTaskOfReferenceType() => new ValueTask(result: null); - - [Benchmark] public byte ReturnByte() => 0; - public struct Byte1 { public byte _1; } - [Benchmark] public Byte1 ReturnByte1() => new Byte1(); - public struct Byte2 { public byte _1, _2; } - [Benchmark] public Byte2 ReturnByte2() => new Byte2(); - public struct Byte3 { public byte _1, _2, _3; } - [Benchmark] public Byte3 ReturnByte3() => new Byte3(); - public struct Byte4 { public byte _1, _2, _3, _4; } - [Benchmark] public Byte4 ReturnByte4() => new Byte4(); - - [Benchmark] public short ReturnShort() => 0; - public struct Short1 { public short _1; } - [Benchmark] public Short1 ReturnShort1() => new Short1(); - public struct Short2 { public short _1, _2; } - [Benchmark] public Short2 ReturnShort2() => new Short2(); - public struct Short3 { public short _1, _2, _3; } - [Benchmark] public Short3 ReturnShort3() => new Short3(); - public struct Short4 { public short _1, _2, _3, _4; } - [Benchmark] public Short4 ReturnShort4() => new Short4(); - - [Benchmark] public int ReturnInt() => 0; - public struct Int1 { public int _1; } - [Benchmark] public Int1 ReturnInt1() => new Int1(); - public struct Int2 { public int _1, _2; } - [Benchmark] public Int2 ReturnInt2() => new Int2(); - public struct Int3 { public int _1, _2, _3; } - [Benchmark] public Int3 ReturnInt3() => new Int3(); - public struct Int4 { public int _1, _2, _3, _4; } - [Benchmark] public Int4 ReturnInt4() => new Int4(); - - [Benchmark] public long ReturnLong() => 0; - public struct Long1 { public long _1; } - [Benchmark] public Long1 ReturnLong1() => new Long1(); - public struct Long2 { public long _1, _2; } - [Benchmark] public Long2 ReturnLong2() => new Long2(); - public struct Long3 { public long _1, _2, _3; } - [Benchmark] public Long3 ReturnLong3() => new Long3(); - public struct Long4 { public long _1, _2, _3, _4; } - [Benchmark] public Long4 ReturnLong4() => new Long4(); - } - // ReSharper restore InconsistentNaming -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests.Static/BenchmarkDotNet.IntegrationTests.Static.csproj b/tests/BenchmarkDotNet.IntegrationTests.Static/BenchmarkDotNet.IntegrationTests.Static.csproj index 0944134348..3e06189dc7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.Static/BenchmarkDotNet.IntegrationTests.Static.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.Static/BenchmarkDotNet.IntegrationTests.Static.csproj @@ -14,7 +14,7 @@ - + \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs index 459c23c2c2..36ea5688ad 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs @@ -196,7 +196,7 @@ public async ValueTask GlobalCleanup() public class AllSetupAndCleanupAttributeBenchmarksValueTaskSource { - private readonly ValueTaskSource valueTaskSource = new (); + private readonly ValueTaskSource valueTaskSource = new(); private int setupCounter; private int cleanupCounter; @@ -228,7 +228,7 @@ public ValueTask GlobalCleanup() public class AllSetupAndCleanupAttributeBenchmarksGenericValueTaskSource { - private readonly ValueTaskSource valueTaskSource = new (); + private readonly ValueTaskSource valueTaskSource = new(); private int setupCounter; private int cleanupCounter; diff --git a/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs index 8c5453a247..021ff5d96d 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs @@ -1,13 +1,13 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Numerics; -using System.Threading.Tasks; -using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.InProcess.Emit; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Numerics; +using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; @@ -24,7 +24,7 @@ public static IEnumerable GetToolchains() public ArgumentsTests(ITestOutputHelper output) : base(output) { } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArgumentsArePassedToBenchmarks(IToolchain toolchain) => CanExecute(toolchain); public class WithArguments @@ -61,7 +61,7 @@ public ValueTask SimpleValueTaskAsync(bool boolean, int number) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArgumentsFromSourceArePassedToBenchmarks(IToolchain toolchain) => CanExecute(toolchain); public class WithArgumentsSource @@ -81,7 +81,94 @@ public IEnumerable ArgumentsProvider() } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + public void ArgumentsFromSourceInAnotherClassArePassedToBenchmarks(IToolchain toolchain) => CanExecute(toolchain); + + public class WithArgumentsSourceInAnotherClass + { + [Benchmark] + [ArgumentsSource(typeof(ExternalClassWithArgumentsSource), nameof(ExternalClassWithArgumentsSource.OnePrimitiveType))] + public void OnePrimitiveType(int number) + { + if (number % 2 != 1) + throw new InvalidOperationException("Incorrect values were passed"); + } + + [Benchmark] + [ArgumentsSource(typeof(ExternalClassWithArgumentsSource), nameof(ExternalClassWithArgumentsSource.TwoPrimitiveTypes))] + public void TwoPrimitiveTypes(bool boolean, int number) + { + if (boolean && number != 1 || !boolean && number != 2) + throw new InvalidOperationException("Incorrect values were passed"); + } + + [Benchmark] + [ArgumentsSource(typeof(ExternalClassWithArgumentsSource), nameof(ExternalClassWithArgumentsSource.OneNonPrimitiveType))] + public void OneNonPrimitiveType(Version version) + { + int[] versionNumbers = { version.Major, version.Minor, version.MinorRevision, version.Build }; + if (versionNumbers.Distinct().Count() != 4) + throw new InvalidOperationException("Incorrect values were passed"); + } + + [Benchmark] + [ArgumentsSource(typeof(ExternalClassWithArgumentsSource), nameof(ExternalClassWithArgumentsSource.TwoNonPrimitiveTypes))] + public void TwoNonPrimitiveTypes(Version version, DateTime dateTime) + { + int[] versionNumbers = { version.Major, version.Minor, version.MinorRevision, version.Build }; + if (versionNumbers.Distinct().Count() != 4) + throw new InvalidOperationException("Incorrect values were passed"); + + if (dateTime.Month != dateTime.Day) + throw new InvalidOperationException("Incorrect values were passed"); + } + + [Benchmark] + [ArgumentsSource(typeof(ExternalClassWithArgumentsSource), nameof(ExternalClassWithArgumentsSource.OnePrimitiveAndOneNonPrimitive))] + public void OnePrimitiveAndOneNonPrimitive(Version version, int number) + { + int[] versionNumbers = { version.Major, version.Minor, version.MinorRevision, version.Build }; + if (versionNumbers.Distinct().Count() != 4) + throw new InvalidOperationException("Incorrect values were passed"); + + if (number != version.Major) + throw new InvalidOperationException("Incorrect values were passed"); + } + } + public static class ExternalClassWithArgumentsSource + { + public static IEnumerable OnePrimitiveType() + { + yield return 3; + yield return 5; + } + + public static IEnumerable TwoPrimitiveTypes() + { + yield return new object[] { true, 1 }; + yield return new object[] { false, 2 }; + } + + public static IEnumerable OneNonPrimitiveType() + { + yield return new Version(1, 2, 3, 4); + yield return new Version(5, 6, 7, 8); + } + + public static IEnumerable TwoNonPrimitiveTypes() + { + yield return new object[] { new Version(1, 2, 3, 4), new DateTime(2011, 11, 11) }; + yield return new object[] { new Version(5, 6, 7, 8), new DateTime(2002, 02, 02) }; + } + + public static IEnumerable OnePrimitiveAndOneNonPrimitive() + { + yield return new object[] { new Version(1, 2, 3, 4), 1 }; + yield return new object[] { new Version(5, 6, 7, 8), 5 }; + } + } + + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArgumentsCanBePassedByReferenceToBenchmark(IToolchain toolchain) => CanExecute(toolchain); public class WithRefArguments @@ -96,7 +183,7 @@ public void Simple(ref bool boolean, ref int number) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArgumentsCanBePassedByReadonlyReferenceToBenchmark(IToolchain toolchain) => CanExecute(toolchain); public class WithInArguments @@ -111,7 +198,7 @@ public void Simple(in bool boolean, in int number) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void NonCompileTimeConstantsCanBeReturnedFromSource(IToolchain toolchain) => CanExecute(toolchain); public class WithComplexTypesReturnedFromSources @@ -168,7 +255,7 @@ public struct SomeStruct } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArrayCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithArray @@ -186,7 +273,7 @@ public void AcceptingArray(int[] array) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void IEnumerableCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithIEnumerable @@ -211,7 +298,7 @@ public IEnumerable Sources() public void Any(string name, IEnumerable source) => source.Any(); } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void JaggedArrayCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithJaggedArray @@ -247,7 +334,7 @@ public IEnumerable CreateMatrix() } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void GenericTypeCanBePassedByRefAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithGenericByRef @@ -283,7 +370,7 @@ public IEnumerable GetInputData() } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void AnArrayOfTypeWithNoParameterlessCtorCanBePassedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithArrayOfStringAsArgument @@ -300,7 +387,7 @@ public void TypeReflectionArrayGetType(object anArray) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void AnArrayCanBePassedToBenchmarkAsSpan(IToolchain toolchain) => CanExecute(toolchain); public class WithArrayToSpan @@ -320,7 +407,7 @@ public void AcceptsSpan(Span span) [TheoryEnvSpecific("The implicit cast operator is available only in .NET Core 2.1+ (See https://github.com/dotnet/corefx/issues/30121 for more)", EnvRequirement.DotNetCoreOnly)] - [MemberData(nameof(GetToolchains))] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void StringCanBePassedToBenchmarkAsReadOnlySpan(IToolchain toolchain) => CanExecute(toolchain); public class WithStringToReadOnlySpan @@ -338,7 +425,7 @@ public void AcceptsReadOnlySpan(ReadOnlySpan notString) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void AnArrayOfStringsCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); @@ -358,7 +445,7 @@ public void TypeReflectionArrayGetType(string[] array) } } - [Theory, MemberData(nameof(GetToolchains))] // make sure BDN mimics xunit's MemberData behaviour + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] // make sure BDN mimics xunit's MemberData behaviour public void AnIEnumerableOfArrayOfObjectsCanBeUsedAsArgumentForBenchmarkAcceptingSingleArgument(IToolchain toolchain) => CanExecute(toolchain); @@ -378,7 +465,7 @@ public void SingleArgument(bool boolean) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void BenchmarkCanAcceptFewArrays(IToolchain toolchain) => CanExecute(toolchain); public class FewArrays @@ -407,7 +494,7 @@ public void AcceptsArrays(int[] even, int[] notEven) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void VeryBigIntegersAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithVeryBigInteger @@ -428,7 +515,7 @@ public void Method(BigInteger passed) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void SpecialDoubleValuesAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithSpecialDoubleValues @@ -473,7 +560,7 @@ public void Method(double passed, string name) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void SpecialFloatValuesAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithSpecialFloatValues @@ -518,7 +605,7 @@ public void Method(float passed, string name) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void SpecialDecimalValuesAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithSpecialDecimalValues @@ -547,7 +634,7 @@ public void Method(decimal passed, string name) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void DateTimeCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithDateTime @@ -568,7 +655,7 @@ public void Test(DateTime passed) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void CustomTypeThatAlsoExistsInTheSystemNamespaceAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class CustomTypeThatAlsoExistsInTheSystemNamespace @@ -589,7 +676,7 @@ public void Test(Action passed) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void EnumFlagsAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithEnumFlags @@ -626,7 +713,7 @@ public void Test(LongFlagEnum passedLongFlagEnum, ByteFlagEnum passedByteFlagEnu } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void UndefinedEnumValuesAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithUndefinedEnumValue @@ -652,7 +739,7 @@ public void Test(SomeEnum defined, SomeEnum undefined, SomeEnum undefinedNegativ } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void StaticMethodsAndPropertiesCanBeUsedAsSources_EnumerableOfObjects(IToolchain toolchain) => CanExecute(toolchain); @@ -700,7 +787,7 @@ public void TestProperty(int argument) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void StaticMethodsAndPropertiesCanBeUsedAsSources_EnumerableOfArrayOfObjects(IToolchain toolchain) => CanExecute(toolchain); @@ -747,7 +834,64 @@ public void TestProperty(int argument) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + public void MethodsAndPropertiesFromAnotherClassCanBeUsedAsSources(IToolchain toolchain) + => CanExecute(toolchain); + + public class ParamsSourcePointingToAnotherClass + { + [ParamsSource(typeof(ExternalClassWithParamsSource), nameof(ExternalClassWithParamsSource.PrimitiveTypeMethod))] + public int ParamOne { get; set; } + + [ParamsSource(typeof(ExternalClassWithParamsSource), nameof(ExternalClassWithParamsSource.PrimitiveTypeProperty))] + public int ParamTwo { get; set; } + + [ParamsSource(typeof(ExternalClassWithParamsSource), nameof(ExternalClassWithParamsSource.NonPrimitiveTypeMethod))] + public Version ParamThree { get; set; } + + [ParamsSource(typeof(ExternalClassWithParamsSource), nameof(ExternalClassWithParamsSource.NonPrimitiveTypeProperty))] + public Version ParamFour { get; set; } + + [Benchmark] + public void Test() + { + if (ParamOne != 123) + throw new ArgumentException("The ParamOne value is incorrect!"); + if (ParamTwo != 456) + throw new ArgumentException("The ParamTwo value is incorrect!"); + if (ParamThree != new Version(1, 2, 3, 4)) + throw new ArgumentException("The ParamThree value is incorrect!"); + if (ParamFour != new Version(5, 6, 7, 8)) + throw new ArgumentException("The ParamFour value is incorrect!"); + } + } + public static class ExternalClassWithParamsSource + { + public static IEnumerable PrimitiveTypeMethod() + { + yield return 123; + } + public static IEnumerable PrimitiveTypeProperty + { + get + { + yield return 456; + } + } + public static IEnumerable NonPrimitiveTypeMethod() + { + yield return new Version(1, 2, 3, 4); + } + public static IEnumerable NonPrimitiveTypeProperty + { + get + { + yield return new Version(5, 6, 7, 8); + } + } + } + + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void VeryLongStringsAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithVeryLongString @@ -771,7 +915,7 @@ public void Test(string first, string second) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ComplexStringPattersAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class Perf_Regex_Industry_RustLang_Sherlock diff --git a/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs b/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs index d795b1a102..a2158362ef 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs @@ -63,7 +63,7 @@ public void TaskReturningMethodsAreAwaited() public class TaskDelayMethods { - private readonly ValueTaskSource valueTaskSource = new (); + private readonly ValueTaskSource valueTaskSource = new(); private const int MillisecondsDelay = 100; @@ -112,8 +112,8 @@ public ValueTask ReturningGenericValueTaskBackByIValueTaskSource() public class TaskImmediateMethods { - private readonly ValueTaskSource valueTaskSource = new (); - private readonly ValueTaskSourceCallbackOnly valueTaskSourceCallbackOnly = new (); + private readonly ValueTaskSource valueTaskSource = new(); + private readonly ValueTaskSourceCallbackOnly valueTaskSourceCallbackOnly = new(); [Benchmark] public Task ReturningTask() => Task.CompletedTask; diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj index a928af1424..46e9b0edac 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj @@ -18,6 +18,8 @@ Always + + @@ -32,24 +34,26 @@ - - - - - + + + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + - + - - + + diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs index e9a95fce81..622aee3ffd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkSwitcherTest.cs @@ -66,7 +66,7 @@ public void WhenInvalidTypeIsProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAre .Run(new[] { "--filter", "*" }, config); Assert.Empty(summaries); - Assert.Contains("Type BenchmarkDotNet.IntegrationTests.ClassC is invalid.", logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(ClassC)), logger.GetLog()); } [Fact] @@ -80,7 +80,7 @@ public void WhenNoTypesAreProvidedAnErrorMessageIsDisplayedAndNoBenchmarksAreExe .Run(new[] { "--filter", "*" }, config); Assert.Empty(summaries); - Assert.Contains("No benchmarks to choose from. Make sure you provided public non-sealed non-static types with public [Benchmark] methods.", logger.GetLog()); + Assert.Contains("No benchmarks were found.", logger.GetLog()); } [Fact] @@ -369,6 +369,11 @@ public IReadOnlyList AskUser(IReadOnlyList allTypes, ILogger logger) return returnValue; } } + + private string GetValidationErrorForType(Type type) + { + return $"No [Benchmark] attribute found on '{type.Name}' benchmark case."; + } } } @@ -424,6 +429,8 @@ public override void ExportToLog(Summary summary, ILogger logger) exported = true; } } + + } namespace BenchmarkDotNet.NOTIntegrationTests diff --git a/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs b/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs index e672bcc906..101fa50139 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs @@ -1,6 +1,7 @@ using System; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -19,7 +20,7 @@ public void WhenBuildTakesMoreTimeThanTheTimeoutTheBuildIsCancelled() { if (!RuntimeInformation.Is64BitPlatform()) // NativeAOT does not support 32bit yet return; - if (RuntimeInformation.IsMacOS()) + if (OsDetector.IsMacOS()) return; // currently not supported // we use NativeAOT on purpose because it takes a LOT of time to build it diff --git a/tests/BenchmarkDotNet.IntegrationTests/ContinuousIntegration.cs b/tests/BenchmarkDotNet.IntegrationTests/ContinuousIntegration.cs index 0f112aa328..8b8710efe5 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ContinuousIntegration.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ContinuousIntegration.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet.Portability; using System; +using BenchmarkDotNet.Detectors; namespace BenchmarkDotNet.IntegrationTests { @@ -8,7 +9,7 @@ internal static class ContinuousIntegration private static bool IsGitHubActions() => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTION")); internal static bool IsGitHubActionsOnWindows() - => RuntimeInformation.IsWindows() && IsGitHubActions(); + => OsDetector.IsWindows() && IsGitHubActions(); internal static bool IsLocalRun() => !IsGitHubActions(); } diff --git a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs index 43a7bac9ee..f6d0a10f83 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs @@ -74,8 +74,8 @@ public RunResults Run() return new RunResults( new List { - new (1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1), - new (1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1) + new(1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1), + new(1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1) }, OutlierMode.DontRemove, default, diff --git a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs index bd2f6d48c2..488d76ca31 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Disassemblers; using BenchmarkDotNet.Engines; @@ -36,12 +37,12 @@ public static IEnumerable GetAllJits() { yield return new object[] { Jit.RyuJit, Platform.X64, CoreRuntime.Core80 }; // .NET Core x64 } - else if (RuntimeInformation.GetCurrentPlatform() is Platform.Arm64 && RuntimeInformation.IsLinux()) + else if (RuntimeInformation.GetCurrentPlatform() is Platform.Arm64 && OsDetector.IsLinux()) { yield return new object[] { Jit.RyuJit, Platform.Arm64, CoreRuntime.Core80 }; // .NET Core arm64 } } - if (RuntimeInformation.IsMacOS()) + if (OsDetector.IsMacOS()) { // This scope of tests is not supported on macOS // However, when the MemberData method provides no data, xUnit throws an "No data found" InvalidOperationException @@ -87,41 +88,49 @@ public void Recursive() } [Theory] - [MemberData(nameof(GetAllJits))] + [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, Runtime runtime) { - if (RuntimeInformation.IsMacOS()) return; // currently not supported + if (OsDetector.IsMacOS()) return; // currently not supported + var printSource = IsPrintSourceSupported(platform); var disassemblyDiagnoser = new DisassemblyDiagnoser( - new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); + new DisassemblyDiagnoserConfig(printSource: printSource, maxDepth: 3)); CanExecute(CreateConfig(jit, platform, runtime, disassemblyDiagnoser, RunStrategy.ColdStart)); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Benchmark)}(Int32)"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Benchmark)}(Boolean)"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Static)}()"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Instance)}()"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Recursive)}()"); + DisassemblyResult result = disassemblyDiagnoser.Results.Single().Value; + + Assert.Empty(result.Errors); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Benchmark)}(Int32)"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Benchmark)}(Boolean)"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Static)}()"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Instance)}()"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Recursive)}()"); } [Theory] - [MemberData(nameof(GetAllJits))] + [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, Runtime runtime) { - if (RuntimeInformation.IsMacOS()) return; // currently not supported + if (OsDetector.IsMacOS()) return; // currently not supported + var printSource = IsPrintSourceSupported(platform); var disassemblyDiagnoser = new DisassemblyDiagnoser( - new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 1, filters: new[] { "*WithCalls*" })); + new DisassemblyDiagnoserConfig(printSource: printSource, maxDepth: 1, filters: new[] { "*WithCalls*" })); CanExecute(CreateConfig(jit, platform, runtime, disassemblyDiagnoser, RunStrategy.ColdStart)); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Benchmark)}(Int32)"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Benchmark)}(Boolean)"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Static)}()"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Instance)}()"); - AssertDisassembled(disassemblyDiagnoser, $"{nameof(WithCalls.Recursive)}()"); + DisassemblyResult result = disassemblyDiagnoser.Results.Single().Value; + + Assert.Empty(result.Errors); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Benchmark)}(Int32)"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Benchmark)}(Boolean)"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Static)}()"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Instance)}()"); + AssertDisassemblyResult(result, $"{nameof(WithCalls.Recursive)}()"); } public class Generic where T : new() @@ -131,19 +140,21 @@ public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, } [Theory] - [MemberData(nameof(GetAllJits))] + [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleGenericTypes(Jit jit, Platform platform, Runtime runtime) { - if (RuntimeInformation.IsMacOS()) return; // currently not supported + if (OsDetector.IsMacOS()) return; // currently not supported + var printSource = IsPrintSourceSupported(platform); var disassemblyDiagnoser = new DisassemblyDiagnoser( - new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); + new DisassemblyDiagnoserConfig(printSource: printSource, maxDepth: 3)); CanExecute>(CreateConfig(jit, platform, runtime, disassemblyDiagnoser, RunStrategy.Monitoring)); var result = disassemblyDiagnoser.Results.Values.Single(); + Assert.Empty(result.Errors); Assert.Contains(result.Methods, method => method.Maps.Any(map => map.SourceCodes.OfType().Any())); } @@ -153,19 +164,21 @@ [Benchmark] public void JustReturn() { } } [Theory] - [MemberData(nameof(GetAllJits))] + [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleInlinableBenchmarks(Jit jit, Platform platform, Runtime runtime) { - if (RuntimeInformation.IsMacOS()) return; // currently not supported + if (OsDetector.IsMacOS()) return; // currently not supported + var printSource = IsPrintSourceSupported(platform); var disassemblyDiagnoser = new DisassemblyDiagnoser( - new DisassemblyDiagnoserConfig(printSource: true, maxDepth: 3)); + new DisassemblyDiagnoserConfig(printSource: printSource, maxDepth: 3)); CanExecute(CreateConfig(jit, platform, runtime, disassemblyDiagnoser, RunStrategy.Monitoring)); var disassemblyResult = disassemblyDiagnoser.Results.Values.Single(result => result.Methods.Count(method => method.Name.Contains(nameof(WithInlineable.JustReturn))) == 1); + Assert.Empty(disassemblyResult.Errors); Assert.Contains(disassemblyResult.Methods, method => method.Maps.Any(map => map.SourceCodes.OfType().All(asm => asm.ToString().Contains("ret")))); } @@ -180,12 +193,13 @@ private IConfig CreateConfig(Jit jit, Platform platform, Runtime runtime, IDiagn .AddDiagnoser(disassemblyDiagnoser) .AddLogger(new OutputLogger(Output)); - private void AssertDisassembled(DisassemblyDiagnoser diagnoser, string methodSignature) + private void AssertDisassemblyResult(DisassemblyResult result, string methodSignature) { - DisassemblyResult result = diagnoser.Results.Single().Value; - Assert.Contains(methodSignature, result.Methods.Select(m => m.Name.Split('.').Last()).ToArray()); Assert.Contains(result.Methods.Single(m => m.Name.EndsWith(methodSignature)).Maps, map => map.SourceCodes.Any()); } + + private static bool IsPrintSourceSupported(Platform platform) + => platform != Platform.X86; // Workaround for https://github.com/dotnet/BenchmarkDotNet/issues/2789 } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/DotMemoryTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DotMemoryTests.cs index 1e8dc79d00..2f21fb9f3c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DotMemoryTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DotMemoryTests.cs @@ -4,6 +4,7 @@ using System.Linq; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnostics.dotMemory; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -20,7 +21,7 @@ public DotMemoryTests(ITestOutputHelper output) : base(output) { } [Fact] public void DotMemorySmokeTest() { - if (!RuntimeInformation.IsWindows() && RuntimeInformation.IsMono) + if (!OsDetector.IsWindows() && RuntimeInformation.IsMono) { Output.WriteLine("Skip Mono on non-Windows"); return; diff --git a/tests/BenchmarkDotNet.IntegrationTests/DotTraceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DotTraceTests.cs index 80f11519e8..3c40ee31a0 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DotTraceTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DotTraceTests.cs @@ -4,6 +4,7 @@ using System.Linq; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnostics.dotTrace; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -20,7 +21,7 @@ public DotTraceTests(ITestOutputHelper output) : base(output) { } [Fact] public void DotTraceSmokeTest() { - if (!RuntimeInformation.IsWindows() && RuntimeInformation.IsMono) + if (!OsDetector.IsWindows() && RuntimeInformation.IsMono) { Output.WriteLine("Skip Mono on non-Windows"); return; diff --git a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs index 4cede6835b..d188be01d4 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs @@ -23,8 +23,10 @@ public class EventProcessorTests public void WhenUsingEventProcessorAndNoBenchmarks() { var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassEmpty) }); - Assert.Single(events); + Assert.Equal(3, events.Count); Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); + Assert.Equal(nameof(EventProcessor.OnValidationError), events[1].EventType); + Assert.Equal(nameof(EventProcessor.OnEndValidationStage), events[2].EventType); } [Fact] @@ -134,12 +136,13 @@ public void WhenUsingEventProcessorWithUnsupportedBenchmark() var toolchain = new AllUnsupportedToolchain(); var events = RunBenchmarksAndRecordEvents(new[] { typeof(ClassA) }, toolchain: toolchain); - Assert.Equal(3, events.Count); + Assert.Equal(4, events.Count); Assert.Equal(nameof(EventProcessor.OnStartValidationStage), events[0].EventType); Assert.Equal(nameof(EventProcessor.OnValidationError), events[1].EventType); Assert.Equal(typeof(ClassA).GetMethod(nameof(ClassA.Method1)), (events[1].Args[0] as ValidationError).BenchmarkCase.Descriptor.WorkloadMethod); Assert.Equal(nameof(EventProcessor.OnValidationError), events[2].EventType); Assert.Equal(typeof(ClassA).GetMethod(nameof(ClassA.Method2)), (events[2].Args[0] as ValidationError).BenchmarkCase.Descriptor.WorkloadMethod); + Assert.Equal(nameof(EventProcessor.OnEndValidationStage), events[3].EventType); } [Fact] diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs index a48315c2ef..4b9ea2005b 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs @@ -10,6 +10,7 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.InProcess.Emit; @@ -65,13 +66,16 @@ private void DiffEmit(Summary summary) { // .Net Core does not support assembly saving so far // SEE https://github.com/dotnet/corefx/issues/4491 + // TODO: Use new PersistedAssemblyBuilder when BDN and tests are updated to net9.0 or newer. if (!Portability.RuntimeInformation.IsFullFramework) return; - var caseName = summary.BenchmarksCases.First().Job.ToString(); + var benchmarkCase = summary.BenchmarksCases.First(); + // The benchmark config built jobs with 2 toolchains, 1 InProcessEmit and 1 Roslyn, + // so we need to subtract 1 from the partition counter to obtain the emit output. NaiveRunnableEmitDiff.RunDiff( - $@"{caseName}.exe", - $@"{caseName}Emitted.dll", + $@"{BuildPartition.GetProgramName(benchmarkCase, BuildPartition.s_partitionCounter)}.exe", + $@"{BuildPartition.GetProgramName(benchmarkCase, BuildPartition.s_partitionCounter - 1)}Emitted.dll", ConsoleLogger.Default); } @@ -239,35 +243,146 @@ public static ValueTask InvokeOnceStaticValueTaskOfT() } } - [Fact] - public void InProcessEmitToolchainSupportsIterationSetupAndCleanup() + [Theory] + [InlineData(typeof(IterationSetupCleanup))] + [InlineData(typeof(GlobalSetupCleanupTask))] + [InlineData(typeof(GlobalSetupCleanupValueTask))] + [InlineData(typeof(GlobalSetupCleanupValueTaskSource))] + public void InProcessEmitToolchainSupportsSetupAndCleanup(Type benchmarkType) { var logger = new OutputLogger(Output); var config = CreateInProcessConfig(logger); - WithIterationSetupAndCleanup.SetupCounter = 0; - WithIterationSetupAndCleanup.BenchmarkCounter = 0; - WithIterationSetupAndCleanup.CleanupCounter = 0; + Counters.SetupCounter = 0; + Counters.BenchmarkCounter = 0; + Counters.CleanupCounter = 0; - var summary = CanExecute(config); + var summary = CanExecute(benchmarkType, config); - Assert.Equal(1, WithIterationSetupAndCleanup.SetupCounter); - Assert.Equal(16, WithIterationSetupAndCleanup.BenchmarkCounter); - Assert.Equal(1, WithIterationSetupAndCleanup.CleanupCounter); + Assert.Equal(1, Counters.SetupCounter); + Assert.Equal(16, Counters.BenchmarkCounter); + Assert.Equal(1, Counters.CleanupCounter); } - public class WithIterationSetupAndCleanup + private static class Counters { public static int SetupCounter, BenchmarkCounter, CleanupCounter; + } + public class IterationSetupCleanup + { [IterationSetup] - public void Setup() => Interlocked.Increment(ref SetupCounter); + public void Setup() => Interlocked.Increment(ref Counters.SetupCounter); [Benchmark] - public void Benchmark() => Interlocked.Increment(ref BenchmarkCounter); + public void Benchmark() => Interlocked.Increment(ref Counters.BenchmarkCounter); [IterationCleanup] - public void Cleanup() => Interlocked.Increment(ref CleanupCounter); + public void Cleanup() => Interlocked.Increment(ref Counters.CleanupCounter); + } + + public class GlobalSetupCleanupTask + { + [GlobalSetup] + public static async Task GlobalSetup() + { + await Task.Yield(); + Interlocked.Increment(ref Counters.SetupCounter); + } + + [GlobalCleanup] + public async Task GlobalCleanup() + { + await Task.Yield(); + Interlocked.Increment(ref Counters.CleanupCounter); + return 42; + } + + [Benchmark] + public void InvokeOnceVoid() + { + Interlocked.Increment(ref Counters.BenchmarkCounter); + } + } + + public class GlobalSetupCleanupValueTask + { + [GlobalSetup] + public static async ValueTask GlobalSetup() + { + await Task.Yield(); + Interlocked.Increment(ref Counters.SetupCounter); + } + + [GlobalCleanup] + public async ValueTask GlobalCleanup() + { + await Task.Yield(); + Interlocked.Increment(ref Counters.CleanupCounter); + return 42; + } + + [Benchmark] + public void InvokeOnceVoid() + { + Interlocked.Increment(ref Counters.BenchmarkCounter); + } + } + + public class GlobalSetupCleanupValueTaskSource + { + private readonly static ValueTaskSource valueTaskSource = new(); + + [GlobalSetup] + public static ValueTask GlobalSetup() + { + valueTaskSource.Reset(); + Task.Delay(1).ContinueWith(_ => + { + Interlocked.Increment(ref Counters.SetupCounter); + valueTaskSource.SetResult(42); + }); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + + [GlobalCleanup] + public ValueTask GlobalCleanup() + { + valueTaskSource.Reset(); + Task.Delay(1).ContinueWith(_ => + { + Interlocked.Increment(ref Counters.CleanupCounter); + valueTaskSource.SetResult(42); + }); + return new ValueTask(valueTaskSource, valueTaskSource.Token); + } + + [Benchmark] + public void InvokeOnceVoid() + { + Interlocked.Increment(ref Counters.BenchmarkCounter); + } + } + + +#if NET8_0_OR_GREATER + [Fact] + public void ParamsSupportRequiredProperty() + { + var config = CreateInProcessConfig(null); + CanExecute(config); + } + + public class ParamsTestRequiredProperty + { + private const string Expected = "a"; + + [Params(Expected)] + public required string ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() => Assert.Equal(Expected, ParamProperty); } +#endif } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs index 619eaa5740..97c467a2bd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs @@ -273,5 +273,26 @@ public ValueTask InvokeOnceValueTaskOfT() return new ValueTask(DecimalResult); } } + + +#if NET8_0_OR_GREATER + [Fact] + public void ParamsSupportRequiredProperty() + { + var config = CreateInProcessConfig(); + CanExecute(config); + } + + public class ParamsTestRequiredProperty + { + private const string Expected = "a"; + + [Params(Expected)] + public required string ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() => Assert.Equal(Expected, ParamProperty); + } +#endif } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index fc6bc14362..7221a80595 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -8,6 +8,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.IntegrationTests.Xunit; @@ -35,7 +36,11 @@ public class MemoryDiagnoserTests public static IEnumerable GetToolchains() { yield return new object[] { Job.Default.GetToolchain() }; - yield return new object[] { InProcessEmitToolchain.Instance }; + // InProcessEmit reports flaky allocations in current .Net 8. + if (!RuntimeInformation.IsNetCore) + { + yield return new object[] { InProcessEmitToolchain.Instance }; + } } public class AccurateAllocations @@ -46,7 +51,7 @@ public class AccurateAllocations [Benchmark] public Task AllocateTask() => Task.FromResult(-12345); } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurate(IToolchain toolchain) { @@ -66,10 +71,10 @@ public void MemoryDiagnoserIsAccurate(IToolchain toolchain) }); } - [FactEnvSpecific("We don't want to test NativeAOT twice (for .NET Framework 4.6.2 and .NET 7.0)", EnvRequirement.DotNetCoreOnly)] + [FactEnvSpecific("We don't want to test NativeAOT twice (for .NET Framework 4.6.2 and .NET 8.0)", EnvRequirement.DotNetCoreOnly)] public void MemoryDiagnoserSupportsNativeAOT() { - if (RuntimeInformation.IsMacOS()) + if (OsDetector.IsMacOS()) return; // currently not supported MemoryDiagnoserIsAccurate(NativeAotToolchain.Net80); @@ -104,7 +109,7 @@ private void AllocateUntilGcWakesUp() } } - [Theory(Skip = "#1542 Tiered JIT Thread allocates memory in the background"), MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup(IToolchain toolchain) { @@ -114,24 +119,45 @@ public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup(IToolcha }); } - public class NoAllocationsAtAll + public class EmptyBenchmark { [Benchmark] public void EmptyMethod() { } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void EngineShouldNotInterfereAllocationResults(IToolchain toolchain) { - if (RuntimeInformation.IsFullFramework && toolchain.IsInProcess) + AssertAllocations(toolchain, typeof(EmptyBenchmark), new Dictionary { - return; // this test is flaky on Full Framework + { nameof(EmptyBenchmark.EmptyMethod), 0 } + }); + } + + public class TimeConsumingBenchmark + { + [Benchmark] + public ulong TimeConsuming() + { + var r = 1ul; + for (var i = 0; i < 50_000_000; i++) + { + r /= 1; + } + return r; } + } - AssertAllocations(toolchain, typeof(NoAllocationsAtAll), new Dictionary + // #1542 + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] + public void TieredJitShouldNotInterfereAllocationResults(IToolchain toolchain) + { + AssertAllocations(toolchain, typeof(TimeConsumingBenchmark), new Dictionary { - { nameof(NoAllocationsAtAll.EmptyMethod), 0 } - }); + { nameof(TimeConsumingBenchmark.TimeConsuming), 0 } + }, + disableTieredJit: false, iterationCount: 10); // 1 iteration is not enough to repro the problem } public class NoBoxing @@ -139,7 +165,7 @@ public class NoBoxing [Benchmark] public ValueTuple ReturnsValueType() => new ValueTuple(0); } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void EngineShouldNotIntroduceBoxing(IToolchain toolchain) { @@ -160,15 +186,10 @@ public class NonAllocatingAsynchronousBenchmarks [Benchmark] public ValueTask CompletedValueTaskOfT() => new ValueTask(default(int)); } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AwaitingTasksShouldNotInterfereAllocationResults(IToolchain toolchain) { - if (toolchain.IsInProcess) - { - return; // it's flaky: https://github.com/dotnet/BenchmarkDotNet/issues/1925 - } - AssertAllocations(toolchain, typeof(NonAllocatingAsynchronousBenchmarks), new Dictionary { { nameof(NonAllocatingAsynchronousBenchmarks.CompletedTask), 0 }, @@ -190,7 +211,7 @@ public void WithOperationsPerInvoke() private void DoNotInline(object left, object right) { } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocatedMemoryShouldBeScaledForOperationsPerInvoke(IToolchain toolchain) { @@ -216,67 +237,93 @@ public byte[] SixtyFourBytesArray() } } - [Theory(Skip = "#1542 Tiered JIT Thread allocates memory in the background"), MemberData(nameof(GetToolchains))] - //[TheoryNetCoreOnly("Only .NET Core 2.0+ API is bug free for this case"), MemberData(nameof(GetToolchains))] + [TheoryEnvSpecific("Full Framework cannot measure precisely enough for low invocation counts.", EnvRequirement.DotNetCoreOnly)] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) { long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length + int warmupCount = OsDetector.IsMacOS() + ? 5 // Workaround setting for macos. https://github.com/dotnet/BenchmarkDotNet/issues/2779 + : 0; // Other OS don't need warmup AssertAllocations(toolchain, typeof(TimeConsuming), new Dictionary { { nameof(TimeConsuming.SixtyFourBytesArray), 64 + objectAllocationOverhead + arraySizeOverhead } - }); + }, warmupCount: warmupCount); } public class MultiThreadedAllocation { - public const int Size = 1_000_000; + public const int Size = 1024; public const int ThreadsCount = 10; + // We cache the threads in GlobalSetup and reuse them for each benchmark invocation + // to avoid measuring the cost of thread start and join, which varies across different runtimes. private Thread[] threads; + private volatile bool keepRunning = true; + private readonly Barrier barrier = new(ThreadsCount + 1); + private readonly CountdownEvent countdownEvent = new(ThreadsCount); - [IterationSetup] - public void SetupIteration() + [GlobalSetup] + public void Setup() { threads = Enumerable.Range(0, ThreadsCount) - .Select(_ => new Thread(() => GC.KeepAlive(new byte[Size]))) + .Select(_ => new Thread(() => + { + while (keepRunning) + { + barrier.SignalAndWait(); + GC.KeepAlive(new byte[Size]); + countdownEvent.Signal(); + } + })) .ToArray(); + foreach (var thread in threads) + { + thread.Start(); + } } - [Benchmark] - public void Allocate() + [GlobalCleanup] + public void Cleanup() { + keepRunning = false; + barrier.SignalAndWait(); foreach (var thread in threads) { - thread.Start(); thread.Join(); } } + + [Benchmark] + public void Allocate() + { + countdownEvent.Reset(ThreadsCount); + barrier.SignalAndWait(); + countdownEvent.Wait(); + } } - [TheoryEnvSpecific(".NET Core 3.0 preview6+ exposes a GC.GetTotalAllocatedBytes method which makes it possible to work", - EnvRequirement.DotNetCore30Only)] - [MemberData(nameof(GetToolchains))] + [TheoryEnvSpecific("Full Framework cannot measure precisely enough", EnvRequirement.DotNetCoreOnly)] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolchain) { long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length long memoryAllocatedPerArray = (MultiThreadedAllocation.Size + objectAllocationOverhead + arraySizeOverhead); - long threadStartAndJoinOverhead = 112; // this is more or less a magic number taken from memory profiler - long allocatedMemoryPerThread = memoryAllocatedPerArray + threadStartAndJoinOverhead; AssertAllocations(toolchain, typeof(MultiThreadedAllocation), new Dictionary { - { nameof(MultiThreadedAllocation.Allocate), allocatedMemoryPerThread * MultiThreadedAllocation.ThreadsCount } + { nameof(MultiThreadedAllocation.Allocate), memoryAllocatedPerArray * MultiThreadedAllocation.ThreadsCount } }); } - private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators) + private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators, bool disableTieredJit = true, int iterationCount = 1, int warmupCount = 0) { - var config = CreateConfig(toolchain); + var config = CreateConfig(toolchain, disableTieredJit, iterationCount, warmupCount); var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType, config); var summary = BenchmarkRunner.Run(benchmarks); @@ -312,20 +359,37 @@ private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Diction } } - private IConfig CreateConfig(IToolchain toolchain) - => ManualConfig.CreateEmpty() - .AddJob(Job.ShortRun - .WithEvaluateOverhead(false) // no need to run idle for this test - .WithWarmupCount(0) // don't run warmup to save some time for our CI runs - .WithIterationCount(1) // single iteration is enough for us - .WithGcForce(false) - .WithGcServer(false) - .WithGcConcurrent(false) - .WithEnvironmentVariable("COMPlus_TieredCompilation", "0") // Tiered JIT can allocate some memory on a background thread, let's disable it to make our tests less flaky (#1542) - .WithToolchain(toolchain)) + private IConfig CreateConfig(IToolchain toolchain, + // Tiered JIT can allocate some memory on a background thread, let's disable it by default to make our tests less flaky (#1542). + // This was mostly fixed in net7.0, but tiered jit thread is not guaranteed to not allocate, so we disable it just in case. + bool disableTieredJit = true, + // Single iteration is enough for most of the tests. + int iterationCount = 1, + // Don't run warmup by default to save some time for our CI runs + int warmupCount = 0) + { + var job = Job.ShortRun + .WithEvaluateOverhead(false) // no need to run idle for this test + .WithWarmupCount(warmupCount) + .WithIterationCount(iterationCount) + .WithGcForce(false) + .WithGcServer(false) + .WithGcConcurrent(false) + // To prevent finalizers allocating out of our control, we hang the finalizer thread. + // https://github.com/dotnet/runtime/issues/101536#issuecomment-2077647417 + .WithEnvironmentVariable(Engines.Engine.UnitTestBlockFinalizerEnvKey, Engines.Engine.UnitTestBlockFinalizerEnvValue) + .WithToolchain(toolchain); + return ManualConfig.CreateEmpty() + .AddJob(disableTieredJit + ? job.WithEnvironmentVariables( + new EnvironmentVariable("DOTNET_TieredCompilation", "0"), + new EnvironmentVariable("COMPlus_TieredCompilation", "0") + ) + : job) .AddColumnProvider(DefaultColumnProviders.Instance) .AddDiagnoser(MemoryDiagnoser.Default) .AddLogger(toolchain.IsInProcess ? ConsoleLogger.Default : new OutputLogger(output)); // we can't use OutputLogger for the InProcess toolchains because it allocates memory on the same thread + } // note: don't copy, never use in production systems (it should work but I am not 100% sure) private int CalculateRequiredSpace() diff --git a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs index ce39b0572e..169fb3a0b3 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs @@ -1,6 +1,7 @@ using System; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; @@ -22,7 +23,7 @@ public void LatestNativeAotVersionIsSupported() return; if (ContinuousIntegration.IsGitHubActionsOnWindows()) // no native dependencies installed return; - if (RuntimeInformation.IsMacOS()) + if (OsDetector.IsMacOS()) return; // currently not supported var toolchain = NativeAotToolchain.CreateBuilder().UseNuGet().IlcInstructionSet(IsAvx2Supported() ? "avx2" : "").ToToolchain(); diff --git a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs index 4d38427aab..e9f9469f13 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs @@ -1,4 +1,7 @@ using System; +using System.Collections.Immutable; +using System.Linq; +using System.Reflection; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -6,7 +9,6 @@ using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.Roslyn; -using Newtonsoft.Json; using Xunit; using Xunit.Abstractions; @@ -19,24 +21,32 @@ public NuGetReferenceTests(ITestOutputHelper output) : base(output) } [Fact] + [Obsolete] public void UserCanSpecifyCustomNuGetPackageDependency() { var toolchain = RuntimeInformation.GetCurrentRuntime().GetToolchain(preferMsBuildToolchains: true); - var job = Job.Dry.WithToolchain(toolchain).WithNuGet("Newtonsoft.Json", "13.0.2"); + const string targetVersion = "9.0.5"; + + var job = Job.Dry.WithToolchain(toolchain).WithNuGet("System.Collections.Immutable", targetVersion); var config = CreateSimpleConfig(job: job); - CanExecute(config); + var report = CanExecute(config); + + // Validate NuGet package version output message + var stdout = GetSingleStandardOutput(report); + Assert.Contains($"System.Collections.Immutable: {targetVersion}", stdout); } [FactEnvSpecific("Roslyn toolchain does not support .NET Core", EnvRequirement.FullFrameworkOnly)] + [Obsolete] public void RoslynToolchainDoesNotSupportNuGetPackageDependency() { var toolchain = RoslynToolchain.Instance; - var unsupportedJob = Job.Dry.WithToolchain(toolchain).WithNuGet("Newtonsoft.Json", "13.0.2"); + var unsupportedJob = Job.Dry.WithToolchain(toolchain).WithNuGet("System.Collections.Immutable", "9.0.5"); var unsupportedJobConfig = CreateSimpleConfig(job: unsupportedJob); - var unsupportedJobBenchmark = BenchmarkConverter.TypeToBenchmarks(typeof(WithCallToNewtonsoft), unsupportedJobConfig); + var unsupportedJobBenchmark = BenchmarkConverter.TypeToBenchmarks(typeof(WithCallToImmutableArray), unsupportedJobConfig); foreach (var benchmarkCase in unsupportedJobBenchmark.BenchmarksCases) { @@ -45,16 +55,39 @@ public void RoslynToolchainDoesNotSupportNuGetPackageDependency() var supportedJob = Job.Dry.WithToolchain(toolchain); var supportedConfig = CreateSimpleConfig(job: supportedJob); - var supportedBenchmark = BenchmarkConverter.TypeToBenchmarks(typeof(WithCallToNewtonsoft), supportedConfig); + var supportedBenchmark = BenchmarkConverter.TypeToBenchmarks(typeof(WithCallToImmutableArray), supportedConfig); foreach (var benchmarkCase in supportedBenchmark.BenchmarksCases) { Assert.Empty(toolchain.Validate(benchmarkCase, BenchmarkRunnerClean.DefaultResolver)); } } - public class WithCallToNewtonsoft + public class WithCallToImmutableArray { - [Benchmark] public void SerializeAnonymousObject() => JsonConvert.SerializeObject(new { hello = "world", price = 1.99, now = DateTime.UtcNow }); + private readonly double[] values; + + public WithCallToImmutableArray() + { + var rand = new Random(Seed: 0); + values = Enumerable.Range(1, 10_000) + .Select(x => rand.NextDouble()) + .ToArray(); + + // Gets assembly version text from AssemblyInformationalVersion attribute. + var version = typeof(ImmutableArray).Assembly + .GetCustomAttribute()! + .InformationalVersion + .Split('+')[0]; + + // Print referenced NuGet package version to stdout. + Console.WriteLine($"System.Collections.Immutable: {version}"); + } + + [Benchmark] + public void ToImmutableArrayBenchmark() + { + var results = values.ToImmutableArray(); + } } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/ParamSourceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ParamSourceTests.cs index ed105f8290..e58f6a67d7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ParamSourceTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ParamSourceTests.cs @@ -96,7 +96,7 @@ public static IEnumerable GetSource() public int Benchmark() => ParamsTarget?.Data ?? 0; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void PrivateClassWithPublicInterface_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(PrivateClassWithPublicInterface), toolchain); public class PrivateClassWithPublicInterface_Array @@ -116,7 +116,7 @@ public IEnumerable GetSource() public int Benchmark() => ParamsTarget?.Sum(p => p?.Data ?? 0) ?? 0; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void PrivateClassWithPublicInterface_Array_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(PrivateClassWithPublicInterface_Array), toolchain); public class PrivateClassWithPublicInterface_Enumerable @@ -137,7 +137,7 @@ public IEnumerable> GetSource() public int Benchmark() => ParamsTarget?.Sum(p => p?.Data ?? 0) ?? 0; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void PrivateClassWithPublicInterface_Enumerable_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(PrivateClassWithPublicInterface_Enumerable), toolchain); public class PrivateClassWithPublicInterface_AsObject @@ -156,7 +156,7 @@ public static IEnumerable GetSource() public int Benchmark() => ParamsTarget?.Data ?? 0; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void PrivateClassWithPublicInterface_AsObject_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(PrivateClassWithPublicInterface_AsObject), toolchain); public class PublicSource diff --git a/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs index 0285856bcb..0f703bf1cd 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs @@ -217,5 +217,28 @@ public void Test() throw new ArgumentException($"{nameof(StaticParamProperty)} has wrong value: {StaticParamProperty}!"); } } + +#if NET8_0_OR_GREATER + [Fact] + public void ParamsSupportRequiredProperty() + { + var summary = CanExecute(); + var standardOutput = GetCombinedStandardOutput(summary); + + foreach (var param in new[] { "a", "b" }) + { + Assert.Contains($"// ### New Parameter {param} ###", standardOutput); + } + } + + public class ParamsTestRequiredProperty + { + [Params("a", "b")] + public required string ParamProperty { get; set; } + + [Benchmark] + public void Benchmark() => Console.WriteLine($"// ### New Parameter {ParamProperty} ###"); + } +#endif } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/PowerRequest.cs b/tests/BenchmarkDotNet.IntegrationTests/PowerRequest.cs new file mode 100644 index 0000000000..676ad23d40 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/PowerRequest.cs @@ -0,0 +1,9 @@ +namespace BenchmarkDotNet.IntegrationTests; + +internal class PowerRequest(string requestType, string requesterType, string requesterName, string? reason) +{ + public string RequestType { get; } = requestType; + public string RequesterType { get; } = requesterType; + public string RequesterName { get; } = requesterName; + public string? Reason { get; } = reason; +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/PowerRequestsParser.cs b/tests/BenchmarkDotNet.IntegrationTests/PowerRequestsParser.cs new file mode 100644 index 0000000000..bb634b5228 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/PowerRequestsParser.cs @@ -0,0 +1,187 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace BenchmarkDotNet.IntegrationTests; + +/// +/// Parses the output of 'powercfg /requests' command into a list of s. +/// +/// +/// +/// Not using Sprache. It is superseded by Superpower. +/// Not using Superpower. I gained more knowledge +/// implementing this class from scratch. +/// +/// Example input: +/// +/// DISPLAY: +/// [PROCESS] \Device\HarddiskVolume3\Program Files (x86)\Google\Chrome\Application\chrome.exe +/// Video Wake Lock +/// +/// SYSTEM: +/// [DRIVER] Realtek High Definition Audio(SST) ... +/// Er wordt momenteel een audiostream gebruikt. +/// [PROCESS] \Device\HarddiskVolume3\...\NoSleep.exe +/// [PROCESS] \Device\HarddiskVolume3\Program Files (x86)\Google\Chrome\Application\chrome.exe +/// Video Wake Lock +/// +/// AWAYMODE: +/// None. +/// +/// EXECUTION: +/// [PROCESS] \Device\HarddiskVolume3\Program Files (x86)\Google\Chrome\Application\chrome.exe +/// Playing audio +/// +/// PERFBOOST: +/// None. +/// +/// ACTIVELOCKSCREEN: +/// None. +/// +/// +/// +internal class PowerRequestsParser +{ + /// + /// Parses output of 'powercfg /requests' into a list of s. + /// + /// + /// + /// This method takes a list of s. Examines next token and decides how to + /// parse. + /// + /// + /// Output of 'powercfg /requests'. + public static IEnumerable Parse(string input) + { + using TokenStream tokens = new TokenStream(Tokens(input)); + while (tokens.TryPeek().HasValue) + { + foreach (PowerRequest item in ParseRequestType(tokens)) + { + yield return item; + } + } + } + + private static IEnumerable ParseRequestType(TokenStream tokens) + { + Token requestType = tokens.Take(TokenType.RequestType); + if (tokens.Peek().TokenType == TokenType.RequesterType) + { + while (tokens.Peek().TokenType == TokenType.RequesterType) + { + yield return ParseRequesterType(requestType, tokens); + } + } + else + { + _ = tokens.Take(TokenType.None); + } + _ = tokens.Take(TokenType.EmptyLine); + } + + private static PowerRequest ParseRequesterType(Token requestType, TokenStream tokens) + { + Token requesterType = tokens.Take(TokenType.RequesterType); + Token requesterName = tokens.Take(TokenType.RequesterName); + Token? reason = null; + if (tokens.Peek().TokenType == TokenType.Reason) + { + reason = tokens.Take(TokenType.Reason); + } + return new PowerRequest(requestType.Value, requesterType.Value, requesterName.Value, reason?.Value); + } + + /// + /// Converts the input into a list of s. + /// + /// + /// + /// Looking at above sample, tokenizing is made simple when done line by line. Each line + /// contains one or two s. + /// + /// + /// Output of 'powercfg /requests'. + private static IEnumerable Tokens(string input) + { + // Contrary to calling input.Split('\r', '\n'), StringReader's ReadLine method does not + // return an empty string when CR is followed by LF. + StringReader reader = new StringReader(input); + string? line; + while ((line = reader.ReadLine()) != null) + { + if (line.Length == 0) + { + yield return new Token(TokenType.EmptyLine, ""); + } + else if (line[line.Length - 1] == ':') + { + yield return new Token(TokenType.RequestType, line.Substring(0, line.Length - 1).ToString()); + } + else if (string.Equals(line, "None.", StringComparison.InvariantCulture)) + { + yield return new Token(TokenType.None, line); + } + else if (line[0] == '[') + { + int pos = line.IndexOf(']'); + yield return new Token(TokenType.RequesterType, line.Substring(1, pos - 1)); + yield return new Token(TokenType.RequesterName, line.Substring(pos + 2)); + } + else + { + yield return new Token(TokenType.Reason, line); + } + } + } + + /// + /// Adds and to an of + /// s. + /// + /// + private class TokenStream(IEnumerable tokens) : IDisposable + { + private readonly IEnumerator tokens = tokens.GetEnumerator(); + private Token? cached; + + public Token? TryPeek() => cached ??= tokens.MoveNext() ? tokens.Current : null; + + public Token Peek() => TryPeek() ?? throw new EndOfStreamException(); + + public Token Take(TokenType requestType) + { + Token peek = Peek(); + if (peek.TokenType == requestType) + { + cached = null; + return peek; + } + else + { + throw new InvalidCastException($"Unexpected Token of type '{peek.TokenType}'. Expected type '{requestType}'."); + } + } + + public void Dispose() => tokens.Dispose(); + } + + private enum TokenType + { + EmptyLine, + None, + Reason, + RequesterName, + RequesterType, + RequestType + } + + private readonly struct Token(TokenType tokenType, string value) + { + public TokenType TokenType { get; } = tokenType; + + public string Value { get; } = value; + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs index ab9fbbb645..71988ba7c9 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs @@ -53,7 +53,7 @@ public class NonTailCallBenchmarks } [TheoryEnvSpecific(WindowsOnly, EnvRequirement.WindowsOnly)] - [MemberData(nameof(GetJits))] + [MemberData(nameof(GetJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void TailCallDiagnoserCatchesTailCallEvents(Jit jit, Platform platform, Runtime runtime) { @@ -63,7 +63,7 @@ public void TailCallDiagnoserCatchesTailCallEvents(Jit jit, Platform platform, R } [TheoryEnvSpecific(WindowsOnly, EnvRequirement.WindowsOnly)] - [MemberData(nameof(GetJits))] + [MemberData(nameof(GetJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void TailCallDiagnoserNotCatchesTailCallEvents(Jit jit, Platform platform, Runtime runtime) { diff --git a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs index 213bc3af02..c1a45d4e01 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs @@ -14,6 +14,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Portability; using Xunit; @@ -32,7 +33,7 @@ public static IEnumerable GetToolchains() yield return new object[] { Job.Default.GetToolchain() }; if (!ContinuousIntegration.IsGitHubActionsOnWindows() // no native dependencies - && !RuntimeInformation.IsMacOS()) // currently not supported + && !OsDetector.IsMacOS()) // currently not supported { yield return new object[]{ NativeAotToolchain.Net80 }; } @@ -40,7 +41,7 @@ public static IEnumerable GetToolchains() // yield return new object[] { InProcessEmitToolchain.Instance }; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void CompletedWorkItemCountIsAccurate(IToolchain toolchain) { var config = CreateConfig(toolchain); @@ -81,7 +82,7 @@ public void CompleteOneWorkItem() public void DoNothing() { } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void LockContentionCountIsAccurate(IToolchain toolchain) { var config = CreateConfig(toolchain); diff --git a/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs b/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs index f6a5d10428..e4eb071029 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs @@ -22,7 +22,7 @@ public static IEnumerable GetToolchains() => new[] new object[] { InProcessEmitToolchain.Instance }, }; - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void AnyValueCanBeReturned(IToolchain toolchain) => CanExecute(ManualConfig.CreateEmpty().AddJob(Job.Dry.WithToolchain(toolchain))); public class ValuesReturnedByBenchmark diff --git a/tests/BenchmarkDotNet.IntegrationTests/WakeLockTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WakeLockTests.cs new file mode 100644 index 0000000000..cf8d0d39b9 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/WakeLockTests.cs @@ -0,0 +1,179 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Tests.XUnit; +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.Versioning; +using System.Threading; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.IntegrationTests; + +public class WakeLockTests : BenchmarkTestExecutor +{ + private const string PingEventName = @"Global\WakeLockTests-ping"; + private const string PongEventName = @"Global\WakeLockTests-pong"; + private static readonly TimeSpan testTimeout = TimeSpan.FromMinutes(1); + private readonly OutputLogger logger; + + public WakeLockTests(ITestOutputHelper output) : base(output) + { + logger = new OutputLogger(Output); + } + + [Fact] + public void ConfigurationDefaultValue() + { + Assert.Equal(WakeLockType.System, DefaultConfig.Instance.WakeLock); + Assert.Equal(WakeLockType.None, new DebugBuildConfig().WakeLock); + Assert.Equal(WakeLockType.None, new DebugInProcessConfig().WakeLock); + } + + [TheoryEnvSpecific(EnvRequirement.NonWindows)] + [InlineData(WakeLockType.None)] + [InlineData(WakeLockType.System)] + [InlineData(WakeLockType.Display)] + public void WakeLockIsWindowsOnly(WakeLockType wakeLockType) + { + using IDisposable wakeLock = WakeLock.Request(wakeLockType, "dummy", logger); + Assert.Null(wakeLock); + } + + [FactEnvSpecific(EnvRequirement.WindowsOnly)] + public void WakeLockSleepOrDisplayIsAllowed() + { + using IDisposable wakeLock = WakeLock.Request(WakeLockType.None, "dummy", logger); + Assert.Null(wakeLock); + } + + [FactEnvSpecific(EnvRequirement.WindowsOnly, EnvRequirement.NeedsPrivilegedProcess)] + public void WakeLockRequireSystem() + { + using (IDisposable wakeLock = WakeLock.Request(WakeLockType.System, "WakeLockTests", logger)) + { + Assert.NotNull(wakeLock); + Assert.Equal("SYSTEM", GetPowerRequests("WakeLockTests")); + } + Assert.Equal("", GetPowerRequests()); + } + + [FactEnvSpecific(EnvRequirement.WindowsOnly, EnvRequirement.NeedsPrivilegedProcess)] + public void WakeLockRequireDisplay() + { + using (IDisposable wakeLock = WakeLock.Request(WakeLockType.Display, "WakeLockTests", logger)) + { + Assert.NotNull(wakeLock); + Assert.Equal("DISPLAY, SYSTEM", GetPowerRequests("WakeLockTests")); + } + Assert.Equal("", GetPowerRequests()); + } + + [FactEnvSpecific(EnvRequirement.NonWindows)] + public void BenchmarkRunnerIgnoresWakeLock() => + _ = CanExecute(fullValidation: false); + + [WakeLock(WakeLockType.Display)] + public class IgnoreWakeLock + { + [Benchmark] public void Sleep() { } + } + +#if !NET462 + [SupportedOSPlatform("windows")] +#endif + [TheoryEnvSpecific(EnvRequirement.WindowsOnly, EnvRequirement.NeedsPrivilegedProcess)] + [InlineData(typeof(Default), "SYSTEM")] + [InlineData(typeof(None), "")] + [InlineData(typeof(RequireSystem), "SYSTEM")] + [InlineData(typeof(RequireDisplay), "DISPLAY, SYSTEM")] + public async Task BenchmarkRunnerAcquiresWakeLock(Type type, string expected) + { + using EventWaitHandle + ping = new EventWaitHandle(false, EventResetMode.AutoReset, PingEventName), + pong = new EventWaitHandle(false, EventResetMode.AutoReset, PongEventName); + string pwrRequests = null; + Task task = WaitForBenchmarkRunningAndGetPowerRequests(); + _ = CanExecute(type, fullValidation: false); + await task; + + Assert.Equal(expected, pwrRequests); + + async Task WaitForBenchmarkRunningAndGetPowerRequests() + { + await AsTask(ping, testTimeout); + pwrRequests = GetPowerRequests("BenchmarkDotNet Running Benchmarks"); + pong.Set(); + } + } + + public class Default : Base { } + + [WakeLock(WakeLockType.None)] public class None : Base { } + + [WakeLock(WakeLockType.System)] public class RequireSystem : Base { } + + [WakeLock(WakeLockType.Display)] public class RequireDisplay : Base { } + + public class Base + { + [Benchmark] +#if !NET462 + [SupportedOSPlatform("windows")] +#endif + public void SignalBenchmarkRunningAndWaitForGetPowerRequests() + { + using EventWaitHandle + ping = EventWaitHandle.OpenExisting(PingEventName), + pong = EventWaitHandle.OpenExisting(PongEventName); + ping.Set(); + pong.WaitOne(testTimeout); + } + } + + private string GetPowerRequests(string? expectedReason = null) + { + string pwrRequests = ProcessHelper.RunAndReadOutput("powercfg", "/requests"); + Output.WriteLine(pwrRequests); // Useful to analyse failing tests. + string fileName = Process.GetCurrentProcess().MainModule.FileName; + string mustEndWith = fileName.Substring(Path.GetPathRoot(fileName).Length); + + return string.Join(", ", + from pr in PowerRequestsParser.Parse(pwrRequests) + where + pr.RequesterName.EndsWith(mustEndWith, StringComparison.InvariantCulture) && + string.Equals(pr.RequesterType, "PROCESS", StringComparison.InvariantCulture) && + (expectedReason == null || string.Equals(pr.Reason, expectedReason, StringComparison.InvariantCulture)) + select pr.RequestType); + } + + private Task AsTask(WaitHandle waitHandle, TimeSpan timeout) + { + TaskCompletionSource tcs = new TaskCompletionSource(); + RegisteredWaitHandle rwh = null; + rwh = ThreadPool.RegisterWaitForSingleObject( + waitHandle, + (object state, bool timedOut) => + { + rwh.Unregister(null); + if (timedOut) + { + tcs.SetException(new TimeoutException()); + } + else + { + tcs.SetResult(true); + } + }, + null, + timeout, + true); + return tcs.Task; + } +} diff --git a/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesVerifyTests.cs b/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesVerifyTests.cs index a1086da1de..8da86700c7 100644 --- a/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesVerifyTests.cs +++ b/tests/BenchmarkDotNet.Tests/Attributes/ParamsAllValuesVerifyTests.cs @@ -9,6 +9,7 @@ using BenchmarkDotNet.Tests.Mocks; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Infra; using BenchmarkDotNet.Validators; using JetBrains.Annotations; using VerifyXunit; @@ -17,7 +18,6 @@ namespace BenchmarkDotNet.Tests.Attributes { [Collection("VerifyTests")] - [UsesVerify] public class ParamsAllValuesVerifyTests : IDisposable { private readonly CultureInfo initCulture; @@ -53,7 +53,7 @@ public Task BenchmarkShouldProduceSummary(Type benchmarkType) foreach (var error in errors) logger.WriteLineError("* " + error.Message); - var settings = VerifySettingsFactory.Create(); + var settings = VerifyHelper.Create(); settings.UseTextForParameters(benchmarkType.Name); return Verifier.Verify(logger.GetLog(), settings); } diff --git a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj index fcdc416bbf..b37e22deda 100755 --- a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj +++ b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj @@ -15,20 +15,18 @@ - - - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + @@ -43,6 +41,16 @@ - + + + + + CpuInfoFormatterTests + CpuInfoFormatterTests.cs + + + + + diff --git a/tests/BenchmarkDotNet.Tests/Builders/HostEnvironmentInfoBuilder.cs b/tests/BenchmarkDotNet.Tests/Builders/HostEnvironmentInfoBuilder.cs index d22256f0a9..29d690892b 100644 --- a/tests/BenchmarkDotNet.Tests/Builders/HostEnvironmentInfoBuilder.cs +++ b/tests/BenchmarkDotNet.Tests/Builders/HostEnvironmentInfoBuilder.cs @@ -1,11 +1,14 @@ using System; +using System.Diagnostics.CodeAnalysis; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Portability.Cpu; using Perfolizer.Horology; +using Perfolizer.Models; namespace BenchmarkDotNet.Tests.Builders { + [SuppressMessage("ReSharper", "InconsistentNaming")] public class HostEnvironmentInfoBuilder { private string architecture = "64mock"; @@ -20,16 +23,18 @@ public class HostEnvironmentInfoBuilder private bool isServerGC = false; private string jitInfo = "RyuJIT-v4.6.x.mock"; private string jitModules = "clrjit-v4.6.x.mock"; - private string osVersion = "Microsoft Windows NT 10.0.x.mock"; + private OsInfo os = new() { Display = "Microsoft Windows NT 10.0.x.mock" }; private string runtimeVersion = "Clr 4.0.x.mock"; - private CpuInfo cpuInfo = new CpuInfo("MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", - physicalProcessorCount: 1, - physicalCoreCount: 4, - logicalCoreCount: 8, - nominalFrequency: Frequency.FromMHz(3100), - maxFrequency: Frequency.FromMHz(3100), - minFrequency: Frequency.FromMHz(3100)); + private readonly CpuInfo cpu = new() + { + ProcessorName = "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 4, + LogicalCoreCount = 8, + NominalFrequencyHz = Frequency.FromMHz(3100).Hertz.RoundToLong(), + MaxFrequencyHz = Frequency.FromMHz(3100).Hertz.RoundToLong() + }; private VirtualMachineHypervisor? virtualMachineHypervisor = HyperV.Default; @@ -55,7 +60,7 @@ public HostEnvironmentInfo Build() { return new MockHostEnvironmentInfo(architecture, benchmarkDotNetVersion, chronometerFrequency, configuration, dotNetSdkVersion, hardwareTimerKind, hasAttachedDebugger, hasRyuJit, isConcurrentGC, isServerGC, - jitInfo, jitModules, osVersion, cpuInfo, runtimeVersion, virtualMachineHypervisor); + jitInfo, jitModules, os, cpu, runtimeVersion, virtualMachineHypervisor); } } @@ -64,7 +69,7 @@ internal class MockHostEnvironmentInfo : HostEnvironmentInfo public MockHostEnvironmentInfo( string architecture, string benchmarkDotNetVersion, Frequency chronometerFrequency, string configuration, string dotNetSdkVersion, HardwareTimerKind hardwareTimerKind, bool hasAttachedDebugger, bool hasRyuJit, bool isConcurrentGC, bool isServerGC, - string jitInfo, string jitModules, string osVersion, CpuInfo cpuInfo, + string jitInfo, string jitModules, OsInfo os, CpuInfo cpu, string runtimeVersion, VirtualMachineHypervisor virtualMachineHypervisor) { Architecture = architecture; @@ -79,8 +84,8 @@ public MockHostEnvironmentInfo( IsServerGC = isServerGC; JitInfo = jitInfo; HardwareIntrinsicsShort = ""; - OsVersion = new Lazy(() => osVersion); - CpuInfo = new Lazy(() => cpuInfo); + Os = new Lazy(() => os); + Cpu = new Lazy(() => cpu); RuntimeVersion = runtimeVersion; VirtualMachineHypervisor = new Lazy(() => virtualMachineHypervisor); } diff --git a/tests/BenchmarkDotNet.Tests/Builders/VerifySettingsFactory.cs b/tests/BenchmarkDotNet.Tests/Builders/VerifySettingsFactory.cs deleted file mode 100644 index 027b5272a7..0000000000 --- a/tests/BenchmarkDotNet.Tests/Builders/VerifySettingsFactory.cs +++ /dev/null @@ -1,15 +0,0 @@ -using VerifyTests; - -namespace BenchmarkDotNet.Tests.Builders -{ - public static class VerifySettingsFactory - { - public static VerifySettings Create() - { - var result = new VerifySettings(); - result.UseDirectory("VerifiedFiles"); - result.DisableDiff(); - return result; - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index 5da3fd7a21..dd25527e8a 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -162,7 +162,7 @@ public void SpecifyingCoreRunWithFullFrameworkTargetsMostRecentTfm() CoreRunToolchain coreRunToolchain = (CoreRunToolchain)coreRunJob.GetToolchain(); DotNetCliGenerator generator = (DotNetCliGenerator)coreRunToolchain.Generator; - Assert.Equal("net9.0", generator.TargetFrameworkMoniker); + Assert.Equal("net10.0", generator.TargetFrameworkMoniker); } [FactEnvSpecific("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process", EnvRequirement.DotNetCoreOnly)] @@ -233,8 +233,8 @@ public void UserCanSpecifyMultipleCoreRunPaths() var jobs = config.GetJobs().ToArray(); Assert.Equal(2, jobs.Length); - Assert.Single(jobs.Where(job => job.GetToolchain() is CoreRunToolchain toolchain && toolchain.SourceCoreRun.FullName == fakeCoreRunPath_1)); - Assert.Single(jobs.Where(job => job.GetToolchain() is CoreRunToolchain toolchain && toolchain.SourceCoreRun.FullName == fakeCoreRunPath_2)); + Assert.Single(jobs, job => job.GetToolchain() is CoreRunToolchain toolchain && toolchain.SourceCoreRun.FullName == fakeCoreRunPath_1); + Assert.Single(jobs, job => job.GetToolchain() is CoreRunToolchain toolchain && toolchain.SourceCoreRun.FullName == fakeCoreRunPath_2); } [Fact] @@ -244,7 +244,7 @@ public void MonoPathParsedCorrectly() var config = ConfigParser.Parse(new[] { "-r", "mono", "--monoPath", fakeMonoPath }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is MonoRuntime mono && mono.CustomPath == fakeMonoPath)); + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is MonoRuntime mono && mono.CustomPath == fakeMonoPath); } [FactEnvSpecific("Testing local builds of Full .NET Framework is supported only on Windows", EnvRequirement.WindowsOnly)] @@ -254,7 +254,7 @@ public void ClrVersionParsedCorrectly() var config = ConfigParser.Parse(new[] { "--clrVersion", clrVersion }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is ClrRuntime clr && clr.Version == clrVersion)); + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is ClrRuntime clr && clr.Version == clrVersion); } [Fact] @@ -275,6 +275,12 @@ public void IlCompilerPathParsedCorrectly() [InlineData("netcoreapp2.1", true)] [InlineData("netcoreapp2.2", true)] [InlineData("netcoreapp3.0", true)] + [InlineData("netcoreapp3.1", true)] + [InlineData("net5.0", true)] + [InlineData("net6.0", true)] + [InlineData("net7.0", true)] + [InlineData("net8.0", true)] + [InlineData("net9.0", true)] [InlineData("net462", false)] [InlineData("net48", false)] public void DotNetCliParsedCorrectly(string tfm, bool isCore) @@ -333,7 +339,7 @@ public void WhenConfigOptionsFlagsAreNotSpecifiedTheyAreNotSet() public void PackagesPathParsedCorrectly() { var fakeRestoreDirectory = new FileInfo(typeof(object).Assembly.Location).Directory.FullName; - var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.0", "--packages", fakeRestoreDirectory }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.1", "--packages", fakeRestoreDirectory }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); CsProjCoreToolchain toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain; @@ -345,7 +351,7 @@ public void PackagesPathParsedCorrectly() public void UserCanSpecifyBuildTimeout() { const int timeoutInSeconds = 10; - var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.0", "--buildTimeout", timeoutInSeconds.ToString() }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.1", "--buildTimeout", timeoutInSeconds.ToString() }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); CsProjCoreToolchain toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain; @@ -356,7 +362,7 @@ public void UserCanSpecifyBuildTimeout() [Fact] public void WhenUserDoesNotSpecifyTimeoutTheDefaultValueIsUsed() { - var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.0" }, new OutputLogger(Output)).config; + var config = ConfigParser.Parse(new[] { "-r", "netcoreapp3.1" }, new OutputLogger(Output)).config; Assert.Single(config.GetJobs()); CsProjCoreToolchain toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain; @@ -364,6 +370,22 @@ public void WhenUserDoesNotSpecifyTimeoutTheDefaultValueIsUsed() Assert.Equal(DefaultConfig.Instance.BuildTimeout, config.BuildTimeout); } + [Fact] + public void UserCanSpecifyWakeLock() + { + var config = ConfigParser.Parse(["--wakeLock", "Display"], new OutputLogger(Output)).config; + + Assert.Equal(WakeLockType.Display, config.WakeLock); + } + + [Fact] + public void WhenUserDoesNotSpecifyWakeLockTheDefaultValueIsUsed() + { + var config = ConfigParser.Parse([], new OutputLogger(Output)).config; + + Assert.Equal(DefaultConfig.Instance.WakeLock, config.WakeLock); + } + [Theory] [InlineData("net461")] [InlineData("net462")] @@ -384,10 +406,17 @@ public void NetFrameworkMonikerParsedCorrectly(string tfm) [Theory] [InlineData("net50")] + [InlineData("net5.0")] [InlineData("net60")] + [InlineData("net6.0")] [InlineData("net70")] + [InlineData("net7.0")] [InlineData("net80")] + [InlineData("net8.0")] [InlineData("net90")] + [InlineData("net9.0")] + [InlineData("net10_0")] + [InlineData("net10.0")] public void NetMonikersAreRecognizedAsNetCoreMonikers(string tfm) { var config = ConfigParser.Parse(new[] { "-r", tfm }, new OutputLogger(Output)).config; @@ -414,24 +443,24 @@ public void PlatformSpecificMonikersAreSupported(string msBuildMoniker) [Fact] public void CanCompareFewDifferentRuntimes() { - var config = ConfigParser.Parse(new[] { "--runtimes", "net462", "MONO", "netcoreapp3.0", "nativeaot6.0", "nativeAOT7.0", "nativeAOT8.0" }, + var config = ConfigParser.Parse(["--runtimes", "net462", "MONO", "netcoreapp2.0", "nativeaot6.0", "nativeAOT7.0", "nativeAOT8.0"], new OutputLogger(Output)).config; Assert.True(config.GetJobs().First().Meta.Baseline); // when the user provides multiple runtimes the first one should be marked as baseline - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is ClrRuntime clrRuntime && clrRuntime.MsBuildMoniker == "net462")); - Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is MonoRuntime)); - Assert.Single(config.GetJobs().Where(job => - job.Environment.Runtime is CoreRuntime coreRuntime && coreRuntime.MsBuildMoniker == "netcoreapp3.0" && - coreRuntime.RuntimeMoniker == RuntimeMoniker.NetCoreApp30)); - Assert.Single(config.GetJobs().Where(job => + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is ClrRuntime clrRuntime && clrRuntime.MsBuildMoniker == "net462"); + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is MonoRuntime); + Assert.Single(config.GetJobs(), job => + job.Environment.Runtime is CoreRuntime coreRuntime && coreRuntime.MsBuildMoniker == "netcoreapp2.0" && + coreRuntime.RuntimeMoniker == RuntimeMoniker.NetCoreApp20); + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net6.0" && - nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot60)); - Assert.Single(config.GetJobs().Where(job => + nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot60); + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net7.0" && - nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot70)); - Assert.Single(config.GetJobs().Where(job => + nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot70); + Assert.Single(config.GetJobs(), job => job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net8.0" && - nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot80)); + nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot80); } [Theory] @@ -439,7 +468,7 @@ public void CanCompareFewDifferentRuntimes() [InlineData("10ms")] public void CanUseStatisticalTestsToCompareFewDifferentRuntimes(string threshold) { - string[] arguments = ["--runtimes", "netcoreapp2.1", "netcoreapp2.2", "--statisticalTest", threshold]; + string[] arguments = ["--runtimes", "net6.0", "net8.0", "--statisticalTest", threshold]; var config = ConfigParser.Parse(arguments, new OutputLogger(Output)).config; var mockSummary = MockFactory.CreateSummary(config); @@ -469,8 +498,8 @@ public void CanParseHardwareCounters() new OutputLogger(Output)).config; Assert.Equal(2, config.GetHardwareCounters().Count()); - Assert.Single(config.GetHardwareCounters().Where(counter => counter == HardwareCounter.CacheMisses)); - Assert.Single(config.GetHardwareCounters().Where(counter => counter == HardwareCounter.InstructionRetired)); + Assert.Single(config.GetHardwareCounters(), counter => counter == HardwareCounter.CacheMisses); + Assert.Single(config.GetHardwareCounters(), counter => counter == HardwareCounter.InstructionRetired); } [Fact] diff --git a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs index 2eae58c7c1..c68134b7cd 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs @@ -249,10 +249,10 @@ public void MissingDependencyIsNotAddedWhenItIsAlreadyPresent() [Fact] public void WhenTwoConfigsAreAddedTheRegularJobsAreJustAdded() { - var configWithClrJob = CreateConfigFromJobs(Job.Default.WithRuntime(CoreRuntime.Core21)); - var configWithCoreJob = CreateConfigFromJobs(Job.Default.WithRuntime(ClrRuntime.Net462)); + var configWithClrJob = CreateConfigFromJobs(Job.Default.WithRuntime(ClrRuntime.Net462)); + var configWithCoreJob = CreateConfigFromJobs(Job.Default.WithRuntime(CoreRuntime.Core80)); - foreach (var added in AddLeftToTheRightAndRightToTheLef(configWithClrJob, configWithCoreJob)) + foreach (var added in AddLeftToTheRightAndRightToTheLef(configWithCoreJob, configWithClrJob)) { var runnableJobs = added.GetJobs(); @@ -269,7 +269,7 @@ public void WhenTwoConfigsAreAddedTheMutatorJobsAreAppliedToAllOtherJobs() var configWithMutatorJob = CreateConfigFromJobs(Job.Default.WithWarmupCount(warmupCount).AsMutator()); var configWithTwoStandardJobs = CreateConfigFromJobs( Job.Default.WithRuntime(ClrRuntime.Net462), - Job.Default.WithRuntime(CoreRuntime.Core21)); + Job.Default.WithRuntime(CoreRuntime.Core80)); foreach (var added in AddLeftToTheRightAndRightToTheLef(configWithTwoStandardJobs, configWithMutatorJob)) { @@ -312,7 +312,7 @@ public void WhenTwoConfigsAreAddedTheMutatorJobsAreAppliedToDefaultJobIfCustomDe Assert.Equal(warmupCount, mergedJob.Run.WarmupCount); Assert.False(mergedJob.Meta.IsDefault); // after the merge the "child" job becomes a standard job Assert.False(mergedJob.Meta.IsMutator); // after the merge the "child" job becomes a standard job - Assert.Single(mergedJob.GetCharacteristicsWithValues(), changedCharacteristic => ReferenceEquals(changedCharacteristic, Jobs.RunMode.WarmupCountCharacteristic)); + Assert.Single(mergedJob.GetCharacteristicsWithValues(), changedCharacteristic => ReferenceEquals(changedCharacteristic, BenchmarkDotNet.Jobs.RunMode.WarmupCountCharacteristic)); } } @@ -376,6 +376,45 @@ public void WhenTwoCustomTimeoutsAreProvidedTheLongerOneIsUsed(bool direction) Assert.Equal(TimeSpan.FromSeconds(2), final.BuildTimeout); } + [Fact] + public void WhenWakeLockIsNotSpecifiedTheDefaultValueIsUsed() + { + var mutable = ManualConfig.CreateEmpty(); + var final = ImmutableConfigBuilder.Create(mutable); + Assert.Equal(DefaultConfig.Instance.WakeLock, final.WakeLock); + } + + [Fact] + public void CustomWakeLockHasPrecedenceOverDefaultWakeLock() + { + WakeLockType customWakeLock = WakeLockType.Display; + var mutable = ManualConfig.CreateEmpty().WithWakeLock(customWakeLock); + var final = ImmutableConfigBuilder.Create(mutable); + Assert.Equal(customWakeLock, final.WakeLock); + } + + [Theory] + [InlineData(WakeLockType.None, WakeLockType.None, WakeLockType.None)] + [InlineData(WakeLockType.System, WakeLockType.None, WakeLockType.None)] + [InlineData(WakeLockType.Display, WakeLockType.None, WakeLockType.Display)] + [InlineData(WakeLockType.None, WakeLockType.System, WakeLockType.None)] + [InlineData(WakeLockType.System, WakeLockType.System, WakeLockType.System)] + [InlineData(WakeLockType.Display, WakeLockType.System, WakeLockType.Display)] + [InlineData(WakeLockType.None, WakeLockType.Display, WakeLockType.Display)] + [InlineData(WakeLockType.System, WakeLockType.Display, WakeLockType.Display)] + [InlineData(WakeLockType.Display, WakeLockType.Display, WakeLockType.Display)] + public void WhenTwoCustomWakeLocksAreProvidedDisplayBeatsNoneBeatsDefault( + WakeLockType left, WakeLockType right, WakeLockType expected) + { + var l = ManualConfig.CreateEmpty().WithWakeLock(left); + var r = ManualConfig.CreateEmpty().WithWakeLock(right); + + l.Add(r); + + var final = ImmutableConfigBuilder.Create(l); + Assert.Equal(expected, final.WakeLock); + } + private static ManualConfig CreateConfigFromJobs(params Job[] jobs) { var config = ManualConfig.CreateEmpty(); diff --git a/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs b/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs index 050b84de7c..d7fb7519f3 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs @@ -403,7 +403,7 @@ public static void Test06CharacteristicHacks() [Fact] public static void MutatorAppliedToOtherJobOverwritesOnlyTheConfiguredSettings() { - var jobBefore = Job.Default.WithRuntime(CoreRuntime.Core30); // this is a default job with Runtime set to Core + var jobBefore = Job.Default.WithRuntime(CoreRuntime.Core80); // this is a default job with Runtime set to Core var copy = jobBefore.UnfreezeCopy(); Assert.False(copy.HasValue(RunMode.MaxIterationCountCharacteristic)); @@ -441,6 +441,7 @@ public static void AllJobModesPropertyNamesMatchCharacteristicNames() // it's ma } [Fact] + [Obsolete] public static void WithNuGet() { var j = new Job("SomeId"); diff --git a/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs b/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs index 706e74b561..ef1225432e 100644 --- a/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs +++ b/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs @@ -26,7 +26,7 @@ public class CsProjGeneratorTests [Theory] [InlineData("net471", false)] - [InlineData("netcoreapp3.0", true)] + [InlineData("netcoreapp3.1", true)] public void ItsPossibleToCustomizeProjectSdkBasedOnProjectSdkFromTheProjectFile(string targetFrameworkMoniker, bool isNetCore) { const string withCustomProjectSdk = @" @@ -41,7 +41,7 @@ public void ItsImpossibleToCustomizeProjectSdkForFullFrameworkAppsBasedOnTheImpo { const string withCustomProjectImport = @" - + "; AssertParsedSdkName(withCustomProjectImport, "net471", "Microsoft.NET.Sdk", false); @@ -52,10 +52,10 @@ public void ItsPossibleToCustomizeProjectSdkForNetCoreAppsBasedOnTheImportOfSdk( { const string withCustomProjectImport = @" - + "; - AssertParsedSdkName(withCustomProjectImport, "netcoreapp3.0", "Microsoft.NET.Sdk.WindowsDesktop", true); + AssertParsedSdkName(withCustomProjectImport, "netcoreapp3.1", "Microsoft.NET.Sdk.WindowsDesktop", true); } [AssertionMethod] @@ -87,7 +87,7 @@ public void UseWpfSettingGetsCopied() "; - var sut = new CsProjGenerator("netcoreapp3.0", null, null, null, true); + var sut = new CsProjGenerator("netcoreapp3.1", null, null, null, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(withUseWpfTrue); @@ -117,7 +117,7 @@ public void SettingsFromPropsFileImportedUsingAbsolutePathGetCopies() "; - var sut = new CsProjGenerator("netcoreapp3.0", null, null, null, true); + var sut = new CsProjGenerator("netcoreapp3.1", null, null, null, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(importingAbsolutePath); @@ -149,7 +149,7 @@ public void SettingsFromPropsFileImportedUsingRelativePathGetCopies() "; - var sut = new CsProjGenerator("netcoreapp3.0", null, null, null, true); + var sut = new CsProjGenerator("netcoreapp3.1", null, null, null, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(importingRelativePath); @@ -171,7 +171,7 @@ public void RuntimeHostConfigurationOptionIsCopied() {runtimeHostConfigurationOptionChunk} "; - var sut = new CsProjGenerator("netcoreapp3.0", null, null, null, true); + var sut = new CsProjGenerator("netcoreapp3.1", null, null, null, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(source); @@ -196,7 +196,7 @@ public void TheDefaultFilePathShouldBeUsedWhenAnAssemblyLocationIsEmpty() var benchmarkCase = BenchmarkCase.Create(target, Job.Default, null, config); var benchmarks = new[] { new BenchmarkBuildInfo(benchmarkCase, config.CreateImmutableConfig(), 999) }; - var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.0", null, null, null, true); + var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.1", null, null, null, true); string binariesPath = projectGenerator.ResolvePathForBinaries(new BuildPartition(benchmarks, new Resolver()), programName); string expectedPath = Path.Combine(Path.Combine(Directory.GetCurrentDirectory(), "BenchmarkDotNet.Bin"), programName); @@ -210,7 +210,7 @@ public void TestAssemblyFilePathIsUsedWhenTheAssemblyLocationIsNotEmpty() var target = new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo); var benchmarkCase = BenchmarkCase.Create(target, Job.Default, null, ManualConfig.CreateEmpty().CreateImmutableConfig()); var benchmarks = new[] { new BenchmarkBuildInfo(benchmarkCase, ManualConfig.CreateEmpty().CreateImmutableConfig(), 0) }; - var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.0", null, null, null, true); + var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.1", null, null, null, true); var buildPartition = new BuildPartition(benchmarks, new Resolver()); string binariesPath = projectGenerator.ResolvePathForBinaries(buildPartition, programName); diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/CpuInfoFormatterTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/CpuInfoFormatterTests.cs new file mode 100644 index 0000000000..4f487a7b3b --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/CpuInfoFormatterTests.cs @@ -0,0 +1,37 @@ +using System.Text; +using System.Threading.Tasks; +using BenchmarkDotNet.Tests.Infra; +using Perfolizer.Helpers; +using Perfolizer.Models; +using VerifyXunit; +using Xunit; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +[Collection("VerifyTests")] +public class CpuInfoFormatterTests +{ + [Fact] + public Task FormatTest() + { + var captions = new StringBuilder(); + foreach (var processorName in new[] { null, "", "Intel" }) + foreach (var physicalProcessorCount in new int?[] { null, 0, 1, 2 }) + foreach (var physicalCoreCount in new int?[] { null, 0, 1, 2 }) + foreach (var logicalCoreCount in new int?[] { null, 0, 1, 2 }) + { + var cpu = new CpuInfo + { + ProcessorName = processorName, + PhysicalProcessorCount = physicalProcessorCount, + PhysicalCoreCount = physicalCoreCount, + LogicalCoreCount = logicalCoreCount, + }; + + captions.AppendLine(cpu.ToFullBrandName()); + } + + var settings = VerifyHelper.Create(); + return Verifier.Verify(captions.ToString(), settings); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/LinuxCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/LinuxCpuInfoParserTests.cs new file mode 100644 index 0000000000..0d4de2d9cc --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/LinuxCpuInfoParserTests.cs @@ -0,0 +1,173 @@ +using BenchmarkDotNet.Detectors.Cpu.Linux; +using Perfolizer.Models; +using Xunit; +using Xunit.Abstractions; +using static Perfolizer.Horology.Frequency; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +// ReSharper disable StringLiteralTypo +public class LinuxCpuInfoParserTests(ITestOutputHelper output) +{ + private ITestOutputHelper Output { get; } = output; + + [Fact] + public void EmptyTest() + { + var actual = LinuxCpuInfoParser.Parse("", ""); + var expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void MalformedTest() + { + var actual = LinuxCpuInfoParser.Parse("malformedkey: malformedvalue\n\nmalformedkey2: malformedvalue2", string.Empty); + var expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void TwoProcessorWithDifferentCoresCountTest() + { + string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoProcessorWithDifferentCoresCount.txt"); + var actual = LinuxCpuInfoParser.Parse(cpuInfo, string.Empty); + var expected = new CpuInfo + { + ProcessorName = "Unknown processor with 2 cores and hyper threading, Unknown processor with 4 cores", + PhysicalProcessorCount = 2, + PhysicalCoreCount = 6, + LogicalCoreCount = 8, + NominalFrequencyHz = 2_500_000_000, + MaxFrequencyHz = 2_500_000_000 + }; + Output.AssertEqual(expected, actual); + } + + + [Fact] + public void RealOneProcessorTwoCoresTest() + { + string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoRealOneProcessorTwoCores.txt"); + var actual = LinuxCpuInfoParser.Parse(cpuInfo, string.Empty); + var expected = new CpuInfo + { + ProcessorName = "Intel(R) Core(TM) i5-6200U CPU @ 2.30GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 2, + LogicalCoreCount = 4, + NominalFrequencyHz = 2_300_000_000, + MaxFrequencyHz = 2_300_000_000 + }; + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealOneProcessorFourCoresTest() + { + string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoRealOneProcessorFourCores.txt"); + var actual = LinuxCpuInfoParser.Parse(cpuInfo, string.Empty); + var expected = new CpuInfo + { + ProcessorName = "Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 4, + LogicalCoreCount = 8, + NominalFrequencyHz = 2_494_300_000, + MaxFrequencyHz = 2_500_000_000 + }; + Output.AssertEqual(expected, actual); + } + + // https://github.com/dotnet/BenchmarkDotNet/issues/2577 + [Fact] + public void Issue2577Test() + { + const string cpuInfo = + """ + processor : 0 + BogoMIPS : 50.00 + Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm lrcpc dcpop asimddp + CPU implementer : 0x41 + CPU architecture: 8 + CPU variant : 0x3 + CPU part : 0xd0c + CPU revision : 1 + """; + const string lscpu = + """ + Architecture: aarch64 + CPU op-mode(s): 32-bit, 64-bit + Byte Order: Little Endian + CPU(s): 16 + On-line CPU(s) list: 0-15 + Vendor ID: ARM + Model name: Neoverse-N1 + Model: 1 + Thread(s) per core: 1 + Core(s) per socket: 16 + Socket(s): 1 + Stepping: r3p1 + BogoMIPS: 50.00 + Flags: fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm lrcpc dcpop asimddp + """; + var actual = LinuxCpuInfoParser.Parse(cpuInfo, lscpu); + var expected = new CpuInfo { ProcessorName = "Neoverse-N1", PhysicalCoreCount = 16 }; + Output.AssertEqual(expected, actual); + } + + [Fact] + public void AmdRyzen9_7950X() + { + string cpuInfo = TestHelper.ReadTestFile("ryzen9-cpuinfo.txt"); + const string lscpu = + """ + Architecture: x86_64 + CPU op-mode(s): 32-bit, 64-bit + Address sizes: 48 bits physical, 48 bits virtual + Byte Order: Little Endian + CPU(s): 32 + On-line CPU(s) list: 0-31 + Vendor ID: AuthenticAMD + Model name: AMD Ryzen 9 7950X 16-Core Processor + CPU family: 25 + Model: 97 + Thread(s) per core: 2 + Core(s) per socket: 16 + Socket(s): 1 + Stepping: 2 + CPU(s) scaling MHz: 41% + CPU max MHz: 5881.0000 + CPU min MHz: 400.0000 + BogoMIPS: 8983.23 + Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl + pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb + bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512 + cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean + flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succo + r smca fsrm flush_l1d + """; + var actual = LinuxCpuInfoParser.Parse(cpuInfo, lscpu); + var expected = new CpuInfo + { + ProcessorName = "AMD Ryzen 9 7950X 16-Core Processor", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 16, + LogicalCoreCount = 32, + NominalFrequencyHz = 400_000_000, + MaxFrequencyHz = 5_881_000_000 + }; + Output.AssertEqual(expected, actual); + } + + [Theory] + [InlineData("Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", 2.50)] + [InlineData("Intel(R) Core(TM) i5-6200U CPU @ 2.30GHz", 2.30)] + [InlineData("Unknown processor with 2 cores and hyper threading, Unknown processor with 4 cores", 0)] + [InlineData("Intel(R) Core(TM) i5-2500 CPU @ 3.30GHz", 3.30)] + public void ParseFrequencyFromBrandStringTests(string brandString, double expectedGHz) + { + var frequency = LinuxCpuInfoParser.ParseFrequencyFromBrandString(brandString) ?? Zero; + Assert.Equal(FromGHz(expectedGHz), frequency); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/PowershellWmiCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/PowershellWmiCpuInfoParserTests.cs new file mode 100644 index 0000000000..a781d70021 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/PowershellWmiCpuInfoParserTests.cs @@ -0,0 +1,122 @@ +using BenchmarkDotNet.Detectors.Cpu.Windows; +using Perfolizer.Models; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +public class PowershellWmiCpuInfoParserTests(ITestOutputHelper output) +{ + private ITestOutputHelper Output { get; } = output; + + + [Fact] + public void EmptyTest() + { + CpuInfo? actual = PowershellWmiCpuInfoParser.Parse(string.Empty); + CpuInfo expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + + [Fact] + public void MalformedTest() + { + CpuInfo? actual = PowershellWmiCpuInfoParser + .Parse("malformedkey=malformedvalue\n\nmalformedkey2=malformedvalue2"); + CpuInfo expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealTwoProcessorEightCoresTest() + { + const string cpuInfo = + """ + MaxClockSpeed:2400 + Name:Intel(R) Xeon(R) CPU E5-2630 v3 + NumberOfCores:8 + NumberOfLogicalProcessors:16 + + + MaxClockSpeed:2400 + Name:Intel(R) Xeon(R) CPU E5-2630 v3 + NumberOfCores:8 + NumberOfLogicalProcessors:16 + + """; + CpuInfo? actual = PowershellWmiCpuInfoParser.Parse(cpuInfo); + + CpuInfo expected = new CpuInfo + { + ProcessorName = "Intel(R) Xeon(R) CPU E5-2630 v3", + PhysicalProcessorCount = 2, + PhysicalCoreCount = 16, + LogicalCoreCount = 32, + NominalFrequencyHz = 2_400_000_000, + MaxFrequencyHz = 2_400_000_000, + }; + + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealTwoProcessorEightCoresWithWmicBugTest() + { + const string cpuInfo = + "\r\r\n" + + "\r\r\n" + + "MaxClockSpeed:3111\r\r\n" + + "Name:Intel(R) Xeon(R) CPU E5-2687W 0\r\r\n" + + "NumberOfCores:8\r\r\n" + + "NumberOfLogicalProcessors:16\r\r\n" + + "\r\r\n" + + "\r\r\n" + + "MaxClockSpeed:3111\r\r\n" + + "Name:Intel(R) Xeon(R) CPU E5-2687W 0\r\r\n" + + "NumberOfCores:8\r\r\n" + + "NumberOfLogicalProcessors:16\r\r\n" + + "\r\r\n" + + "\r\r\n" + + "\r\r\n"; + + CpuInfo? actual = PowershellWmiCpuInfoParser.Parse(cpuInfo); + + CpuInfo expected = new CpuInfo + { + ProcessorName = "Intel(R) Xeon(R) CPU E5-2687W 0", + PhysicalProcessorCount = 2, + PhysicalCoreCount = 16, + LogicalCoreCount = 32, + NominalFrequencyHz = 3_111_000_000, + MaxFrequencyHz = 3_111_000_000, + }; + + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealOneProcessorFourCoresTest() + { + const string cpuInfo = """ + + MaxClockSpeed:2500 + Name:Intel(R) Core(TM) i7-4710MQ + NumberOfCores:4 + NumberOfLogicalProcessors:8 + """; + + CpuInfo? actual = PowershellWmiCpuInfoParser.Parse(cpuInfo); + CpuInfo expected = new CpuInfo + { + ProcessorName = "Intel(R) Core(TM) i7-4710MQ", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 4, + LogicalCoreCount = 8, + NominalFrequencyHz = 2_500_000_000, + MaxFrequencyHz = 2_500_000_000, + }; + + Output.AssertEqual(expected, actual); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/SysctlCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/SysctlCpuInfoParserTests.cs new file mode 100644 index 0000000000..bdf8cbb10e --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/SysctlCpuInfoParserTests.cs @@ -0,0 +1,45 @@ +using BenchmarkDotNet.Detectors.Cpu.macOS; +using Perfolizer.Models; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +// ReSharper disable StringLiteralTypo +public class SysctlCpuInfoParserTests(ITestOutputHelper output) +{ + private ITestOutputHelper Output { get; } = output; + + [Fact] + public void EmptyTest() + { + var actual = SysctlCpuInfoParser.Parse(string.Empty); + var expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void MalformedTest() + { + var actual = SysctlCpuInfoParser.Parse("malformedkey=malformedvalue\n\nmalformedkey2=malformedvalue2"); + var expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealOneProcessorFourCoresTest() + { + string cpuInfo = TestHelper.ReadTestFile("SysctlRealOneProcessorFourCores.txt"); + var actual = SysctlCpuInfoParser.Parse(cpuInfo); + var expected = new CpuInfo + { + ProcessorName = "Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 4, + LogicalCoreCount = 8, + NominalFrequencyHz = 2_200_000_000, + MaxFrequencyHz = 2_200_000_000 + }; + Output.AssertEqual(expected, actual); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoProcessorWithDifferentCoresCount.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoProcessorWithDifferentCoresCount.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoProcessorWithDifferentCoresCount.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoProcessorWithDifferentCoresCount.txt diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoRealOneProcessorFourCores.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoRealOneProcessorFourCores.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoRealOneProcessorFourCores.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoRealOneProcessorFourCores.txt diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoRealOneProcessorTwoCores.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoRealOneProcessorTwoCores.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/ProcCpuInfoRealOneProcessorTwoCores.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ProcCpuInfoRealOneProcessorTwoCores.txt diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/SysctlRealOneProcessorFourCores.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/SysctlRealOneProcessorFourCores.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/TestFiles/SysctlRealOneProcessorFourCores.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/SysctlRealOneProcessorFourCores.txt diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ryzen9-cpuinfo.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ryzen9-cpuinfo.txt new file mode 100644 index 0000000000..d942a8d484 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestFiles/ryzen9-cpuinfo.txt @@ -0,0 +1,896 @@ +processor : 0 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5346.955 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 0 +cpu cores : 16 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 1 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5487.348 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 1 +cpu cores : 16 +apicid : 2 +initial apicid : 2 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 2 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5479.603 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 2 +cpu cores : 16 +apicid : 4 +initial apicid : 4 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 3 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5601.456 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 3 +cpu cores : 16 +apicid : 6 +initial apicid : 6 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 4 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5410.513 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 4 +cpu cores : 16 +apicid : 8 +initial apicid : 8 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 5 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5493.255 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 5 +cpu cores : 16 +apicid : 10 +initial apicid : 10 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 6 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4422.161 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 6 +cpu cores : 16 +apicid : 12 +initial apicid : 12 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 7 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5505.019 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 7 +cpu cores : 16 +apicid : 14 +initial apicid : 14 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 8 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5414.920 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 8 +cpu cores : 16 +apicid : 16 +initial apicid : 16 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 9 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5060.967 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 9 +cpu cores : 16 +apicid : 18 +initial apicid : 18 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 10 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 3352.555 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 10 +cpu cores : 16 +apicid : 20 +initial apicid : 20 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 11 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5400.772 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 11 +cpu cores : 16 +apicid : 22 +initial apicid : 22 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 12 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5415.333 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 12 +cpu cores : 16 +apicid : 24 +initial apicid : 24 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 13 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 3236.400 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 13 +cpu cores : 16 +apicid : 26 +initial apicid : 26 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 14 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4519.257 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 14 +cpu cores : 16 +apicid : 28 +initial apicid : 28 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 15 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5414.841 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 15 +cpu cores : 16 +apicid : 30 +initial apicid : 30 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 16 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4738.471 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 0 +cpu cores : 16 +apicid : 1 +initial apicid : 1 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 17 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5599.000 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 1 +cpu cores : 16 +apicid : 3 +initial apicid : 3 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 18 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5477.067 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 2 +cpu cores : 16 +apicid : 5 +initial apicid : 5 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 19 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5507.543 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 3 +cpu cores : 16 +apicid : 7 +initial apicid : 7 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 20 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4401.379 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 4 +cpu cores : 16 +apicid : 9 +initial apicid : 9 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 21 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 400.000 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 5 +cpu cores : 16 +apicid : 11 +initial apicid : 11 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 22 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5607.945 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 6 +cpu cores : 16 +apicid : 13 +initial apicid : 13 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 23 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5280.720 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 7 +cpu cores : 16 +apicid : 15 +initial apicid : 15 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 24 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5414.893 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 8 +cpu cores : 16 +apicid : 17 +initial apicid : 17 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 25 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5414.888 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 9 +cpu cores : 16 +apicid : 19 +initial apicid : 19 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 26 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5385.641 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 10 +cpu cores : 16 +apicid : 21 +initial apicid : 21 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 27 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 3130.265 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 11 +cpu cores : 16 +apicid : 23 +initial apicid : 23 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 28 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5400.876 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 12 +cpu cores : 16 +apicid : 25 +initial apicid : 25 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 29 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 4279.794 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 13 +cpu cores : 16 +apicid : 27 +initial apicid : 27 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 30 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 3124.980 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 14 +cpu cores : 16 +apicid : 29 +initial apicid : 29 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + +processor : 31 +vendor_id : AuthenticAMD +cpu family : 25 +model : 97 +model name : AMD Ryzen 9 7950X 16-Core Processor +stepping : 2 +microcode : 0xa601201 +cpu MHz : 5389.561 +cache size : 1024 KB +physical id : 0 +siblings : 32 +core id : 15 +cpu cores : 16 +apicid : 31 +initial apicid : 31 +fpu : yes +fpu_exception : yes +cpuid level : 16 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d +bugs : sysret_ss_attrs spectre_v1 spectre_v2 spec_store_bypass srso +bogomips : 8983.23 +TLB size : 3584 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] + diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestHelper.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestHelper.cs new file mode 100644 index 0000000000..91e6e686a3 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/TestHelper.cs @@ -0,0 +1,38 @@ +using System.IO; +using System.Reflection; +using JetBrains.Annotations; +using Perfolizer.Helpers; +using Perfolizer.Models; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +public static class TestHelper +{ + public static string ReadTestFile(string name) + { + var assembly = typeof(TestHelper).GetTypeInfo().Assembly; + string resourceName = $"{typeof(TestHelper).Namespace}.TestFiles.{name}"; + + using var stream = assembly.GetManifestResourceStream(resourceName); + if (stream == null) + throw new FileNotFoundException($"Resource {resourceName} not found in {assembly.FullName}"); + + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } + + [AssertionMethod] + public static void AssertEqual(this ITestOutputHelper output, CpuInfo expected, CpuInfo actual) + { + output.WriteLine($"Expected : {expected.ToFullBrandName()}"); + output.WriteLine($"Actual : {actual.ToFullBrandName()}"); + Assert.Equal(expected.ProcessorName, actual.ProcessorName); + Assert.Equal(expected.PhysicalProcessorCount, actual.PhysicalProcessorCount); + Assert.Equal(expected.PhysicalCoreCount, actual.PhysicalCoreCount); + Assert.Equal(expected.LogicalCoreCount, actual.LogicalCoreCount); + Assert.Equal(expected.NominalFrequency(), actual.NominalFrequency()); + Assert.Equal(expected.MaxFrequency(), actual.MaxFrequency()); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/VerifiedFiles/CpuInfoFormatterTests.FormatTest.verified.txt b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/VerifiedFiles/CpuInfoFormatterTests.FormatTest.verified.txt similarity index 100% rename from tests/BenchmarkDotNet.Tests/Portability/Cpu/VerifiedFiles/CpuInfoFormatterTests.FormatTest.verified.txt rename to tests/BenchmarkDotNet.Tests/Detectors/Cpu/VerifiedFiles/CpuInfoFormatterTests.FormatTest.verified.txt diff --git a/tests/BenchmarkDotNet.Tests/Detectors/Cpu/WmicCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/WmicCpuInfoParserTests.cs new file mode 100644 index 0000000000..8903f98360 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Detectors/Cpu/WmicCpuInfoParserTests.cs @@ -0,0 +1,115 @@ +using BenchmarkDotNet.Detectors.Cpu.Windows; +using Perfolizer.Models; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Detectors.Cpu; + +// ReSharper disable StringLiteralTypo +public class WmicCpuInfoParserTests(ITestOutputHelper output) +{ + private ITestOutputHelper Output { get; } = output; + + [Fact] + public void EmptyTest() + { + var actual = WmicCpuInfoParser.Parse(string.Empty); + var expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void MalformedTest() + { + var actual = WmicCpuInfoParser.Parse("malformedkey=malformedvalue\n\nmalformedkey2=malformedvalue2"); + var expected = new CpuInfo(); + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealTwoProcessorEightCoresTest() + { + const string cpuInfo = @" + +MaxClockSpeed=2400 +Name=Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz +NumberOfCores=8 +NumberOfLogicalProcessors=16 + + +MaxClockSpeed=2400 +Name=Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz +NumberOfCores=8 +NumberOfLogicalProcessors=16 + +"; + var actual = WmicCpuInfoParser.Parse(cpuInfo); + var expected = new CpuInfo + { + ProcessorName = "Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz", + PhysicalProcessorCount = 2, + PhysicalCoreCount = 16, + LogicalCoreCount = 32, + NominalFrequencyHz = 2_400_000_000, + MaxFrequencyHz = 2_400_000_000, + }; + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealTwoProcessorEightCoresWithWmicBugTest() + { + const string cpuInfo = + "\r\r\n" + + "\r\r\n" + + "MaxClockSpeed=3111\r\r\n" + + "Name=Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz\r\r\n" + + "NumberOfCores=8\r\r\n" + + "NumberOfLogicalProcessors=16\r\r\n" + + "\r\r\n" + + "\r\r\n" + + "MaxClockSpeed=3111\r\r\n" + + "Name=Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz\r\r\n" + + "NumberOfCores=8\r\r\n" + + "NumberOfLogicalProcessors=16\r\r\n" + + "\r\r\n" + + "\r\r\n" + + "\r\r\n"; + var actual = WmicCpuInfoParser.Parse(cpuInfo); + var expected = new CpuInfo + { + ProcessorName = "Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz", + PhysicalProcessorCount = 2, + PhysicalCoreCount = 16, + LogicalCoreCount = 32, + NominalFrequencyHz = 3_111_000_000, + MaxFrequencyHz = 3_111_000_000, + }; + Output.AssertEqual(expected, actual); + } + + [Fact] + public void RealOneProcessorFourCoresTest() + { + const string cpuInfo = @" + +MaxClockSpeed=2500 +Name=Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz +NumberOfCores=4 +NumberOfLogicalProcessors=8 + +"; + + var actual = WmicCpuInfoParser.Parse(cpuInfo); + var expected = new CpuInfo + { + ProcessorName = "Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", + PhysicalProcessorCount = 1, + PhysicalCoreCount = 4, + LogicalCoreCount = 8, + NominalFrequencyHz = 2_500_000_000, + MaxFrequencyHz = 2_500_000_000, + }; + Output.AssertEqual(expected, actual); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineGeneralStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineActualStageTests.cs similarity index 85% rename from tests/BenchmarkDotNet.Tests/Engine/EngineGeneralStageTests.cs rename to tests/BenchmarkDotNet.Tests/Engine/EngineActualStageTests.cs index 5811a0f68c..97078eb953 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineGeneralStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineActualStageTests.cs @@ -36,17 +36,14 @@ private void AutoTest(Func measure, int min, int ma if (max == -1) max = min; var job = Job.Default; - var stage = CreateStage(job, measure); - var measurements = stage.Run(1, iterationMode, true, 1); + var engine = new MockEngine(output, job, measure); + var stage = iterationMode == IterationMode.Overhead + ? EngineActualStage.GetOverhead(engine) + : EngineActualStage.GetWorkload(engine, RunStrategy.Throughput); + var (_, measurements) = engine.Run(stage); int count = measurements.Count; output.WriteLine($"MeasurementCount = {count} (Min= {min}, Max = {max})"); Assert.InRange(count, min, max); } - - private EngineActualStage CreateStage(Job job, Func measure) - { - var engine = new MockEngine(output, job, measure); - return new EngineActualStage(engine); - } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs index d5b7b8c65c..fd3f4887ff 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; using BenchmarkDotNet.Tests.Mocks; using Perfolizer.Horology; using Xunit; @@ -46,8 +48,8 @@ private void AutoTest(Frequency clockFrequency, TimeInterval operationTime, doub Infrastructure = { Clock = new MockClock(clockFrequency) }, Accuracy = { MaxRelativeError = maxRelativeError } }.Freeze(); - var stage = CreateStage(job, data => data.InvokeCount * operationTime); - long invokeCount = stage.Run().PerfectInvocationCount; + var engine = new MockEngine(output, job, data => data.InvokeCount * operationTime); + var (invokeCount, _) = engine.Run(EnginePilotStage.GetStage(engine)); output.WriteLine($"InvokeCount = {invokeCount} (Min= {minInvokeCount}, Max = {MaxPossibleInvokeCount})"); Assert.InRange(invokeCount, minInvokeCount, MaxPossibleInvokeCount); } @@ -59,16 +61,10 @@ private void SpecificTest(TimeInterval iterationTime, TimeInterval operationTime Infrastructure = { Clock = new MockClock(Frequency.MHz) }, Run = { IterationTime = iterationTime } }.Freeze(); - var stage = CreateStage(job, data => data.InvokeCount * operationTime); - long invokeCount = stage.Run().PerfectInvocationCount; + var engine = new MockEngine(output, job, data => data.InvokeCount * operationTime); + var (invokeCount, _) = engine.Run(EnginePilotStage.GetStage(engine)); output.WriteLine($"InvokeCount = {invokeCount} (Min= {minInvokeCount}, Max = {maxInvokeCount})"); Assert.InRange(invokeCount, minInvokeCount, maxInvokeCount); } - - private EnginePilotStage CreateStage(Job job, Func measure) - { - var engine = new MockEngine(output, job, measure); - return new EnginePilotStage(engine); - } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs index 1f966fbcaa..b9c465b87b 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineWarmupStageTests.cs @@ -16,7 +16,7 @@ public class EngineWarmupStageTests { private const int MinIterationCount = EngineResolver.DefaultMinWarmupIterationCount; private const int MaxIterationCount = EngineResolver.DefaultMaxWarmupIterationCount; - private const int MaxOverheadIterationCount = DefaultStoppingCriteriaFactory.MaxOverheadIterationCount; + private const int MaxOverheadIterationCount = EngineWarmupStage.MaxOverheadIterationCount; private readonly ITestOutputHelper output; @@ -85,17 +85,14 @@ private void AutoTest(Func measure, int min, int ma { if (max == -1) max = min; - var stage = CreateStage(job ?? Job.Default, measure); - var measurements = stage.Run(1, mode, 1, RunStrategy.Throughput); + var engine = new MockEngine(output, job ?? Job.Default, measure); + var stage = mode == IterationMode.Overhead + ? EngineWarmupStage.GetOverhead() + : EngineWarmupStage.GetWorkload(engine, RunStrategy.Throughput); + var (_, measurements) = engine.Run(stage); int count = measurements.Count; output.WriteLine($"MeasurementCount = {count} (Min= {min}, Max = {max})"); Assert.InRange(count, min, max); } - - private EngineWarmupStage CreateStage(Job job, Func measure) - { - var engine = new MockEngine(output, job, measure); - return new EngineWarmupStage(engine); - } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/AutoWarmupStoppingCriteriaTests.cs b/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/AutoWarmupStoppingCriteriaTests.cs deleted file mode 100644 index 39eb94a225..0000000000 --- a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/AutoWarmupStoppingCriteriaTests.cs +++ /dev/null @@ -1,80 +0,0 @@ -using System.Collections.Generic; -using BenchmarkDotNet.Engines; -using JetBrains.Annotations; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Engine.StoppingCriteria -{ - public class AutoWarmupStoppingCriteriaTests : StoppingCriteriaTestsBase - { - public AutoWarmupStoppingCriteriaTests(ITestOutputHelper output) : base(output) { } - - [Theory] - [MemberData(nameof(EvaluateData))] - public void EvaluateTest(int minIterationCount, int maxIterationCount, int minFluctuationCount, double[] values, int expectedCount) - { - var criteria = new AutoWarmupStoppingCriteria(minIterationCount, maxIterationCount, minFluctuationCount); - ResolutionsAreCorrect(criteria, values, expectedCount); - } - - public static IEnumerable EvaluateData() - { - yield return new object[] { 0, 8, 4, new double[] { 0, 10, 0, 10, 0, 10, 0, 10, 0, 10 }, 5 }; - yield return new object[] { 0, 5, 1, new double[] { 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 }, 5 }; - yield return new object[] { 0, 5, 1, new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 2 }; - yield return new object[] { 0, 600, 2, new double[] { 10, 9, 8, 7, 8, 9, 8, 7, 8, 9 }, 7 }; - yield return new object[] { 5, 8, 0, new double[] { 0, 10, 0, 10, 0, 10, 0, 10, 0, 10 }, 5 }; - } - - [Theory] - [InlineData(0, 0, 0, "AutoWarmupStoppingCriteria(minIterationCount=0, maxIterationCount=0, minFluctuationCount=0)")] - [InlineData(1, 22, 333, "AutoWarmupStoppingCriteria(minIterationCount=1, maxIterationCount=22, minFluctuationCount=333)")] - public void AutoWarmupTitleTest(int minIterationCount, int maxIterationCount, int minFluctuationCount, string expectedTitle) - { - var criteria = new AutoWarmupStoppingCriteria(minIterationCount, maxIterationCount, minFluctuationCount); - Assert.Equal(expectedTitle, criteria.Title); - } - - [Theory] - [MemberData(nameof(WarningsDataNames))] - public void WarningsTest(string dataName) - { - var (criteria, expectedWarnings) = WarningsData[dataName]; - Output.WriteLine("Criteria:" + criteria.Title); - Assert.Equal(expectedWarnings.Messages, criteria.Warnings); - } - - private static readonly IDictionary WarningsData = new Dictionary - { - { - "0/0/0", new WaringTestData( - new AutoWarmupStoppingCriteria(0, 0, 0), - Warnings.Empty) - }, - { - "5/4/0", new WaringTestData( - new AutoWarmupStoppingCriteria(5, 4, 0), - new Warnings("Min Iteration Count (0) is greater than Max Iteration Count (4)")) - }, - { - "-5/0/0", new WaringTestData( - new AutoWarmupStoppingCriteria(-5, 0, 0), - new Warnings("Min Iteration Count (-5) is negative")) - }, - { - "0/0/-5", new WaringTestData( - new AutoWarmupStoppingCriteria(0, 0, -5), - new Warnings("Min Fluctuation Count (-5) is negative")) - }, - { - "0/-5/0", new WaringTestData( - new AutoWarmupStoppingCriteria(0, -5, 0), - new Warnings("Max Iteration Count (-5) is negative", "Min Iteration Count (0) is greater than Max Iteration Count (-5)")) - } - }; - - [UsedImplicitly] - public static TheoryData WarningsDataNames = TheoryDataHelper.Create(WarningsData.Keys); - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/FixedStoppingCriteriaTests.cs b/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/FixedStoppingCriteriaTests.cs deleted file mode 100644 index 0a352d8c74..0000000000 --- a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/FixedStoppingCriteriaTests.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using BenchmarkDotNet.Engines; -using JetBrains.Annotations; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Engine.StoppingCriteria -{ - public class FixedStoppingCriteriaTests : StoppingCriteriaTestsBase - { - public FixedStoppingCriteriaTests(ITestOutputHelper output) : base(output) { } - - [Theory] - [InlineData(1)] - [InlineData(2)] - [InlineData(10)] - [InlineData(100)] - public void EvaluateTest(int expectedCount) - { - var values = Enumerable.Range(0, expectedCount * 2).Select(x => (double) x).ToArray(); - var criteria = new FixedStoppingCriteria(expectedCount); - ResolutionsAreCorrect(criteria, values, expectedCount); - } - - [Theory] - [InlineData(0, "FixedStoppingCriteria(iterationCount=0)")] - [InlineData(1, "FixedStoppingCriteria(iterationCount=1)")] - [InlineData(42, "FixedStoppingCriteria(iterationCount=42)")] - public void TitleTest(int iterationTitle, string expectedTitle) - { - var criteria = new FixedStoppingCriteria(iterationTitle); - Assert.Equal(expectedTitle, criteria.Title); - } - - [Theory] - [MemberData(nameof(WarningsDataNames))] - public void WarningsTest(string dataName) - { - var (criteria, expectedWarnings) = WarningsData[dataName]; - Output.WriteLine("Criteria:" + criteria.Title); - Assert.Equal(expectedWarnings.Messages, criteria.Warnings); - } - - private static readonly IDictionary WarningsData = new Dictionary - { - { - "0", new WaringTestData( - new FixedStoppingCriteria(0), - Warnings.Empty) - }, - { - "1", new WaringTestData( - new FixedStoppingCriteria(1), - Warnings.Empty) - }, - { - "-1", new WaringTestData( - new FixedStoppingCriteria(-1), - new Warnings("Iteration count (-1) is negative")) - } - }; - - [UsedImplicitly] - public static TheoryData WarningsDataNames = TheoryDataHelper.Create(WarningsData.Keys); - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/StoppingCriteriaTestsBase.cs b/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/StoppingCriteriaTestsBase.cs deleted file mode 100644 index 39b83da67a..0000000000 --- a/tests/BenchmarkDotNet.Tests/Engine/StoppingCriteria/StoppingCriteriaTestsBase.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Reports; -using JetBrains.Annotations; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Engine.StoppingCriteria -{ - [Collection("StoppingCriteriaTests")] - public abstract class StoppingCriteriaTestsBase - { - protected readonly ITestOutputHelper Output; - - protected StoppingCriteriaTestsBase(ITestOutputHelper output) => Output = output; - - [AssertionMethod] - protected void ResolutionsAreCorrect(IStoppingCriteria criteria, double[] values, int expectedCount) - { - var measurements = Generate(values); - for (int iteration = 1; iteration <= measurements.Count; iteration++) - { - var subMeasurements = measurements.Take(iteration).ToList(); - var resolution = criteria.Evaluate(subMeasurements); - bool actualIsFinished = resolution.IsFinished; - bool expectedIsFinished = iteration >= expectedCount; - Output.WriteLine($"#{iteration} " + - $"Expected: {expectedIsFinished}, " + - $"Actual: {actualIsFinished}{(resolution.Message != null ? $" [{resolution.Message}]" : "")}, " + - $"Measurements: <{string.Join(",", values.Take(iteration))}>"); - Assert.Equal(expectedIsFinished, actualIsFinished); - } - } - - private static IReadOnlyList Generate(params double[] values) - { - var measurements = new List(values.Length); - for (int i = 1; i <= values.Length; i++) - measurements.Add(new Measurement(1, IterationMode.Unknown, IterationStage.Unknown, i, 1, values[i - 1])); - return measurements; - } - - public class Warnings - { - public static readonly Warnings Empty = new Warnings(); - - public string[] Messages { get; } - - public Warnings(params string[] messages) => Messages = messages; - - public override string ToString() => $"Messages: {Messages.Length}"; - } - - public class WaringTestData - { - private IStoppingCriteria Criteria { get; } - private Warnings ExpectedWarnings { get; } - - public WaringTestData(IStoppingCriteria criteria, Warnings expectedWarnings) - { - Criteria = criteria; - ExpectedWarnings = expectedWarnings; - } - - public void Deconstruct(out IStoppingCriteria criteria, out Warnings expectedWarnings) - { - criteria = Criteria; - expectedWarnings = ExpectedWarnings; - } - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Environments/HostEnvironmentInfoTests.cs b/tests/BenchmarkDotNet.Tests/Environments/HostEnvironmentInfoTests.cs index 977be28869..a49d9ea2e7 100644 --- a/tests/BenchmarkDotNet.Tests/Environments/HostEnvironmentInfoTests.cs +++ b/tests/BenchmarkDotNet.Tests/Environments/HostEnvironmentInfoTests.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Portability; using BenchmarkDotNet.Tests.Builders; using JetBrains.Annotations; +using Perfolizer.Helpers; using Xunit; namespace BenchmarkDotNet.Tests.Environments @@ -22,7 +23,7 @@ public void ReturnsHypervisorNameWhenItsDetected(string hypervisorName) string line = info.ToFormattedString().First(); string expected = $"{HostEnvironmentInfo.BenchmarkDotNetCaption} v{info.BenchmarkDotNetVersion}, " + - $"{info.OsVersion.Value} ({hypervisor.Name})"; + $"{info.Os.Value.ToBrandString()} ({hypervisor.Name})"; Assert.Equal(expected, line); } @@ -46,7 +47,7 @@ public void DoesntReturnHypervisorNameWhenItsNotDetected() string line = info.ToFormattedString().First(); string expected = $"{HostEnvironmentInfo.BenchmarkDotNetCaption} v{info.BenchmarkDotNetVersion}, " + - $"{info.OsVersion.Value}"; + $"{info.Os.Value.ToBrandString()}"; Assert.Equal(expected, line); } } diff --git a/tests/BenchmarkDotNet.Tests/Environments/OsBrandStringTests.cs b/tests/BenchmarkDotNet.Tests/Environments/OsBrandStringTests.cs deleted file mode 100644 index 5690fbe6b7..0000000000 --- a/tests/BenchmarkDotNet.Tests/Environments/OsBrandStringTests.cs +++ /dev/null @@ -1,70 +0,0 @@ -using BenchmarkDotNet.Environments; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Environments -{ - public class OsBrandStringTests - { - private readonly ITestOutputHelper output; - - public OsBrandStringTests(ITestOutputHelper output) => this.output = output; - - private void Check(string actual, string expected) - { - output.WriteLine("LENGTH : " + actual.Length); - output.WriteLine("ACTUAL : " + actual); - output.WriteLine("EXPECTED : " + expected); - Assert.Equal(expected, actual); - - // The line with OsBrandString is one of the longest lines in the summary. - // When people past in on GitHub, it can be a reason of an ugly horizontal scrollbar. - // To avoid this, we are trying to minimize this line and use the minimum possible number of characters. - // In this test, we check that the length of the OS brand string for typical Windows versions - // is less than 61 characters. - const int maxLength = 61; - Assert.True(actual.Length <= maxLength, $"The brand string name length should be <= {maxLength}"); - } - - [Theory] - [InlineData("6.3.9600", "Windows 8.1 (6.3.9600)")] - [InlineData("10.0.14393", "Windows 10 (10.0.14393/1607/AnniversaryUpdate/Redstone1)")] - public void WindowsIsPrettified(string originalVersion, string prettifiedName) - => Check(OsBrandStringHelper.Prettify("Windows", originalVersion), prettifiedName); - - [Theory] - [InlineData("10.0.10240", 17797, "Windows 10 (10.0.10240.17797/1507/RTM/Threshold1)")] - [InlineData("10.0.10586", 1478, "Windows 10 (10.0.10586.1478/1511/NovemberUpdate/Threshold2)")] - [InlineData("10.0.14393", 2156, "Windows 10 (10.0.14393.2156/1607/AnniversaryUpdate/Redstone1)")] - [InlineData("10.0.15063", 997, "Windows 10 (10.0.15063.997/1703/CreatorsUpdate/Redstone2)")] - [InlineData("10.0.16299", 334, "Windows 10 (10.0.16299.334/1709/FallCreatorsUpdate/Redstone3)")] - [InlineData("10.0.17134", 48, "Windows 10 (10.0.17134.48/1803/April2018Update/Redstone4)")] - [InlineData("10.0.17763", 1, "Windows 10 (10.0.17763.1/1809/October2018Update/Redstone5)")] - [InlineData("10.0.18362", 693, "Windows 10 (10.0.18362.693/1903/May2019Update/19H1)")] - [InlineData("10.0.18363", 657, "Windows 10 (10.0.18363.657/1909/November2019Update/19H2)")] - [InlineData("10.0.19041", 1, "Windows 10 (10.0.19041.1/2004/May2020Update/20H1)")] - [InlineData("10.0.19042", 746, "Windows 10 (10.0.19042.746/20H2/October2020Update)")] - [InlineData("10.0.19043", 964, "Windows 10 (10.0.19043.964/21H1/May2021Update)")] - [InlineData("10.0.19044", 1147, "Windows 10 (10.0.19044.1147/21H2/November2021Update)")] - [InlineData("10.0.19099", 1729, "Windows 10 (10.0.19099.1729)")] - [InlineData("10.0.19045", 0, "Windows 10 (10.0.19045.0/22H2/2022Update)")] - [InlineData("10.0.22000", 348, "Windows 11 (10.0.22000.348/21H2/SunValley)")] - [InlineData("10.0.22518", 1012, "Windows 11 (10.0.22518.1012)")] - [InlineData("10.0.22621", 0, "Windows 11 (10.0.22621.0/22H2/2022Update/SunValley2)")] - [InlineData("10.0.22631", 2428, "Windows 11 (10.0.22631.2428/23H2/2023Update/SunValley3)")] - public void WindowsWithUbrIsPrettified(string originalVersion, int ubr, string prettifiedName) - => Check(OsBrandStringHelper.Prettify("Windows", originalVersion, ubr), prettifiedName); - - [Theory] - [InlineData("macOS 10.12.6 (16G29)", "Darwin 16.7.0", "macOS Sierra 10.12.6 (16G29) [Darwin 16.7.0]")] - [InlineData("macOS 10.13.4 (17E199)", "Darwin 17.5.0", "macOS High Sierra 10.13.4 (17E199) [Darwin 17.5.0]")] - [InlineData("macOS 10.15.4 (19E266)", "Darwin 19.4.0", "macOS Catalina 10.15.4 (19E266) [Darwin 19.4.0]")] - [InlineData("macOS 11.1 (20C69)", "Darwin 20.2.0", "macOS Big Sur 11.1 (20C69) [Darwin 20.2.0]")] - [InlineData("macOS 11.3.1 (20E241)", "Darwin 20.4.0", "macOS Big Sur 11.3.1 (20E241) [Darwin 20.4.0]")] - [InlineData("macOS 12.1 (21C52)", "Darwin 21.2.0", "macOS Monterey 12.1 (21C52) [Darwin 21.2.0]")] - [InlineData("macOS 13.0.1 (22A400)", "Darwin 22.1.0", "macOS Ventura 13.0.1 (22A400) [Darwin 22.1.0]")] - [InlineData("macOS 14.0.0", "Darwin 23.0.0", "macOS Sonoma 14.0.0 [Darwin 23.0.0]")] - public void MacOSXIsPrettified(string systemVersion, string kernelVersion, string prettifiedName) - => Check(OsBrandStringHelper.PrettifyMacOSX(systemVersion, kernelVersion), prettifiedName); - } -} diff --git a/tests/BenchmarkDotNet.Tests/Environments/ProcessorBrandStringTests.cs b/tests/BenchmarkDotNet.Tests/Environments/ProcessorBrandStringTests.cs deleted file mode 100644 index 0bd9fbf608..0000000000 --- a/tests/BenchmarkDotNet.Tests/Environments/ProcessorBrandStringTests.cs +++ /dev/null @@ -1,73 +0,0 @@ -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; -using Xunit; - -namespace BenchmarkDotNet.Tests.Environments -{ - public class ProcessorBrandStringTests - { - [Theory] - [InlineData("Intel(R) Pentium(TM) G4560 CPU @ 3.50GHz", "Intel Pentium G4560 CPU 3.50GHz")] - [InlineData("Intel(R) Core(TM) i7 CPU 970 @ 3.20GHz", "Intel Core i7 CPU 970 3.20GHz (Nehalem)")] // Nehalem/Westmere/Gulftown - [InlineData("Intel(R) Core(TM) i7-920 CPU @ 2.67GHz", "Intel Core i7-920 CPU 2.67GHz (Nehalem)")] - [InlineData("Intel(R) Core(TM) i7-2600 CPU @ 3.40GHz", "Intel Core i7-2600 CPU 3.40GHz (Sandy Bridge)")] - [InlineData("Intel(R) Core(TM) i7-3770 CPU @ 3.40GHz", "Intel Core i7-3770 CPU 3.40GHz (Ivy Bridge)")] - [InlineData("Intel(R) Core(TM) i7-4770K CPU @ 3.50GHz", "Intel Core i7-4770K CPU 3.50GHz (Haswell)")] - [InlineData("Intel(R) Core(TM) i7-5775R CPU @ 3.30GHz", "Intel Core i7-5775R CPU 3.30GHz (Broadwell)")] - [InlineData("Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz", "Intel Core i7-6700HQ CPU 2.60GHz (Skylake)")] - [InlineData("Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz", "Intel Core i7-7700 CPU 3.60GHz (Kaby Lake)")] - [InlineData("Intel(R) Core(TM) i7-8650U CPU @ 1.90GHz ", "Intel Core i7-8650U CPU 1.90GHz (Kaby Lake R)")] - [InlineData("Intel(R) Core(TM) i7-8700K CPU @ 3.70GHz", "Intel Core i7-8700K CPU 3.70GHz (Coffee Lake)")] - public void IntelCoreIsPrettified(string originalName, string prettifiedName) => - Assert.Equal(prettifiedName, ProcessorBrandStringHelper.Prettify(new CpuInfo(originalName, nominalFrequency: null))); - - [Theory] - [InlineData("Intel(R) Pentium(TM) G4560 CPU @ 3.50GHz", "Intel Pentium G4560 CPU 3.50GHz (Max: 3.70GHz)", 3.7)] - [InlineData("Intel(R) Core(TM) i5-2500 CPU @ 3.30GHz", "Intel Core i5-2500 CPU 3.30GHz (Max: 3.70GHz) (Sandy Bridge)", 3.7)] - [InlineData("Intel(R) Core(TM) i7-2600 CPU @ 3.40GHz", "Intel Core i7-2600 CPU 3.40GHz (Max: 3.70GHz) (Sandy Bridge)", 3.7)] - [InlineData("Intel(R) Core(TM) i7-3770 CPU @ 3.40GHz", "Intel Core i7-3770 CPU 3.40GHz (Max: 3.50GHz) (Ivy Bridge)", 3.5)] - [InlineData("Intel(R) Core(TM) i7-4770K CPU @ 3.50GHz", "Intel Core i7-4770K CPU 3.50GHz (Max: 3.60GHz) (Haswell)", 3.6)] - [InlineData("Intel(R) Core(TM) i7-5775R CPU @ 3.30GHz", "Intel Core i7-5775R CPU 3.30GHz (Max: 3.40GHz) (Broadwell)", 3.4)] - public void CoreIsPrettifiedWithDiffFrequencies(string originalName, string prettifiedName, double actualFrequency) - { - Assert.Equal(prettifiedName, ProcessorBrandStringHelper.Prettify(new CpuInfo(originalName, nominalFrequency: Frequency.FromGHz(actualFrequency)), includeMaxFrequency: true)); - } - - [Theory] - [InlineData("AMD Ryzen 7 2700X Eight-Core Processor", "AMD Ryzen 7 2700X 4.10GHz", 4.1, 8, 16)] - [InlineData("AMD Ryzen 7 2700X Eight-Core Processor", "AMD Ryzen 7 2700X Eight-Core Processor 4.10GHz", 4.1, null, null)] - public void AmdIsPrettifiedWithDiffFrequencies(string originalName, string prettifiedName, double actualFrequency, int? physicalCoreCount, int? logicalCoreCount) - { - var cpuInfo = new CpuInfo( - originalName, - physicalProcessorCount: null, - physicalCoreCount, - logicalCoreCount, - Frequency.FromGHz(actualFrequency), - minFrequency: null, - maxFrequency: null); - - Assert.Equal(prettifiedName, ProcessorBrandStringHelper.Prettify(cpuInfo, includeMaxFrequency: true)); - } - - [Theory] - [InlineData("8130U", "Kaby Lake")] - [InlineData("9900K", "Coffee Lake")] - [InlineData("8809G", "Kaby Lake G")] - public void IntelCoreMicroarchitecture(string modelNumber, string microarchitecture) - { - Assert.Equal(microarchitecture, ProcessorBrandStringHelper.ParseIntelCoreMicroarchitecture(modelNumber)); - } - - [Theory] - [InlineData("", "Unknown processor")] - [InlineData(null, "Unknown processor")] - public void UnknownProcessorDoesNotThrow(string? originalName, string prettifiedName) - { - var cpuInfo = new CpuInfo(originalName, nominalFrequency: null); - - Assert.Equal(prettifiedName, ProcessorBrandStringHelper.Prettify(cpuInfo, includeMaxFrequency: true)); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs b/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs index 074bbf50ab..7799fd2038 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs +++ b/tests/BenchmarkDotNet.Tests/Exporters/CommonExporterVerifyTests.cs @@ -12,6 +12,7 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Infra; using BenchmarkDotNet.Tests.Mocks; using BenchmarkDotNet.Tests.Reports; using JetBrains.Annotations; @@ -21,7 +22,6 @@ namespace BenchmarkDotNet.Tests.Exporters { [Collection("VerifyTests")] - [UsesVerify] public class CommonExporterVerifyTests : IDisposable { private readonly CultureInfo initCulture; @@ -62,7 +62,7 @@ [new Metric(new FakeMetricDescriptor("CacheMisses", "Hardware counter 'CacheMiss logger); } - var settings = VerifySettingsFactory.Create(); + var settings = VerifyHelper.Create(); settings.UseTextForParameters(GetName(cultureInfo)); return Verifier.Verify(logger.GetLog(), settings); } diff --git a/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterVerifyTests.cs b/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterVerifyTests.cs index fca08d0c0e..2d57e308a0 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterVerifyTests.cs +++ b/tests/BenchmarkDotNet.Tests/Exporters/MarkdownExporterVerifyTests.cs @@ -10,17 +10,15 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Infra; using BenchmarkDotNet.Validators; using JetBrains.Annotations; using VerifyXunit; using Xunit; -using BenchmarkDotNet.Columns; -using System.Reflection; namespace BenchmarkDotNet.Tests.Exporters { [Collection("VerifyTests")] - [UsesVerify] public class MarkdownExporterVerifyTests : IDisposable { private readonly CultureInfo initCulture = Thread.CurrentThread.CurrentCulture; @@ -42,8 +40,9 @@ public Task GroupExporterTest(Type benchmarkType) var logger = new AccumulationLogger(); logger.WriteLine("=== " + benchmarkType.Name + " ==="); + var exporter = MarkdownExporter.Mock; - var summary = MockFactory.CreateSummary(benchmarkType, benchmarkType.GetCustomAttribute()?.Config.GetColumnHidingRules().ToArray() ?? []); + var summary = MockFactory.CreateSummary(benchmarkType); exporter.ExportToLog(summary, logger); var validator = BaselineValidator.FailOnError; @@ -53,7 +52,7 @@ public Task GroupExporterTest(Type benchmarkType) foreach (var error in errors) logger.WriteLineError("* " + error.Message); - var settings = VerifySettingsFactory.Create(); + var settings = VerifyHelper.Create(); settings.UseTextForParameters(benchmarkType.Name); return Verifier.Verify(logger.GetLog(), settings); } @@ -220,8 +219,8 @@ [Benchmark] public void Bar() { } [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2")] public class MethodJobBaseline_MethodsJobs { - [Benchmark(Baseline = true)] public void Foo() { } - [Benchmark] public void Bar() { } + [Benchmark(Baseline = true)] public void Foo() {} + [Benchmark] public void Bar() {} } [RankColumn, LogicalGroupColumn, BaselineColumn] @@ -230,8 +229,8 @@ public class MethodJobBaseline_MethodsJobsParams { [Params(2, 10), UsedImplicitly] public int Param; - [Benchmark(Baseline = true)] public void Foo() { } - [Benchmark] public void Bar() { } + [Benchmark(Baseline = true)] public void Foo() {} + [Benchmark] public void Bar() {} } /* Invalid */ @@ -239,16 +238,16 @@ [Benchmark] public void Bar() { } [RankColumn, LogicalGroupColumn, BaselineColumn] public class Invalid_TwoMethodBaselines { - [Benchmark(Baseline = true)] public void Foo() { } - [Benchmark(Baseline = true)] public void Bar() { } + [Benchmark(Baseline = true)] public void Foo() {} + [Benchmark(Baseline = true)] public void Bar() {} } [RankColumn, LogicalGroupColumn, BaselineColumn] [SimpleJob(id: "Job1", baseline: true), SimpleJob(id: "Job2", baseline: true)] public class Invalid_TwoJobBaselines { - [Benchmark] public void Foo() { } - [Benchmark] public void Bar() { } + [Benchmark] public void Foo() {} + [Benchmark] public void Bar() {} } /* Escape Params */ @@ -261,13 +260,6 @@ public class Escape_ParamsAndArguments [Benchmark] public void Foo(char charArg) {} [Benchmark] public void Bar() {} } - - /* Hide Column */ - [HideColumns(Column.StdDev)] - public class HideColumns_TableMarkDown - { - [Benchmark] public void Foo() { } - } } } } diff --git a/tests/BenchmarkDotNet.Tests/Infra/VerifyHelper.cs b/tests/BenchmarkDotNet.Tests/Infra/VerifyHelper.cs new file mode 100644 index 0000000000..9d563d8ca6 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Infra/VerifyHelper.cs @@ -0,0 +1,16 @@ +using VerifyTests; + +namespace BenchmarkDotNet.Tests.Infra; + +public static class VerifyHelper +{ + public static VerifySettings Create(string? typeName = null) + { + var result = new VerifySettings(); + result.UseDirectory("VerifiedFiles"); + result.DisableDiff(); + if (typeName != null) + result.UseTypeName(typeName); + return result; + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Jobs/JobIdGeneratorTests.cs b/tests/BenchmarkDotNet.Tests/Jobs/JobIdGeneratorTests.cs new file mode 100644 index 0000000000..c0e845bb7a --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Jobs/JobIdGeneratorTests.cs @@ -0,0 +1,27 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.CsProj; +using Xunit; + +namespace BenchmarkDotNet.Tests.Jobs; + +public class JobIdGeneratorTests +{ + [Theory] + [MemberData(nameof(GetTheoryData), DisableDiscoveryEnumeration = true)] + public void AutoGenerateJobId(string expectedId, Job job) + { + // Act + var result = job.ResolvedId; + + // Assert + Assert.Equal(expectedId, result); + } + + public static TheoryData GetTheoryData() => new TheoryData() + { + {"Job-OOTPKI", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp80) }, + {"Job-QAODSR", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp90) }, + {"Job-KHMDUZ", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp80).WithRuntime(CoreRuntime.Core80) }, + }; +} diff --git a/tests/BenchmarkDotNet.Tests/KnownIssue.cs b/tests/BenchmarkDotNet.Tests/KnownIssue.cs index fe6cb83ee5..1831ad3b08 100644 --- a/tests/BenchmarkDotNet.Tests/KnownIssue.cs +++ b/tests/BenchmarkDotNet.Tests/KnownIssue.cs @@ -2,7 +2,7 @@ namespace BenchmarkDotNet.Tests { public class KnownIssue { - public static KnownIssue Issue2299 => new (2299, "Non-supported Mono on Linux", false); + public static KnownIssue Issue2299 => new(2299, "Non-supported Mono on Linux", false); public int Number { get; } public string Description { get; } diff --git a/tests/BenchmarkDotNet.Tests/LightJsonSerializerTests.cs b/tests/BenchmarkDotNet.Tests/LightJsonSerializerTests.cs new file mode 100644 index 0000000000..16ef22efed --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/LightJsonSerializerTests.cs @@ -0,0 +1,56 @@ +using System.Collections.Generic; +using System.Linq; +using SimpleJson; +using Xunit; + +namespace BenchmarkDotNet.Tests +{ + public class LightJsonSerializerTests + { + [Theory] + [InlineData(10.0, "test", double.NaN)] + public void SimpleJson_ReplaceUnsupportedNumericValues_Smoke(object val1, object val2, object val3) + { + //Arrange + var data = new List() + { + val1, val2, val3 + }; + + //Act + for (int i = 0; i < data.Count; i++) + { + data[i] = SimpleJsonSerializer.ReplaceUnsupportedNumericValues(data[i]); + } + + //Assert + Assert.Equal(val1, data[0]); + Assert.Equal(val2, data[1]); + Assert.Equal(SimpleJsonSerializer.JSON_EMPTY_STRING, data[2]); + } + + + [Fact] + public void SimpleJson_SerializeObjectWithUnsupportedNumericValues_ReturnsValidJson() + { + //Arrange + var data = new Dictionary + { + { "Statistic1", float.NaN}, + { "Statistic2", float.NegativeInfinity }, + { "Statistic3", float.PositiveInfinity }, + { "Statistic4", double.NaN}, + { "Statistic5", double.NegativeInfinity }, + { "Statistic6", double.PositiveInfinity } + }; + + //Act + string json = SimpleJsonSerializer.SerializeObject(data); + + //Assert + Assert.True(SimpleJsonSerializer.TryDeserializeObject(json, out var obj)); + var values = (obj as SimpleJson.JsonObject).Select(x => x.Value); + Assert.True(values.All(x => x.Equals(string.Empty))); + } + } +} diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs index 6b1fad64a6..7b17baa098 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Jobs; @@ -56,6 +57,19 @@ public Measurement RunIteration(IterationData data) public RunResults Run() => default; + internal (long invokeCount, List measurements) Run(EngineStage stage, long invokeCount = 0) + { + var measurements = stage.GetMeasurementList(); + int iterationIndex = 1; + while (stage.GetShouldRunIteration(measurements, ref invokeCount)) + { + var measurement = RunIteration(new IterationData(stage.Mode, stage.Stage, iterationIndex, invokeCount, 1)); + measurements.Add(measurement); + ++iterationIndex; + } + return (invokeCount, measurements); + } + public void WriteLine() => output.WriteLine(""); public void WriteLine(string line) => output.WriteLine(line); diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs index 8c0ce0019e..bbe715d660 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs @@ -18,6 +18,7 @@ using Perfolizer.Horology; using Perfolizer.Metrology; using static BenchmarkDotNet.Reports.SummaryTable.SummaryTableColumn; +using Measurement = BenchmarkDotNet.Reports.Measurement; using MethodInfo = System.Reflection.MethodInfo; namespace BenchmarkDotNet.Tests.Mocks @@ -39,6 +40,27 @@ public static Summary CreateSummary(Type benchmarkType, params IColumnHidingRule ImmutableArray.Create(columHidingRules)); } + public static Summary CreateSummaryWithBiasedDistribution(Type benchmarkType, int min, int median, int max, int n) + { + var runInfo = BenchmarkConverter.TypeToBenchmarks(benchmarkType); + return new Summary( + $"MockSummary-N{n}", + runInfo.BenchmarksCases.Select((benchmark, index) => CreateReportWithBiasedDistribution( + benchmark, + (index + 1) * min, + (index + 1) * median, + (index + 1) * max, + n, + Array.Empty())).ToImmutableArray(), + new HostEnvironmentInfoBuilder().WithoutDotNetSdkVersion().Build(), + string.Empty, + string.Empty, + TimeSpan.FromMinutes(1), + TestCultureInfo.Instance, + ImmutableArray.Empty, + ImmutableArray.Empty); + } + public static Summary CreateSummary(IConfig config) => new Summary( "MockSummary", CreateReports(config), @@ -120,6 +142,40 @@ private static BenchmarkReport CreateReport(BenchmarkCase benchmarkCase, bool hu return new BenchmarkReport(true, benchmarkCase, buildResult, buildResult, new List { executeResult }, metrics); } + private static BenchmarkReport CreateReportWithBiasedDistribution(BenchmarkCase benchmarkCase, int min, int median, int max, int n, Metric[] metrics) + { + var buildResult = BuildResult.Success(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty())); + bool isFoo = benchmarkCase.Descriptor.WorkloadMethodDisplayInfo == "Foo"; + bool isBar = benchmarkCase.Descriptor.WorkloadMethodDisplayInfo == "Bar"; + var measurements = from i in Enumerable.Range(0, Math.Max(1, n / 9)) + from m in isFoo ? new[] + { + new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, min), // 1 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 2, 1, min + ((median - min) / 2) + 1), // 3 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 4, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 7, 1, median + ((max - median) / 2)), // 7 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 8, 1, median + ((max - median) / 2)), // 7 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 9, 1, max), // 10 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 9, 1, max), // 10 + } : new[] + { + new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, min), // 1 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, min), // 1 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 2, 1, min + ((median - min) / 2) + 1), // 3 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 2, 1, min + ((median - min) / 2) + 1), // 3 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 4, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 5, 1, median), // 4 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 7, 1, median + ((max - median) / 2)), // 7 + new Measurement(1, IterationMode.Workload, IterationStage.Result, 9, 1, max), // 10 + } + select m; + var executeResult = new ExecuteResult(measurements.Take(n).ToList(), default, default, 0); + return new BenchmarkReport(true, benchmarkCase, buildResult, buildResult, new List { executeResult }, metrics); + } + [LongRunJob] public class MockBenchmarkClass { diff --git a/tests/BenchmarkDotNet.Tests/Mocks/Toolchain/MockToolchain.cs b/tests/BenchmarkDotNet.Tests/Mocks/Toolchain/MockToolchain.cs index c25929854e..365a55cb88 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/Toolchain/MockToolchain.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/Toolchain/MockToolchain.cs @@ -43,7 +43,7 @@ private class MockExecutor : IExecutor public MockExecutor(Func> measurer) => this.measurer = measurer; - public ExecuteResult Execute(ExecuteParameters executeParameters) => new (measurer(executeParameters.BenchmarkCase)); + public ExecuteResult Execute(ExecuteParameters executeParameters) => new(measurer(executeParameters.BenchmarkCase)); } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs b/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs index d9d2e7490e..38b9167e99 100644 --- a/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs +++ b/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs @@ -19,10 +19,10 @@ namespace BenchmarkDotNet.Tests.Order { public class DefaultOrdererTests { - private static Summary CreateMockSummary() => new ("", ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), + private static Summary CreateMockSummary() => new("", ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), "", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray.Empty, ImmutableArray.Empty); - private static BenchmarkCase CreateBenchmarkCase(string category, int parameter, params BenchmarkLogicalGroupRule[] rules) => new ( + private static BenchmarkCase CreateBenchmarkCase(string category, int parameter, params BenchmarkLogicalGroupRule[] rules) => new( new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo, categories: new[] { category }), new Job(), new ParameterInstances(new[] diff --git a/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs b/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs new file mode 100644 index 0000000000..c0e645ea22 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs @@ -0,0 +1,146 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.CsProj; +using System.Linq; +using Xunit; + +namespace BenchmarkDotNet.Tests.Order; + +public class JobOrderTests +{ + [Fact] + public void TestJobOrders_ByJobId() + { + // Arrange + Job[] jobs = + [ + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80) + .WithRuntime(CoreRuntime.Core80) + .WithId("v1.4.1"), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90) + .WithId("v1.4.10"), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0) + .WithRuntime(CoreRuntime.Core10_0) + .WithId("v1.4.2"), + ]; + + // Verify jobs are sorted by JobId's ordinal order. + { + // Act + var comparer = JobComparer.Instance; + var results = jobs.OrderBy(x => x, comparer) + .Select(x => x.Job.Id) + .ToArray(); + + // Assert + Assert.Equal(["v1.4.1", "v1.4.10", "v1.4.2"], results); + } + + // Verify jobs are sorted by JobId's numeric order. + { + // Act + var comparer = JobComparer.Numeric; + var results = jobs.OrderBy(d => d, comparer) + .Select(x => x.Job.Id) + .ToArray(); + // Assert + Assert.Equal(["v1.4.1", "v1.4.2", "v1.4.10"], results); + } + } + + [Fact] + public void TestJobOrders_ByRuntime() + { + // Arrange + Job[] jobs = + [ + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0) + .WithRuntime(CoreRuntime.Core80), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80) + .WithRuntime(CoreRuntime.Core10_0), + ]; + + // Act + // Verify jobs are sorted by Runtime's numeric order. + var results = jobs.OrderBy(d => d, JobComparer.Numeric) + .Select(x => x.Job.Environment.GetRuntime().Name) + .ToArray(); + + // Assert + var expected = new[] + { + CoreRuntime.Core80.Name, + CoreRuntime.Core90.Name, + CoreRuntime.Core10_0.Name + }; + Assert.Equal(expected, results); + } + + [Fact] + public void TestJobOrders_ByToolchain() + { + // Arrange + Job[] jobs = + [ + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80), + ]; + + // Act + // Verify jobs are sorted by Toolchain's numeric order. + var results = jobs.OrderBy(d => d, JobComparer.Numeric) + .Select(x => x.Job.GetToolchain().Name) + .ToArray(); + + // Assert + var expected = new[] + { + CsProjCoreToolchain.NetCoreApp80.Name, + CsProjCoreToolchain.NetCoreApp90.Name, + CsProjCoreToolchain.NetCoreApp10_0.Name, + }; + Assert.Equal(expected, results); + } + + [Theory] + [InlineData("item1", "item1", 0)] + [InlineData("item123", "item123", 0)] + // Compare different values + [InlineData("item1", "item2", -1)] + [InlineData("item2", "item1", 1)] + [InlineData("item2", "item10", -1)] + [InlineData("item10", "item2", 1)] + [InlineData("item1a", "item1b", -1)] + [InlineData("item1b", "item1a", 1)] + [InlineData("item", "item1", -1)] + [InlineData("item10", "item", 1)] + [InlineData(".NET 8", ".NET 10", -1)] + [InlineData(".NET 10", ".NET 8", 1)] + [InlineData("v1.4.1", "v1.4.10", -1)] + [InlineData("v1.4.10", "v1.4.2", 1)] + // Compare zero paddeed numeric string. + [InlineData("item01", "item1", 0)] + [InlineData("item001", "item1", 0)] + [InlineData("item1", "item001", 0)] + [InlineData("item1", "item01", 0)] + [InlineData("item9", "item09", 0)] + [InlineData(".NET 08", ".NET 10", -1)] + [InlineData(".NET 10", ".NET 08", 1)] + // Arguments that contains null + [InlineData(null, "a", -1)] + [InlineData("a", null, 1)] + [InlineData(null, null, 0)] + public void TestNumericComparer(string? a, string? b, int expectedSign) + { + int result = new JobComparer.NumericStringComparer().Compare(a, b); + Assert.Equal(expectedSign, NormalizeSign(result)); + + static int NormalizeSign(int value) + => value == 0 ? 0 : value < 0 ? -1 : 1; + } +} diff --git a/tests/BenchmarkDotNet.Tests/ParamsSourceTests.cs b/tests/BenchmarkDotNet.Tests/ParamsSourceTests.cs index 7ee8695475..c32b48d721 100644 --- a/tests/BenchmarkDotNet.Tests/ParamsSourceTests.cs +++ b/tests/BenchmarkDotNet.Tests/ParamsSourceTests.cs @@ -21,7 +21,7 @@ public static IEnumerable Values() { yield return null; yield return ValueTuple.Create(10); - yield return ValueTuple.Create(10, 20); + yield return (10, 20); yield return (10, 20, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); } diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/Infra/PerfonarMock.cs b/tests/BenchmarkDotNet.Tests/Perfonar/Infra/PerfonarMock.cs new file mode 100644 index 0000000000..f63caeafa4 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/Infra/PerfonarMock.cs @@ -0,0 +1,13 @@ +using BenchmarkDotNet.Properties; +using Perfolizer.Models; + +namespace BenchmarkDotNet.Tests.Perfonar.Infra; + +public static class PerfonarMock +{ + public static readonly EngineInfo Engine = new() + { + Name = BenchmarkDotNetInfo.BenchmarkDotNetCaption, + Version = "0.1729.0-mock" + }; +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/Infra/PerfonarTestExtensions.cs b/tests/BenchmarkDotNet.Tests/Perfonar/Infra/PerfonarTestExtensions.cs new file mode 100644 index 0000000000..3df26943cf --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/Infra/PerfonarTestExtensions.cs @@ -0,0 +1,22 @@ +using Perfolizer.Models; +using Perfolizer.Metrology; + +namespace BenchmarkDotNet.Tests.Perfonar.Infra; + +public static class PerfonarTestExtensions +{ + public static EntryInfo AddMetrics(this EntryInfo entry, params string[] metrics) + { + for (int i = 0; i < metrics.Length; i++) + { + var measurement = Measurement.Parse(metrics[i]); + entry.Add(new EntryInfo + { + IterationIndex = i, + Value = measurement.NominalValue, + Unit = measurement.Unit + }); + } + return entry; + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs b/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs new file mode 100644 index 0000000000..81869534a1 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs @@ -0,0 +1,174 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Models; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Tests.Builders; +using BenchmarkDotNet.Tests.Infra; +using BenchmarkDotNet.Tests.Perfonar.Infra; +using JetBrains.Annotations; +using Perfolizer.Models; +using Perfolizer.Json; +using Perfolizer.Perfonar.Base; +using Perfolizer.Perfonar.Tables; +using VerifyXunit; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Perfonar; + +public class PerfonarTests(ITestOutputHelper output) +{ + [Theory] + [MemberData(nameof(EntryDataKeys))] + public Task PerfonarIndexTest(string key) => VerifyString(key, new PerfonarIndex(TableDataMap[key].RootEntry).Dump()); + + [Theory] + [MemberData(nameof(EntryDataKeys))] + public Task PerfonarTableTest(string key) + { + var table = TableDataMap[key]; + string markdown = table.ToMarkdown(new PerfonarTableStyle()); + string json = LightJsonSerializer.Serialize(table.RootEntry, new LightJsonSettings { Indent = true }); + return VerifyString(key, markdown + "\n" + json); + } + + private static readonly IDictionary TableDataMap = new Dictionary + { + { + "default01", new PerfonarTable( + Root().Add( + Benchmark("Foo", "10ns", "11ns", "12ns"), + Benchmark("Bar", "200ns", "201ns", "202ns") + ), + GetDefaultTableConfig() + ) + }, + { + "default02", new PerfonarTable( + Root().Add( + Job(RuntimeMoniker.Net481).Add( + Benchmark("Foo", "10ns", "11ns", "12ns"), + Benchmark("Bar", "20ns", "21ns", "22ns")), + Job(RuntimeMoniker.Net70).Add( + Benchmark("Foo", "30ns", "31ns", "32ns"), + Benchmark("Bar", "40ns", "41ns", "42ns"))), + GetDefaultTableConfig() + ) + }, + { + "default03", new PerfonarTable( + Root().Add( + Job(RuntimeMoniker.Net70, Jit.RyuJit).Add( + Benchmark("Foo", "30ns", "31ns", "32ns"), + Benchmark("Bar", "40ns", "41ns", "42ns"))), + GetDefaultTableConfig() + ) + }, + { + "default04", new PerfonarTable( + Root().Add( + Job(RuntimeMoniker.Net481, Jit.LegacyJit).Add( + Benchmark("Foo", "10ns", "11ns", "12ns"), + Benchmark("Bar", "20ns", "21ns", "22ns")), + Job(RuntimeMoniker.Net70, Jit.RyuJit).Add( + Benchmark("Foo", "30ns", "31ns", "32ns"), + Benchmark("Bar", "40ns", "41ns", "42ns"))), + GetDefaultTableConfig() + ) + }, + { + "default05", new PerfonarTable( + Root().Add( + Enumerable.Range(0, 20).Select(index => + Job((RuntimeMoniker)index, Jit.RyuJit, index).Add( + Benchmark("Foo", index * 10 + 1 + "ns", index * 10 + 2 + "ns", index * 10 + 3 + "ns"), + Benchmark("Bar", index * 10 + 6 + "ns", index * 10 + 7 + "ns", index * 10 + 8 + "ns") + )).ToArray()), + GetDefaultTableConfig() + ) + }, + { + "sort01", new PerfonarTable( + new EntryInfo().Add(Benchmark("Foo", "10ms"), Benchmark("Bar", "20ms")), + new PerfonarTableConfig + { + ColumnDefinitions = + { + new PerfonarColumnDefinition(".benchmark.method"), + new PerfonarColumnDefinition("=center") + }, + SortPolicies = + [ + new PerfonarSortPolicy("=center", PerfonarSortDirection.Descending) + ] + }) + }, + { + "params01", new PerfonarTable( + new EntryInfo().Add( + new EntryInfo + { + Parameters = new Dictionary { { "A", 1 }, { "B", 2 } } + }.Add(Benchmark("Foo", "10ms"))).Add( + new EntryInfo + { + Parameters = new Dictionary { { "A", 10 }, { "B", 20 } } + }.Add(Benchmark("Bar", "20ms"))), + new PerfonarTableConfig + { + ColumnDefinitions = + { + new PerfonarColumnDefinition(".benchmark.method"), + new PerfonarColumnDefinition(".parameters"), + new PerfonarColumnDefinition("=center") + } + } + ) + } + }; + + [UsedImplicitly] public static TheoryData EntryDataKeys = TheoryDataHelper.Create(TableDataMap.Keys); + + private static EntryInfo Root() => new EntryInfo + { + Engine = PerfonarMock.Engine, + Host = new HostEnvironmentInfoBuilder().Build().ToPerfonar() + }; + + private static PerfonarTableConfig GetDefaultTableConfig() => new() + { + ColumnDefinitions = + [ + new PerfonarColumnDefinition(".engine") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PerfonarColumnDefinition(".host.os") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PerfonarColumnDefinition(".host.cpu") { Cloud = "primary", IsSelfExplanatory = true, IsAtomic = true }, + new PerfonarColumnDefinition(".benchmark") { Cloud = "secondary" }, + new PerfonarColumnDefinition(".job") { Cloud = "secondary", Compressed = true }, + new PerfonarColumnDefinition("=center"), + new PerfonarColumnDefinition("=spread") + ] + }; + + private static EntryInfo Job(RuntimeMoniker? runtime = null, Jit? jit = null, int? affinity = null) => new EntryInfo + { + Job = new JobInfo + { + Environment = new BdnEnvironment { Runtime = runtime, Jit = jit, Affinity = affinity } + } + }; + + private static EntryInfo Benchmark(string name, params string[] metrics) => new EntryInfo + { + Benchmark = new BdnBenchmark { Type = "Bench", Method = name } + }.AddMetrics(metrics); + + private Task VerifyString(string key, string content) + { + output.WriteLine(content); + var settings = VerifyHelper.Create("Perfonar"); + settings.UseParameters(key); + return Verifier.Verify(content, settings); + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default01.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default01.verified.txt new file mode 100644 index 0000000000..a412c88898 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default01.verified.txt @@ -0,0 +1,190 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .unit = ns + .value = 10 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .unit = ns + .value = 11 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .unit = ns + .value = 12 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .unit = ns + .value = 200 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .unit = ns + .value = 201 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .unit = ns + .value = 202 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default02.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default02.verified.txt new file mode 100644 index 0000000000..c8759facf6 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default02.verified.txt @@ -0,0 +1,391 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .job + .job.environment + .job.environment.runtime + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 10 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 11 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 12 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 20 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 21 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.runtime = Net481 + .unit = ns + .value = 22 +[Entry6] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 30 +[Entry7] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 31 +[Entry8] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 32 +[Entry9] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 40 +[Entry10] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 41 +[Entry11] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.runtime = Net70 + .unit = ns + .value = 42 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default03.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default03.verified.txt new file mode 100644 index 0000000000..af8395637c --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default03.verified.txt @@ -0,0 +1,218 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .job + .job.environment + .job.environment.jit + .job.environment.runtime + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 30 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 31 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 32 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 40 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 41 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 42 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default04.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default04.verified.txt new file mode 100644 index 0000000000..1179e76cd5 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default04.verified.txt @@ -0,0 +1,404 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .job + .job.environment + .job.environment.jit + .job.environment.runtime + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 10 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 11 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 12 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 20 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 21 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = LegacyJit + .job.environment.runtime = Net481 + .unit = ns + .value = 22 +[Entry6] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 30 +[Entry7] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 31 +[Entry8] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 32 +[Entry9] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 40 +[Entry10] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 41 +[Entry11] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 42 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default05.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default05.verified.txt new file mode 100644 index 0000000000..ef0556ab95 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default05.verified.txt @@ -0,0 +1,3873 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .engine + .engine.name + .engine.version + .host + .host.chronometerFrequency + .host.configuration + .host.cpu + .host.cpu.logicalCoreCount + .host.cpu.maxFrequencyHz + .host.cpu.nominalFrequencyHz + .host.cpu.physicalCoreCount + .host.cpu.physicalProcessorCount + .host.cpu.processorName + .host.dotNetSdkVersion + .host.hardwareTimerKind + .host.hasAttachedDebugger + .host.hasRyuJit + .host.os + .host.os.display + .host.runtimeVersion + .iterationIndex + .job + .job.environment + .job.environment.affinity + .job.environment.jit + .job.environment.runtime + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 1 +[Entry1] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 2 +[Entry2] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 3 +[Entry3] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 6 +[Entry4] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 7 +[Entry5] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 0 + .job.environment.jit = RyuJit + .job.environment.runtime = HostProcess + .unit = ns + .value = 8 +[Entry6] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 11 +[Entry7] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 12 +[Entry8] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 13 +[Entry9] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 16 +[Entry10] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 17 +[Entry11] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 1 + .job.environment.jit = RyuJit + .job.environment.runtime = NotRecognized + .unit = ns + .value = 18 +[Entry12] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 21 +[Entry13] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 22 +[Entry14] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 23 +[Entry15] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 26 +[Entry16] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 27 +[Entry17] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 2 + .job.environment.jit = RyuJit + .job.environment.runtime = Mono + .unit = ns + .value = 28 +[Entry18] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 31 +[Entry19] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 32 +[Entry20] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 33 +[Entry21] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 36 +[Entry22] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 37 +[Entry23] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 3 + .job.environment.jit = RyuJit + .job.environment.runtime = Net461 + .unit = ns + .value = 38 +[Entry24] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 41 +[Entry25] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 42 +[Entry26] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 43 +[Entry27] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 46 +[Entry28] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 47 +[Entry29] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 4 + .job.environment.jit = RyuJit + .job.environment.runtime = Net462 + .unit = ns + .value = 48 +[Entry30] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 51 +[Entry31] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 52 +[Entry32] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 53 +[Entry33] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 56 +[Entry34] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 57 +[Entry35] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 5 + .job.environment.jit = RyuJit + .job.environment.runtime = Net47 + .unit = ns + .value = 58 +[Entry36] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 61 +[Entry37] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 62 +[Entry38] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 63 +[Entry39] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 66 +[Entry40] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 67 +[Entry41] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 6 + .job.environment.jit = RyuJit + .job.environment.runtime = Net471 + .unit = ns + .value = 68 +[Entry42] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 71 +[Entry43] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 72 +[Entry44] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 73 +[Entry45] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 76 +[Entry46] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 77 +[Entry47] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 7 + .job.environment.jit = RyuJit + .job.environment.runtime = Net472 + .unit = ns + .value = 78 +[Entry48] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 81 +[Entry49] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 82 +[Entry50] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 83 +[Entry51] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 86 +[Entry52] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 87 +[Entry53] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 8 + .job.environment.jit = RyuJit + .job.environment.runtime = Net48 + .unit = ns + .value = 88 +[Entry54] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 91 +[Entry55] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 92 +[Entry56] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 93 +[Entry57] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 96 +[Entry58] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 97 +[Entry59] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 9 + .job.environment.jit = RyuJit + .job.environment.runtime = Net481 + .unit = ns + .value = 98 +[Entry60] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 101 +[Entry61] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 102 +[Entry62] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 103 +[Entry63] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 106 +[Entry64] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 107 +[Entry65] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 10 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp20 + .unit = ns + .value = 108 +[Entry66] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 111 +[Entry67] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 112 +[Entry68] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 113 +[Entry69] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 116 +[Entry70] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 117 +[Entry71] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 11 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp21 + .unit = ns + .value = 118 +[Entry72] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 121 +[Entry73] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 122 +[Entry74] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 123 +[Entry75] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 126 +[Entry76] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 127 +[Entry77] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 12 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp22 + .unit = ns + .value = 128 +[Entry78] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 131 +[Entry79] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 132 +[Entry80] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 133 +[Entry81] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 136 +[Entry82] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 137 +[Entry83] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 13 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp30 + .unit = ns + .value = 138 +[Entry84] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 141 +[Entry85] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 142 +[Entry86] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 143 +[Entry87] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 146 +[Entry88] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 147 +[Entry89] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 14 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp31 + .unit = ns + .value = 148 +[Entry90] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 151 +[Entry91] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 152 +[Entry92] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 153 +[Entry93] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 156 +[Entry94] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 157 +[Entry95] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 15 + .job.environment.jit = RyuJit + .job.environment.runtime = NetCoreApp50 + .unit = ns + .value = 158 +[Entry96] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 161 +[Entry97] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 162 +[Entry98] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 163 +[Entry99] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 166 +[Entry100] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 167 +[Entry101] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 16 + .job.environment.jit = RyuJit + .job.environment.runtime = Net50 + .unit = ns + .value = 168 +[Entry102] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 171 +[Entry103] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 172 +[Entry104] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 173 +[Entry105] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 176 +[Entry106] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 177 +[Entry107] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 17 + .job.environment.jit = RyuJit + .job.environment.runtime = Net60 + .unit = ns + .value = 178 +[Entry108] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 181 +[Entry109] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 182 +[Entry110] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 183 +[Entry111] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 186 +[Entry112] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 187 +[Entry113] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 18 + .job.environment.jit = RyuJit + .job.environment.runtime = Net70 + .unit = ns + .value = 188 +[Entry114] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 191 +[Entry115] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 192 +[Entry116] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 193 +[Entry117] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 0 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 196 +[Entry118] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 1 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 197 +[Entry119] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .engine = BenchmarkDotNet v0.1729.0-mock + .engine.name = BenchmarkDotNet + .engine.version = 0.1729.0-mock + .host = + .host.chronometerFrequency = 2531248 + .host.configuration = CONFIGURATION + .host.cpu = MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + .host.cpu.logicalCoreCount = 8 + .host.cpu.maxFrequencyHz = 3100000000 + .host.cpu.nominalFrequencyHz = 3100000000 + .host.cpu.physicalCoreCount = 4 + .host.cpu.physicalProcessorCount = 1 + .host.cpu.processorName = MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz + .host.dotNetSdkVersion = 1.0.x.mock + .host.hardwareTimerKind = Tsc + .host.hasAttachedDebugger = False + .host.hasRyuJit = True + .host.os = Microsoft Windows NT 10.0.x.mock + .host.os.display = Microsoft Windows NT 10.0.x.mock + .host.runtimeVersion = Clr 4.0.x.mock + .iterationIndex = 2 + .job = + .job.environment = + .job.environment.affinity = 19 + .job.environment.jit = RyuJit + .job.environment.runtime = Net80 + .unit = ns + .value = 198 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=params01.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=params01.verified.txt new file mode 100644 index 0000000000..ccc03f6896 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=params01.verified.txt @@ -0,0 +1,31 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .iterationIndex + .parameters + .parameters.a + .parameters.b + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .iterationIndex = 0 + .parameters = + .parameters.a = 1 + .parameters.b = 2 + .unit = ms + .value = 10 +[Entry1] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .iterationIndex = 0 + .parameters = + .parameters.a = 10 + .parameters.b = 20 + .unit = ms + .value = 20 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=sort01.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=sort01.verified.txt new file mode 100644 index 0000000000..f1c1835a59 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=sort01.verified.txt @@ -0,0 +1,22 @@ +[Keys] + .benchmark + .benchmark.method + .benchmark.type + .iterationIndex + .unit + .value + +[Entry0] + .benchmark = Bench.Foo + .benchmark.method = Foo + .benchmark.type = Bench + .iterationIndex = 0 + .unit = ms + .value = 10 +[Entry1] + .benchmark = Bench.Bar + .benchmark.method = Bar + .benchmark.type = Bench + .iterationIndex = 0 + .unit = ms + .value = 20 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default01.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default01.verified.txt new file mode 100644 index 0000000000..9fe1919a4e --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default01.verified.txt @@ -0,0 +1,84 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench + +| Method | Center | Spread | +|:-------|---------:|--------:| +| Foo | 11.0 ns | 0.81 ns | +| Bar | 201.0 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 11, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 12, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 200, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 201, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 202, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default02.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default02.verified.txt new file mode 100644 index 0000000000..a3ade54aea --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default02.verified.txt @@ -0,0 +1,150 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench + +| Method | Runtime | Center | Spread | +|:-------|:--------|--------:|--------:| +| Foo | Net481 | 11.0 ns | 0.81 ns | +| Bar | Net481 | 21.0 ns | 0.81 ns | +| Foo | Net70 | 31.0 ns | 0.81 ns | +| Bar | Net70 | 41.0 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "job": { + "environment": { + "runtime": "net481" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 11, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 12, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 20, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 21, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 22, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net70" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 30, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 31, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 32, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 40, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 41, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 42, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default03.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default03.verified.txt new file mode 100644 index 0000000000..a940585663 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default03.verified.txt @@ -0,0 +1,94 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench, Runtime = Net70, Jit = RyuJit + +| Method | Center | Spread | +|:-------|--------:|--------:| +| Foo | 31.0 ns | 0.81 ns | +| Bar | 41.0 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "job": { + "environment": { + "runtime": "net70", + "jit": "ryuJit" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 30, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 31, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 32, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 40, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 41, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 42, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default04.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default04.verified.txt new file mode 100644 index 0000000000..b9b8e6556f --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default04.verified.txt @@ -0,0 +1,155 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench + +@Net481LegacyJit: Runtime = Net481, Jit = LegacyJit +@Net70RyuJit: Runtime = Net70, Jit = RyuJit + +| Method | Job | Center | Spread | +|:-------|:----------------|--------:|--------:| +| Foo | Net481LegacyJit | 11.0 ns | 0.81 ns | +| Bar | Net481LegacyJit | 21.0 ns | 0.81 ns | +| Foo | Net70RyuJit | 31.0 ns | 0.81 ns | +| Bar | Net70RyuJit | 41.0 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "job": { + "environment": { + "runtime": "net481", + "jit": "legacyJit" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 11, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 12, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 20, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 21, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 22, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net70", + "jit": "ryuJit" + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 30, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 31, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 32, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 40, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 41, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 42, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default05.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default05.verified.txt new file mode 100644 index 0000000000..0b54da76ec --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default05.verified.txt @@ -0,0 +1,1237 @@ +BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock +MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores + +Type = Bench, Jit = RyuJit + +@HostProcessAffinity0: Runtime = HostProcess, Affinity = 0 +@Squid: Runtime = NotRecognized, Affinity = 1 +@MonoAffinity2: Runtime = Mono, Affinity = 2 +@Net461Affinity3: Runtime = Net461, Affinity = 3 +@Net462Affinity4: Runtime = Net462, Affinity = 4 +@Net47Affinity5: Runtime = Net47, Affinity = 5 +@Net471Affinity6: Runtime = Net471, Affinity = 6 +@Net472Affinity7: Runtime = Net472, Affinity = 7 +@Net48Affinity8: Runtime = Net48, Affinity = 8 +@Net481Affinity9: Runtime = Net481, Affinity = 9 +@Liger: Runtime = NetCoreApp20, Affinity = 10 +@Rat: Runtime = NetCoreApp21, Affinity = 11 +@Skate: Runtime = NetCoreApp22, Affinity = 12 +@Perch: Runtime = NetCoreApp30, Affinity = 13 +@Roach: Runtime = NetCoreApp31, Affinity = 14 +@Roe: Runtime = NetCoreApp50, Affinity = 15 +@Net50Affinity16: Runtime = Net50, Affinity = 16 +@Net60Affinity17: Runtime = Net60, Affinity = 17 +@Net70Affinity18: Runtime = Net70, Affinity = 18 +@Net80Affinity19: Runtime = Net80, Affinity = 19 + +| Method | Job | Center | Spread | +|:-------|:---------------------|----------:|--------:| +| Foo | HostProcessAffinity0 | 2.00 ns | 0.81 ns | +| Bar | HostProcessAffinity0 | 7.00 ns | 0.81 ns | +| Foo | Squid | 12.00 ns | 0.81 ns | +| Bar | Squid | 17.00 ns | 0.81 ns | +| Foo | MonoAffinity2 | 22.00 ns | 0.81 ns | +| Bar | MonoAffinity2 | 27.00 ns | 0.81 ns | +| Foo | Net461Affinity3 | 32.00 ns | 0.81 ns | +| Bar | Net461Affinity3 | 37.00 ns | 0.81 ns | +| Foo | Net462Affinity4 | 42.00 ns | 0.81 ns | +| Bar | Net462Affinity4 | 47.00 ns | 0.81 ns | +| Foo | Net47Affinity5 | 52.00 ns | 0.81 ns | +| Bar | Net47Affinity5 | 57.00 ns | 0.81 ns | +| Foo | Net471Affinity6 | 62.00 ns | 0.81 ns | +| Bar | Net471Affinity6 | 67.00 ns | 0.81 ns | +| Foo | Net472Affinity7 | 72.00 ns | 0.81 ns | +| Bar | Net472Affinity7 | 77.00 ns | 0.81 ns | +| Foo | Net48Affinity8 | 82.00 ns | 0.81 ns | +| Bar | Net48Affinity8 | 87.00 ns | 0.81 ns | +| Foo | Net481Affinity9 | 92.00 ns | 0.81 ns | +| Bar | Net481Affinity9 | 97.00 ns | 0.81 ns | +| Foo | Liger | 102.00 ns | 0.81 ns | +| Bar | Liger | 107.00 ns | 0.81 ns | +| Foo | Rat | 112.00 ns | 0.81 ns | +| Bar | Rat | 117.00 ns | 0.81 ns | +| Foo | Skate | 122.00 ns | 0.81 ns | +| Bar | Skate | 127.00 ns | 0.81 ns | +| Foo | Perch | 132.00 ns | 0.81 ns | +| Bar | Perch | 137.00 ns | 0.81 ns | +| Foo | Roach | 142.00 ns | 0.81 ns | +| Bar | Roach | 147.00 ns | 0.81 ns | +| Foo | Roe | 152.00 ns | 0.81 ns | +| Bar | Roe | 157.00 ns | 0.81 ns | +| Foo | Net50Affinity16 | 162.00 ns | 0.81 ns | +| Bar | Net50Affinity16 | 167.00 ns | 0.81 ns | +| Foo | Net60Affinity17 | 172.00 ns | 0.81 ns | +| Bar | Net60Affinity17 | 177.00 ns | 0.81 ns | +| Foo | Net70Affinity18 | 182.00 ns | 0.81 ns | +| Bar | Net70Affinity18 | 187.00 ns | 0.81 ns | +| Foo | Net80Affinity19 | 192.00 ns | 0.81 ns | +| Bar | Net80Affinity19 | 197.00 ns | 0.81 ns | + +{ + "engine": { + "name": "BenchmarkDotNet", + "version": "0.1729.0-mock" + }, + "host": { + "runtimeVersion": "Clr 4.0.x.mock", + "hasAttachedDebugger": false, + "hasRyuJit": true, + "configuration": "CONFIGURATION", + "dotNetSdkVersion": "1.0.x.mock", + "chronometerFrequency": 2531248, + "hardwareTimerKind": "Tsc", + "os": { + "display": "Microsoft Windows NT 10.0.x.mock" + }, + "cpu": { + "processorName": "MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", + "physicalProcessorCount": 1, + "physicalCoreCount": 4, + "logicalCoreCount": 8, + "nominalFrequencyHz": 3100000000, + "maxFrequencyHz": 3100000000 + } + }, + "nested": [ + { + "job": { + "environment": { + "runtime": "hostProcess", + "jit": "ryuJit", + "affinity": 0 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 1, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 2, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 3, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 6, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 7, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 8, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "notRecognized", + "jit": "ryuJit", + "affinity": 1 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 11, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 12, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 13, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 16, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 17, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 18, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "mono", + "jit": "ryuJit", + "affinity": 2 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 21, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 22, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 23, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 26, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 27, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 28, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net461", + "jit": "ryuJit", + "affinity": 3 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 31, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 32, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 33, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 36, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 37, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 38, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net462", + "jit": "ryuJit", + "affinity": 4 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 41, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 42, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 43, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 46, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 47, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 48, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net47", + "jit": "ryuJit", + "affinity": 5 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 51, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 52, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 53, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 56, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 57, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 58, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net471", + "jit": "ryuJit", + "affinity": 6 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 61, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 62, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 63, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 66, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 67, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 68, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net472", + "jit": "ryuJit", + "affinity": 7 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 71, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 72, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 73, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 76, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 77, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 78, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net48", + "jit": "ryuJit", + "affinity": 8 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 81, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 82, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 83, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 86, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 87, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 88, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net481", + "jit": "ryuJit", + "affinity": 9 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 91, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 92, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 93, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 96, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 97, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 98, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp20", + "jit": "ryuJit", + "affinity": 10 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 101, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 102, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 103, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 106, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 107, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 108, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp21", + "jit": "ryuJit", + "affinity": 11 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 111, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 112, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 113, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 116, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 117, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 118, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp22", + "jit": "ryuJit", + "affinity": 12 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 121, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 122, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 123, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 126, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 127, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 128, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp30", + "jit": "ryuJit", + "affinity": 13 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 131, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 132, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 133, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 136, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 137, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 138, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp31", + "jit": "ryuJit", + "affinity": 14 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 141, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 142, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 143, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 146, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 147, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 148, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "netCoreApp50", + "jit": "ryuJit", + "affinity": 15 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 151, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 152, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 153, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 156, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 157, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 158, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net50", + "jit": "ryuJit", + "affinity": 16 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 161, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 162, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 163, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 166, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 167, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 168, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net60", + "jit": "ryuJit", + "affinity": 17 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 171, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 172, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 173, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 176, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 177, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 178, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net70", + "jit": "ryuJit", + "affinity": 18 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 181, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 182, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 183, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 186, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 187, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 188, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + }, + { + "job": { + "environment": { + "runtime": "net80", + "jit": "ryuJit", + "affinity": 19 + } + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 191, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 192, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 193, + "unit": "ns", + "iterationIndex": 2 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 196, + "unit": "ns", + "iterationIndex": 0 + }, + { + "value": 197, + "unit": "ns", + "iterationIndex": 1 + }, + { + "value": 198, + "unit": "ns", + "iterationIndex": 2 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=params01.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=params01.verified.txt new file mode 100644 index 0000000000..f5f36bb3b7 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=params01.verified.txt @@ -0,0 +1,51 @@ +| Method | A | B | Center | +|:-------|---:|---:|--------:| +| Foo | 1 | 2 | 10.0 ms | +| Bar | 10 | 20 | 20.0 ms | + +{ + "nested": [ + { + "parameters": { + "a": 1, + "b": 2 + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ms", + "iterationIndex": 0 + } + ] + } + ] + }, + { + "parameters": { + "a": 10, + "b": 20 + }, + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 20, + "unit": "ms", + "iterationIndex": 0 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=sort01.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=sort01.verified.txt new file mode 100644 index 0000000000..8c2956eb1a --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=sort01.verified.txt @@ -0,0 +1,35 @@ +| Method | Center | +|:-------|--------:| +| Bar | 20.0 ms | +| Foo | 10.0 ms | + +{ + "nested": [ + { + "benchmark": { + "type": "Bench", + "method": "Foo" + }, + "nested": [ + { + "value": 10, + "unit": "ms", + "iterationIndex": 0 + } + ] + }, + { + "benchmark": { + "type": "Bench", + "method": "Bar" + }, + "nested": [ + { + "value": 20, + "unit": "ms", + "iterationIndex": 0 + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/CpuInfoFormatterTests.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/CpuInfoFormatterTests.cs deleted file mode 100644 index 83d14685a4..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/CpuInfoFormatterTests.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System.Text; -using System.Threading.Tasks; -using BenchmarkDotNet.Portability.Cpu; -using BenchmarkDotNet.Tests.Builders; -using VerifyXunit; -using Xunit; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - [Collection("VerifyTests")] - [UsesVerify] - public class CpuInfoFormatterTests - { - [Fact] - public Task FormatTest() - { - var captions = new StringBuilder(); - foreach (var processorName in new[] { null, "", "Intel" }) - foreach (var physicalProcessorCount in new int?[] { null, 0, 1, 2 }) - foreach (var physicalCoreCount in new int?[] { null, 0, 1, 2 }) - foreach (var logicalCoreCount in new int?[] { null, 0, 1, 2 }) - { - var mockCpuInfo = new CpuInfo(processorName, physicalProcessorCount, physicalCoreCount, logicalCoreCount, null, null, null); - captions.AppendLine(CpuInfoFormatter.Format(mockCpuInfo)); - } - - var settings = VerifySettingsFactory.Create(); - return Verifier.Verify(captions.ToString(), settings); - } - } -} diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/ProcCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/ProcCpuInfoParserTests.cs deleted file mode 100644 index 6d2faa423e..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/ProcCpuInfoParserTests.cs +++ /dev/null @@ -1,90 +0,0 @@ -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; -using Xunit; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - public class ProcCpuInfoParserTests - { - [Fact] - public void EmptyTest() - { - var parser = ProcCpuInfoParser.ParseOutput(string.Empty); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - - Assert.Null(parser.MaxFrequency); - Assert.Null(parser.MinFrequency); - } - - [Fact] - public void MalformedTest() - { - var parser = ProcCpuInfoParser.ParseOutput("malformedkey: malformedvalue\n\nmalformedkey2: malformedvalue2"); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - Assert.Null(parser.MaxFrequency); - Assert.Null(parser.MinFrequency); - } - - [Fact] - public void TwoProcessorWithDifferentCoresCountTest() - { - string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoProcessorWithDifferentCoresCount.txt"); - var parser = ProcCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Unknown processor with 2 cores and hyper threading, Unknown processor with 4 cores", parser.ProcessorName); - Assert.Equal(2, parser.PhysicalProcessorCount); - Assert.Equal(6, parser.PhysicalCoreCount); - Assert.Equal(8, parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - Assert.Equal(0.8 * Frequency.GHz, parser.MinFrequency); - Assert.Equal(2.5 * Frequency.GHz, parser.MaxFrequency); - } - - - [Fact] - public void RealOneProcessorTwoCoresTest() - { - string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoRealOneProcessorTwoCores.txt"); - var parser = ProcCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Core(TM) i5-6200U CPU @ 2.30GHz", parser.ProcessorName); - Assert.Equal(1, parser.PhysicalProcessorCount); - Assert.Equal(2, parser.PhysicalCoreCount); - Assert.Equal(4, parser.LogicalCoreCount); - Assert.Equal(2.3 * Frequency.GHz, parser.NominalFrequency); - Assert.Equal(0.8 * Frequency.GHz, parser.MinFrequency); - Assert.Equal(2.3 * Frequency.GHz, parser.MaxFrequency); - } - - [Fact] - public void RealOneProcessorFourCoresTest() - { - string cpuInfo = TestHelper.ReadTestFile("ProcCpuInfoRealOneProcessorFourCores.txt"); - var parser = ProcCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", parser.ProcessorName); - Assert.Equal(1, parser.PhysicalProcessorCount); - Assert.Equal(4, parser.PhysicalCoreCount); - Assert.Equal(8, parser.LogicalCoreCount); - Assert.Equal(2.5 * Frequency.GHz, parser.NominalFrequency); - Assert.Equal(0.8 * Frequency.GHz, parser.MinFrequency); - Assert.Equal(2.5 * Frequency.GHz, parser.MaxFrequency); - } - - [Theory] - [InlineData("Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", 2.50)] - [InlineData("Intel(R) Core(TM) i5-6200U CPU @ 2.30GHz", 2.30)] - [InlineData("Unknown processor with 2 cores and hyper threading, Unknown processor with 4 cores", 0)] - [InlineData("Intel(R) Core(TM) i5-2500 CPU @ 3.30GHz", 3.30)] - public void ParseFrequencyFromBrandStringTests(string brandString, double expectedGHz) - { - var frequency = ProcCpuInfoParser.ParseFrequencyFromBrandString(brandString); - Assert.Equal(Frequency.FromGHz(expectedGHz), frequency); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/SysctlCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/SysctlCpuInfoParserTests.cs deleted file mode 100644 index 5d2ba82520..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/SysctlCpuInfoParserTests.cs +++ /dev/null @@ -1,43 +0,0 @@ -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; -using Xunit; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - public class SysctlCpuInfoParserTests - { - [Fact] - public void EmptyTest() - { - var parser = SysctlCpuInfoParser.ParseOutput(string.Empty); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - } - - [Fact] - public void MalformedTest() - { - var parser = SysctlCpuInfoParser.ParseOutput("malformedkey=malformedvalue\n\nmalformedkey2=malformedvalue2"); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - } - - [Fact] - public void RealOneProcessorFourCoresTest() - { - string cpuInfo = TestHelper.ReadTestFile("SysctlRealOneProcessorFourCores.txt"); - var parser = SysctlCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Core(TM) i7-4770HQ CPU @ 2.20GHz", parser.ProcessorName); - Assert.Equal(1, parser.PhysicalProcessorCount); - Assert.Equal(4, parser.PhysicalCoreCount); - Assert.Equal(8, parser.LogicalCoreCount); - Assert.Equal(2200 * Frequency.MHz, parser.NominalFrequency); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestHelper.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestHelper.cs deleted file mode 100644 index e4af5025c0..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/TestHelper.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System.IO; -using System.Reflection; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - public static class TestHelper - { - public static string ReadTestFile(string name) - { - var assembly = typeof(TestHelper).GetTypeInfo().Assembly; - string resourceName = $"{typeof(TestHelper).Namespace}.TestFiles.{name}"; - - using (var stream = assembly.GetManifestResourceStream(resourceName)) - using (var reader = new StreamReader(stream)) - { - return reader.ReadToEnd(); - } - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Portability/Cpu/WmicCpuInfoParserTests.cs b/tests/BenchmarkDotNet.Tests/Portability/Cpu/WmicCpuInfoParserTests.cs deleted file mode 100644 index b1faa4b521..0000000000 --- a/tests/BenchmarkDotNet.Tests/Portability/Cpu/WmicCpuInfoParserTests.cs +++ /dev/null @@ -1,103 +0,0 @@ -using BenchmarkDotNet.Portability.Cpu; -using Perfolizer.Horology; -using Xunit; - -namespace BenchmarkDotNet.Tests.Portability.Cpu -{ - public class WmicCpuInfoParserTests - { - [Fact] - public void EmptyTest() - { - var parser = WmicCpuInfoParser.ParseOutput(string.Empty); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - } - - [Fact] - public void MalformedTest() - { - var parser = WmicCpuInfoParser.ParseOutput("malformedkey=malformedvalue\n\nmalformedkey2=malformedvalue2"); - Assert.Null(parser.ProcessorName); - Assert.Null(parser.PhysicalProcessorCount); - Assert.Null(parser.PhysicalCoreCount); - Assert.Null(parser.LogicalCoreCount); - Assert.Null(parser.NominalFrequency); - } - - [Fact] - public void RealTwoProcessorEightCoresTest() - { - const string cpuInfo = @" - -MaxClockSpeed=2400 -Name=Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz -NumberOfCores=8 -NumberOfLogicalProcessors=16 - - -MaxClockSpeed=2400 -Name=Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz -NumberOfCores=8 -NumberOfLogicalProcessors=16 - -"; - var parser = WmicCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz", parser.ProcessorName); - Assert.Equal(2, parser.PhysicalProcessorCount); - Assert.Equal(16, parser.PhysicalCoreCount); - Assert.Equal(32, parser.LogicalCoreCount); - Assert.Equal(2400 * Frequency.MHz, parser.MaxFrequency); - } - - [Fact] - public void RealTwoProcessorEightCoresWithWmicBugTest() - { - const string cpuInfo = - "\r\r\n" + - "\r\r\n" + - "MaxClockSpeed=3111\r\r\n" + - "Name=Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz\r\r\n" + - "NumberOfCores=8\r\r\n" + - "NumberOfLogicalProcessors=16\r\r\n" + - "\r\r\n" + - "\r\r\n" + - "MaxClockSpeed=3111\r\r\n" + - "Name=Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz\r\r\n" + - "NumberOfCores=8\r\r\n" + - "NumberOfLogicalProcessors=16\r\r\n" + - "\r\r\n" + - "\r\r\n" + - "\r\r\n"; - var parser = WmicCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Xeon(R) CPU E5-2687W 0 @ 3.10GHz", parser.ProcessorName); - Assert.Equal(2, parser.PhysicalProcessorCount); - Assert.Equal(16, parser.PhysicalCoreCount); - Assert.Equal(32, parser.LogicalCoreCount); - Assert.Equal(3111 * Frequency.MHz, parser.MaxFrequency); - } - - [Fact] - public void RealOneProcessorFourCoresTest() - { - const string cpuInfo = @" - -MaxClockSpeed=2500 -Name=Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz -NumberOfCores=4 -NumberOfLogicalProcessors=8 - -"; - - var parser = WmicCpuInfoParser.ParseOutput(cpuInfo); - Assert.Equal("Intel(R) Core(TM) i7-4710MQ CPU @ 2.50GHz", parser.ProcessorName); - Assert.Equal(1, parser.PhysicalProcessorCount); - Assert.Equal(4, parser.PhysicalCoreCount); - Assert.Equal(8, parser.LogicalCoreCount); - Assert.Equal(2500 * Frequency.MHz, parser.MaxFrequency); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/ReflectionTests.cs b/tests/BenchmarkDotNet.Tests/ReflectionTests.cs index a32993a034..0d1ea69673 100644 --- a/tests/BenchmarkDotNet.Tests/ReflectionTests.cs +++ b/tests/BenchmarkDotNet.Tests/ReflectionTests.cs @@ -38,7 +38,7 @@ public void GetCorrectCSharpTypeNameSupportsGenericTypesPassedByReference() public class GenericByRef { - public void TheMethod(ref ValueTuple _) { } + public void TheMethod(ref (int, short) _) { } } [Fact] diff --git a/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs b/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs index ebea88b5ad..fb15149aa3 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/SummaryTableTests.cs @@ -1,6 +1,8 @@ using System.Linq; +using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Order; @@ -171,5 +173,311 @@ public void MissingValueInMetricColumnIsNA() // assert Assert.Equal(new[] { "-", "NA" }, actual); } + + #region Issue #2673 + [Fact] + public void DefaultExceptionDiagnoserConfig_WhenExceptionsIsNotZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var exceptionConfig = new ExceptionDiagnoserConfig(); + + var exceptionsFrequencyMetricDescriptor = new ExceptionDiagnoser.ExceptionsFrequencyMetricDescriptor(exceptionConfig); + + var metric = new Metric(exceptionsFrequencyMetricDescriptor, 5); + var metrics = new[] { metric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + var actual = table.Columns.First(c => c.Header == "Exceptions").Content; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, actual); + } + + [Fact] + public void DefaultExceptionDiagnoserConfig_WhenExceptionsIsZero() + { + + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var exceptionConfig = new ExceptionDiagnoserConfig(); + + var exceptionsFrequencyMetricDescriptor = new ExceptionDiagnoser.ExceptionsFrequencyMetricDescriptor(exceptionConfig); + + var metric = new Metric(exceptionsFrequencyMetricDescriptor, 0); + var metrics = new[] { metric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + var actual = table.Columns.First(c => c.Header == "Exceptions").Content; + + // assert + Assert.Equal(new[] { "-", "-" }, actual); + } + + [Fact] + public void HideExceptionDiagnoserConfig_WhenExceptionsIsNotZero() + { + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var exceptionConfig = new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: false); + + var exceptionsFrequencyMetricDescriptor = new ExceptionDiagnoser.ExceptionsFrequencyMetricDescriptor(exceptionConfig); + + var metric = new Metric(exceptionsFrequencyMetricDescriptor, 5); + var metrics = new[] { metric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + var actual = table.Columns.First(c => c.Header == "Exceptions").Content; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, actual); + } + + [Fact] + public void HideExceptionDiagnoserConfig_WhenExceptionsIsZero() + { + + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var exceptionConfig = new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: false); + + var exceptionsFrequencyMetricDescriptor = new ExceptionDiagnoser.ExceptionsFrequencyMetricDescriptor(exceptionConfig); + + var metric = new Metric(exceptionsFrequencyMetricDescriptor, 0); + var metrics = new[] { metric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + var isExist = table.Columns.Any(c => c.Header == "Exceptions"); + + // assert + Assert.False(isExist); + } + + [Fact] + public void DefaultThreadingDiagnoserConfig_WhenDescriptorValuesAreNotZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 5); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 5); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + var lockContentionCount = table.Columns.FirstOrDefault(c => c.Header == "Lock Contentions").Content; + var completedWorkItemCount = table.Columns.FirstOrDefault(c => c.Header == "Completed Work Items").Content; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, lockContentionCount); + Assert.Equal(new[] { "5.0000", "5.0000" }, completedWorkItemCount); + } + + [Fact] + public void DefaultThreadingDiagnoserConfig_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + var lockContentionCount = table.Columns.FirstOrDefault(c => c.Header == "Lock Contentions").Content; + var completedWorkItemCount = table.Columns.FirstOrDefault(c => c.Header == "Completed Work Items").Content; + + // assert + Assert.Equal(new[] { "-", "-" }, lockContentionCount); + Assert.Equal(new[] { "-", "-" }, completedWorkItemCount); + } + + [Fact] + public void HideLockContentionCountThreadingDiagnoserConfig_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayLockContentionWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Null(lockContentionCount); + Assert.Equal(new[] { "-", "-" }, completedWorkItemCount); + } + + [Fact] + public void HideLockContentionCountThreadingDiagnoserConfig_WhenDescriptorValuesAreNotZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayLockContentionWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 5); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 5); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, lockContentionCount); + Assert.Equal(new[] { "5.0000", "5.0000" }, completedWorkItemCount); + } + + [Fact] + public void HideCompletedWorkItemCountThreadingDiagnoserConfig_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Null(completedWorkItemCount); + Assert.Equal(new[] { "-", "-" }, lockContentionCount); + } + + [Fact] + public void HideCompletedWorkItemCountThreadingDiagnoserConfig_WhenDescriptorValuesAreNotZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 5); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 5); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Equal(new[] { "5.0000", "5.0000" }, lockContentionCount); + Assert.Equal(new[] { "5.0000", "5.0000" }, completedWorkItemCount); + } + + [Fact] + public void HideThreadingDiagnoserConfigs_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false, displayLockContentionWhenZero: false); + + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Null(lockContentionCount); + Assert.Null(completedWorkItemCount); + } + + [Fact] + public void DisplayThreadingDiagnoserConfigs_WhenDescriptorValuesAreZero() + { + + // arrange + var config = ManualConfig.Create(DefaultConfig.Instance); + var threadingConfig = new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: true, displayLockContentionWhenZero: true); + var lockContentionCountMetricDescriptor = new ThreadingDiagnoser.LockContentionCountMetricDescriptor(threadingConfig); + var completedWorkItemCountMetricDescriptor = new ThreadingDiagnoser.CompletedWorkItemCountMetricDescriptor(threadingConfig); + + var lockContentionCountMetric = new Metric(lockContentionCountMetricDescriptor, 0); + var completedWorkItemMetric = new Metric(completedWorkItemCountMetricDescriptor, 0); + var metrics = new[] { lockContentionCountMetric, completedWorkItemMetric }; + + // act + var summary = MockFactory.CreateSummary(config, hugeSd: false, metrics); + var table = new SummaryTable(summary); + + string[]? lockContentionCount = table.Columns?.FirstOrDefault(c => c.Header == "Lock Contentions")?.Content ?? null; + string[]? completedWorkItemCount = table.Columns?.FirstOrDefault(c => c.Header == "Completed Work Items")?.Content ?? null; + + // assert + Assert.Equal(new[] { "-", "-" }, lockContentionCount); + Assert.Equal(new[] { "-", "-" }, completedWorkItemCount); + } + #endregion } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs b/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs index 8952accb40..92811e5933 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs @@ -35,7 +35,7 @@ public void SummaryWithFailureReportDoesNotThrowNre() private static IConfig CreateConfig() { // We use runtime as selector later. It is chosen as selector just to be close to initial issue. Nothing particularly special about it. - Job coreJob = new Job(Job.Default).WithRuntime(CoreRuntime.Core20).ApplyAndFreeze(RunMode.Dry); + Job coreJob = new Job(Job.Default).WithRuntime(CoreRuntime.Core80).ApplyAndFreeze(RunMode.Dry); Job clrJob = new Job(Job.Default).WithRuntime(ClrRuntime.Net462).ApplyAndFreeze(RunMode.Dry); return ManualConfig.Create(DefaultConfig.Instance).AddJob(coreJob).AddJob(clrJob); } diff --git a/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs b/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs index 4eb72b5d59..4b3e92d95d 100644 --- a/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs @@ -135,7 +135,7 @@ public void JobMutatorsApplySettingsToAllNonMutatorJobs() typeof(WithMutator), DefaultConfig.Instance .AddJob(Job.Default.WithRuntime(ClrRuntime.Net462)) - .AddJob(Job.Default.WithRuntime(CoreRuntime.Core21))); + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80))); Assert.Equal(2, info.BenchmarksCases.Length); Assert.All(info.BenchmarksCases, benchmark => Assert.Equal(int.MaxValue, benchmark.Job.Run.MaxIterationCount)); diff --git a/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs b/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs index e04409beb9..4ace61168b 100644 --- a/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs @@ -95,6 +95,7 @@ public void CustomClrBuildJobsAreGroupedByVersion() } [Fact] + [System.Obsolete] public void CustomNuGetJobsAreGroupedByPackageVersion() { var config = ManualConfig.Create(DefaultConfig.Instance) diff --git a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs new file mode 100644 index 0000000000..05ce92b2cf --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs @@ -0,0 +1,359 @@ +using System; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Running; +using Xunit; +using System.Reflection; +using System.Reflection.Emit; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using System.Runtime.InteropServices; + +namespace BenchmarkDotNet.Tests.Running +{ + public class RunningEmptyBenchmarkTests + { + #region BenchmarkRunner Methods Overview + /* + * Available BenchmarkRunner.Run methods: + * 1. Generic Type: + * - BenchmarkRunner.Run(IConfig? config = null, string[]? args = null) + * 2. Type-based: + * - BenchmarkRunner.Run(Type type, IConfig? config = null, string[]? args = null) + * - BenchmarkRunner.Run(Type[] types, IConfig? config = null, string[]? args = null) + * - BenchmarkRunner.Run(Type type, MethodInfo[] methods, IConfig? config = null) + * 3. Assembly-based: + * - BenchmarkRunner.Run(Assembly assembly, IConfig? config = null, string[]? args = null) + * 4. BenchmarkRunInfo-based: + * - BenchmarkRunner.Run(BenchmarkRunInfo benchmarkRunInfo) + * - BenchmarkRunner.Run(BenchmarkRunInfo[] benchmarkRunInfos) + * 5. Deprecated methods: + * - BenchmarkRunner.RunUrl(string url, IConfig? config = null) + * - BenchmarkRunner.RunSource(string source, IConfig? config = null) + */ + #endregion + #region Generic Type Tests + /// + /// Tests for BenchmarkRunner.Run method + /// + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void GenericTypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(config, args); + if (args == null) + { + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + } + + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void GenericTypeWithBenchmarkAttribute_RunsSuccessfully(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(config, args); + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region Type-based Tests + /// + /// Tests for BenchmarkRunner.Run(Type) method + /// + [Theory] + [InlineData(null)] + //[InlineData(new object[] { new string[] { " " } })] + public void TypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + + var summary = BenchmarkRunner.Run(typeof(EmptyBenchmark), config, args); + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypeWithBenchmarkAttribute_RunsSuccessfully(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(typeof(NotEmptyBenchmark), config, args); + Assert.False(summaries.HasCriticalValidationErrors); + Assert.DoesNotContain(summaries.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + + /// + /// Tests for BenchmarkRunner.Run(Type[]) method + /// + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypesWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(EmptyBenchmark), typeof(EmptyBenchmark2) }, config, args); + if (args != null) + { + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); + } + else + { + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark2))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); + } + + + } + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void TypesWithBenchmarkAttribute_RunsSuccessfully(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(NotEmptyBenchmark) }, config, args); + var summary = summaries[0]; + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region BenchmarkRunInfo Tests + /// + /// Tests for BenchmarkRunner.Run(BenchmarkRunInfo) method + /// + [Fact] + public void BenchmarkRunInfoWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute() + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), config)); + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + + [Fact] + public void BenchmarkRunInfoWithBenchmarkAttribute_RunsSuccessfully() + { + GetConfigWithLogger(out var logger, out var config); + + var summary = BenchmarkRunner.Run(BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), config)); + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + + /// + /// Tests for BenchmarkRunner.Run(BenchmarkRunInfo[]) method + /// + [Fact] + public void BenchmarkRunInfosWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute() + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { + BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark), config), + BenchmarkConverter.TypeToBenchmarks(typeof(EmptyBenchmark2), config) + }); + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark2))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark2)), logger.GetLog()); + } + + [Fact] + public void BenchmarkRunInfosWithBenchmarkAttribute_RunsSuccessfully() + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(NotEmptyBenchmark), config) }); + var summary = summaries[0]; + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(NotEmptyBenchmark))); + Assert.DoesNotContain(GetValidationErrorForType(typeof(NotEmptyBenchmark)), logger.GetLog()); + } + #endregion + #region Mixed Types Tests + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void MixedTypes_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) + { + GetConfigWithLogger(out var logger, out var config); + + var summaries = BenchmarkRunner.Run(new[] { typeof(EmptyBenchmark), typeof(NotEmptyBenchmark) }, config, args); + if (args != null) + { + Assert.Contains(GetExpandedValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + else + { + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + } + #endregion + #region Assembly Tests + // In this tests there is no config and logger because the logger is initiated at CreateCompositeLogger when the BenchmarkRunInfo[] is empty + // those cannot be inserted using config + [Theory] + + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void AssemblyWithoutBenchmarks_ThrowsValidationError_WhenNoBenchmarksFound(string[]? args) + { + + // Create a mock assembly with no benchmark types + var assemblyName = new AssemblyName("MockAssembly"); + var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run); + var moduleBuilder = assemblyBuilder.DefineDynamicModule("MockModule"); + // Create a simple type in the assembly (no benchmarks) + var typeBuilder = moduleBuilder.DefineType("MockType", TypeAttributes.Public); + typeBuilder.CreateType(); + + Summary[] summaries = null; + if (args != null) + { + GetConfigWithLogger(out var logger, out var config); + summaries = BenchmarkRunner.Run(assemblyBuilder, config, args); + Assert.Contains(GetAssemblylValidationError(assemblyBuilder), logger.GetLog()); + } + else + { + summaries = BenchmarkRunner.Run(assemblyBuilder, null, args); + var summary = summaries[0]; + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetGeneralValidationError()); + } + } + + [Theory] + [InlineData(null)] + [InlineData(new object[] { new string[] { " " } })] + public void AssemblyWithBenchmarks_RunsSuccessfully_WhenBenchmarkAttributePresent(string[]? args) + { + // Skip test on .NET Framework 4.6.2 + if (RuntimeInformation.FrameworkDescription.Contains(".NET Framework 4")) + return; + + // Create a mock assembly with benchmark types + var assemblyName = new AssemblyName("MockAssemblyWithBenchmarks"); + var assemblyBuilder = AssemblyBuilder.DefineDynamicAssembly(assemblyName, AssemblyBuilderAccess.Run); + var moduleBuilder = assemblyBuilder.DefineDynamicModule("MockModule"); + + // Create a benchmark type + var benchmarkTypeBuilder = moduleBuilder.DefineType("MockBenchmark", TypeAttributes.Public); + var benchmarkMethod = benchmarkTypeBuilder.DefineMethod("Benchmark", MethodAttributes.Public, typeof(void), Type.EmptyTypes); + + // Generate method body + var ilGenerator = benchmarkMethod.GetILGenerator(); + ilGenerator.Emit(OpCodes.Ret); // Just return from the method + + var benchmarkAttributeCtor = typeof(BenchmarkAttribute).GetConstructor(new[] { typeof(int), typeof(string) }); + if (benchmarkAttributeCtor == null) + throw new InvalidOperationException("Could not find BenchmarkAttribute constructor"); + benchmarkMethod.SetCustomAttribute(new CustomAttributeBuilder( + benchmarkAttributeCtor, + new object[] { 0, "" })); + benchmarkTypeBuilder.CreateType(); + + Summary[] summaries = null; + if (args != null) + { + GetConfigWithLogger(out var logger, out var config); + summaries = BenchmarkRunner.Run(assemblyBuilder, config, args); + Assert.DoesNotContain(GetAssemblylValidationError(assemblyBuilder), logger.GetLog()); + } + else + { + summaries = BenchmarkRunner.Run(assemblyBuilder); + var summary = summaries[0]; + Assert.False(summary.HasCriticalValidationErrors); + Assert.DoesNotContain(summary.ValidationErrors, validationError => validationError.Message == GetGeneralValidationError()); + } + } + #endregion + #region Helper Methods + private string GetValidationErrorForType(Type type) + { + return $"No [Benchmark] attribute found on '{type.Name}' benchmark case."; + } + + private string GetAssemblylValidationError(Assembly assembly) + { + return $"No [Benchmark] attribute found on '{assembly.GetName().Name}' assembly."; + } + + private string GetExpandedValidationErrorForType(Type type) + { + return $"Type {type} is invalid. Only public, non-generic (closed generic types with public parameterless ctors are supported), non-abstract, non-sealed, non-static types with public instance [Benchmark] method(s) are supported."; + } + + private string GetGeneralValidationError() + { + return $"No benchmarks were found."; + } + + private void GetConfigWithLogger(out AccumulationLogger logger, out ManualConfig manualConfig) + { + logger = new AccumulationLogger(); + manualConfig = ManualConfig.CreateEmpty() + .AddLogger(logger) + .AddColumnProvider(DefaultColumnProviders.Instance); + } + + #endregion + #region Test Classes + public class EmptyBenchmark + { + } + + public class EmptyBenchmark2 + { + } + + [SimpleJob(launchCount: 1, warmupCount: 1, iterationCount: 1, invocationCount: 1, id: "QuickJob")] + public class NotEmptyBenchmark + { + [Benchmark] + public void Benchmark() + { + var sum = 0; + for (int i = 0; i < 1; i++) + { + sum += i; + } + } + } + #endregion + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs b/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs index 29f8a65798..f6286720fd 100644 --- a/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs +++ b/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; +using BenchmarkDotNet.Detectors; using Xunit; namespace BenchmarkDotNet.Tests @@ -17,6 +18,10 @@ public class RuntimeVersionDetectionTests [InlineData(".NETCoreApp,Version=v3.0", RuntimeMoniker.NetCoreApp30, "netcoreapp3.0")] [InlineData(".NETCoreApp,Version=v3.1", RuntimeMoniker.NetCoreApp31, "netcoreapp3.1")] [InlineData(".NETCoreApp,Version=v5.0", RuntimeMoniker.Net50, "net5.0")] + [InlineData(".NETCoreApp,Version=v6.0", RuntimeMoniker.Net60, "net6.0")] + [InlineData(".NETCoreApp,Version=v7.0", RuntimeMoniker.Net70, "net7.0")] + [InlineData(".NETCoreApp,Version=v8.0", RuntimeMoniker.Net80, "net8.0")] + [InlineData(".NETCoreApp,Version=v9.0", RuntimeMoniker.Net90, "net9.0")] [InlineData(".NETCoreApp,Version=v123.0", RuntimeMoniker.NotRecognized, "net123.0")] public void TryGetVersionFromFrameworkNameHandlesValidInput(string frameworkName, RuntimeMoniker expectedTfm, string expectedMsBuildMoniker) { @@ -110,7 +115,7 @@ public void CurrentRuntimeIsProperlyRecognized() var runtime = RuntimeInformation.GetCurrentRuntime(); #if NETFRAMEWORK - if (RuntimeInformation.IsWindows()) + if (OsDetector.IsWindows()) Assert.True(runtime is ClrRuntime); else Assert.True(runtime is MonoRuntime); @@ -124,6 +129,8 @@ public void CurrentRuntimeIsProperlyRecognized() Assert.True(runtime is CoreRuntime coreRuntime && coreRuntime.RuntimeMoniker == RuntimeMoniker.NetCoreApp31); #elif NETCOREAPP5_0 Assert.True(runtime is CoreRuntime coreRuntime && coreRuntime.RuntimeMoniker == RuntimeMoniker.NetCoreApp50); +#elif NET8_0 + Assert.True(runtime is CoreRuntime coreRuntime && coreRuntime.RuntimeMoniker == RuntimeMoniker.Net80); #endif } } diff --git a/tests/BenchmarkDotNet.Tests/SimpleJsonTests.cs b/tests/BenchmarkDotNet.Tests/SimpleJsonTests.cs index 2bb57fe436..fd599aea5b 100644 --- a/tests/BenchmarkDotNet.Tests/SimpleJsonTests.cs +++ b/tests/BenchmarkDotNet.Tests/SimpleJsonTests.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; using System.Linq; using Xunit; -using JsonSerializer = SimpleJson.SimpleJson; +using JsonSerializer = SimpleJson.SimpleJsonSerializer; namespace BenchmarkDotNet.Tests { diff --git a/tests/BenchmarkDotNet.Tests/Validators/CompilationValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/CompilationValidatorTests.cs index 632875d52b..683e258bbe 100644 --- a/tests/BenchmarkDotNet.Tests/Validators/CompilationValidatorTests.cs +++ b/tests/BenchmarkDotNet.Tests/Validators/CompilationValidatorTests.cs @@ -7,6 +7,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Validators; using Xunit; +using System.Reflection; namespace BenchmarkDotNet.Tests.Validators { @@ -59,6 +60,29 @@ public void BenchmarkedMethodNameMustNotUseCsharpKeywords() "Benchmarked method `typeof` contains illegal character(s) or uses C# keyword. Please use `[]` to set custom display name.")); } + [Theory] + /* BenchmarkDotNet can only benchmark public unsealed classes*/ + [InlineData(typeof(BenchMarkPublicClass), false)] + [InlineData(typeof(BenchMarkPublicClass.PublicNestedClass), false)] + [InlineData(typeof(SealedClass.PublicNestedClass), false)] + [InlineData(typeof(OuterClass.PublicNestedClass), true)] + [InlineData(typeof(SealedClass), true)] + [InlineData(typeof(MyPrivateClass), true)] + [InlineData(typeof(MyPublicProtectedClass), true)] + [InlineData(typeof(MyPrivateProtectedClass), true)] + [InlineData(typeof(MyProtectedInternalClass), true)] + [InlineData(typeof(MyInternalClass), true)] + [InlineData(typeof(OuterClass), true)] + [InlineData(typeof(OuterClass.InternalNestedClass), true)] + [InlineData(typeof(BenchMarkPublicClass.InternalNestedClass), true)] + /* Generics Remaining */ + public void Benchmark_Class_Modifers_Must_Be_Public(Type type, bool hasErrors) + { + var validationErrors = CompilationValidator.FailOnError.Validate(BenchmarkConverter.TypeToBenchmarks(type)); + + Assert.Equal(hasErrors, validationErrors.Any()); + } + [Theory] [InlineData(typeof(BenchmarkClassWithStaticMethod), true)] [InlineData(typeof(BenchmarkClass), false)] @@ -115,7 +139,17 @@ protected internal class ProtectedInternalClass { protected internal class ProtectedInternalNestedClass { } } - } + + private class MyPrivateClass{ [Benchmark] public void PublicMethod(){} } + + protected class MyPublicProtectedClass{ [Benchmark] public void PublicMethod(){} } + + private protected class MyPrivateProtectedClass{ [Benchmark] public void PublicMethod(){} } + + internal class MyInternalClass{ [Benchmark] public void PublicMethod(){} } + + protected internal class MyProtectedInternalClass{ [Benchmark] public void PublicMethod() { } } + } public class BenchmarkClassWithStaticMethod { @@ -138,4 +172,29 @@ internal class InternalClass { internal class InternalNestedClass { } } + + public sealed class SealedClass + { + [Benchmark] public void PublicMethod() { } + + public class PublicNestedClass { [Benchmark] public void PublicMethod() { } } + } + + internal class OuterClass + { + [Benchmark] public void PublicMethod(){} + + internal class InternalNestedClass { [Benchmark] public void PublicMethod() { } } + + public class PublicNestedClass { [Benchmark] public void PublicMethod() { } } + } + + public class BenchMarkPublicClass + { + [Benchmark] public void PublicMethod(){} + + public class PublicNestedClass { [Benchmark] public void PublicMethod() { } } + + internal class InternalNestedClass { [Benchmark] public void PublicMethod() { } } + } } diff --git a/tests/BenchmarkDotNet.Tests/Validators/ConfigCompatibilityValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/ConfigCompatibilityValidatorTests.cs deleted file mode 100644 index fc73793980..0000000000 --- a/tests/BenchmarkDotNet.Tests/Validators/ConfigCompatibilityValidatorTests.cs +++ /dev/null @@ -1,180 +0,0 @@ -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Columns; -using BenchmarkDotNet.Order; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Tests.Loggers; -using BenchmarkDotNet.Validators; -using System.Linq; -using BenchmarkDotNet.Portability; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.Tests.Validators -{ - public class ConfigCompatibilityValidatorTests - { - private ITestOutputHelper Output { get; } - - public ConfigCompatibilityValidatorTests(ITestOutputHelper output) => Output = output; - - [Fact] - public void RunningBenchmarksWithIncompatibleConfigsMustFailWithCriticalError() - { - if (!KnownIssue.Issue2299.IsFixed && RuntimeInformation.IsMono && RuntimeInformation.IsLinux()) - { - Output.WriteLine(KnownIssue.Issue2299.IgnoreMessage); - return; - } - - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty().AddLogger(logger); - var summary = - BenchmarkSwitcher - .FromTypes(new[] { typeof(BenchmarkClassWithExtraOrderer1), typeof(BenchmarkClassWithExtraOrderer2) }) - .RunAllJoined(config); - Assert.True(summary.HasCriticalValidationErrors); - Assert.Contains("You use JoinSummary options, but provided configurations cannot be joined", logger.GetLog()); - Assert.Contains("Orderer", logger.GetLog()); - } - - [Fact] - public void JoinedBenchmarksMustNotHaveDifferentExtraOrderers() - { - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithExtraOrderer1)), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithExtraOrderer2)) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.NotEmpty(validationErrors); - Assert.StartsWith("You use JoinSummary options, but provided configurations cannot be joined", validationErrors.Single().Message); - Assert.Contains("Orderer", validationErrors.Single().Message); - } - - [Fact] - public void JoinedBenchmarksMayHaveOneExtraOrderer() - { - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithExtraOrderer1)), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults1)) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.Empty(validationErrors); - } - - [Fact] - public void JoinedBenchmarksMayHaveDefaultOrderers() - { - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults1)), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults2)) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.Empty(validationErrors); - } - - [Fact] - public void JoinedBenchmarksMustNotHaveDifferentExtraSummaryStyles() - { - // Note: RatioStyle.Value would be the same as SummaryStyle.Default (SummaryStyle implements IEquatable). - ManualConfig config1 = DefaultConfig.Instance.WithSummaryStyle( - SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend)); - ManualConfig config2 = DefaultConfig.Instance.WithSummaryStyle( - SummaryStyle.Default.WithRatioStyle(RatioStyle.Percentage)); - - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults1), config1), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults2), config2) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.NotEmpty(validationErrors); - Assert.StartsWith("You use JoinSummary options, but provided configurations cannot be joined", validationErrors.Single().Message); - Assert.Contains("SummaryStyle", validationErrors.Single().Message); - } - - [Fact] - public void JoinedBenchmarksMayHaveOneExtraSummaryStyle() - { - ManualConfig config = DefaultConfig.Instance.WithSummaryStyle( - SummaryStyle.Default.WithZeroMetricValuesInContent()); - var benchmarks = new[] - { - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults1), null), - BenchmarkConverter.TypeToBenchmarks(typeof(BenchmarkClassWithDefaults2), config) - }; - - var cases = benchmarks.SelectMany(b => b.BenchmarksCases).ToArray(); - - var validationErrors = - ConfigCompatibilityValidator - .FailOnError - .Validate(new ValidationParameters(cases, null)) - .ToArray(); - - Assert.Empty(validationErrors); - } - - [Orderer(SummaryOrderPolicy.Method)] - public class BenchmarkClassWithExtraOrderer1 - { - [Benchmark] - public void Foo() { } - } - - [Orderer(SummaryOrderPolicy.Method)] - public class BenchmarkClassWithExtraOrderer2 - { - [Benchmark] - public void Bar() { } - } - - public class BenchmarkClassWithDefaults1 - { - [Benchmark] - public void Baz() { } - } - - public class BenchmarkClassWithDefaults2 - { - [Benchmark] - public void Buzz() { } - } - } -} diff --git a/tests/BenchmarkDotNet.Tests/Validators/ExecutionValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/ExecutionValidatorTests.cs index fece3b3939..70195fd959 100644 --- a/tests/BenchmarkDotNet.Tests/Validators/ExecutionValidatorTests.cs +++ b/tests/BenchmarkDotNet.Tests/Validators/ExecutionValidatorTests.cs @@ -591,7 +591,7 @@ public void AsyncValueTaskBackedByIValueTaskSourceIsAwaitedProperly() public class AsyncValueTaskSource { - private readonly ValueTaskSource valueTaskSource = new (); + private readonly ValueTaskSource valueTaskSource = new(); public static bool WasCalled; @@ -622,7 +622,7 @@ public void AsyncGenericValueTaskBackedByIValueTaskSourceIsAwaitedProperly() public class AsyncGenericValueTaskSource { - private readonly ValueTaskSource valueTaskSource = new (); + private readonly ValueTaskSource valueTaskSource = new(); public static bool WasCalled; diff --git a/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs new file mode 100644 index 0000000000..9eac2dd6f2 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs @@ -0,0 +1,125 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Validators; +using System.Linq; +using Xunit; + +namespace BenchmarkDotNet.Tests.Validators; + +public class RuntimeValidatorTests +{ + [Fact] + public void SameRuntime_Should_Success() + { + // Arrange + var config = new TestConfig1().CreateImmutableConfig(); + var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); + var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); + + // Act + var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray(); + + // Assert + Assert.Empty(errors); + } + + [Fact] + public void NullRuntimeMixed_Should_Failed() + { + // Arrange + var config = new TestConfig2().CreateImmutableConfig(); + var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); + var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); + + // Act + var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray(); + + // Assert + { + var expectedMessage = "Job(Dry) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; + Assert.Contains(expectedMessage, errors); + } + { + var expectedMessage = "Job(Toolchain=.NET 10.0) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; + Assert.Contains(expectedMessage, errors); + } + } + + [Fact] + public void NotNullRuntimeOnly_Should_Success() + { + // Arrange + var config = new TestConfig3().CreateImmutableConfig(); + var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); + var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); + + // Act + var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray(); + + // Assert + Assert.Empty(errors); + } + + public class DummyBenchmark + { + [Benchmark] + public void Benchmark() + { + } + } + + // TestConfig that expicitly specify runtime. + private class TestConfig1 : ManualConfig + { + public TestConfig1() + { + var baseJob = Job.Dry; + + WithOption(ConfigOptions.DisableOptimizationsValidator, true); + AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); + + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80)); + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90)); + } + } + + // TestConfig that contains job that don't specify runtime. + private class TestConfig2 : ManualConfig + { + public TestConfig2() + { + var baseJob = Job.Dry; + + WithOption(ConfigOptions.DisableOptimizationsValidator, true); + AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); + + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80)); + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90)); + + // Validate error message for auto generated jobid. + AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0)); + } + } + + // TestConfig that expicitly specify runtime. + private class TestConfig3 : ManualConfig + { + public TestConfig3() + { + var baseJob = Job.Dry; + + WithOption(ConfigOptions.DisableOptimizationsValidator, true); + AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); + + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80) + .WithRuntime(CoreRuntime.Core80)); ; + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90)); + } + } +} diff --git a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs index cb2b1a3878..79bfb5a1ae 100644 --- a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs +++ b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs @@ -8,5 +8,5 @@ public enum EnvRequirement FullFrameworkOnly, NonFullFramework, DotNetCoreOnly, - DotNetCore30Only + NeedsPrivilegedProcess } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs index e0510b92f6..22f131455d 100644 --- a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs +++ b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Runtime.InteropServices; +using System.Security.Principal; using BenchmarkDotNet.Jobs; using JetBrains.Annotations; using BdnRuntimeInformation = BenchmarkDotNet.Portability.RuntimeInformation; @@ -19,9 +20,19 @@ public static class EnvRequirementChecker EnvRequirement.FullFrameworkOnly => BdnRuntimeInformation.IsFullFramework ? null : "Full .NET Framework-only test", EnvRequirement.NonFullFramework => !BdnRuntimeInformation.IsFullFramework ? null : "Non-Full .NET Framework test", EnvRequirement.DotNetCoreOnly => BdnRuntimeInformation.IsNetCore ? null : ".NET/.NET Core-only test", - EnvRequirement.DotNetCore30Only => IsRuntime(RuntimeMoniker.NetCoreApp30) ? null : ".NET Core 3.0-only test", + EnvRequirement.NeedsPrivilegedProcess => IsPrivilegedProcess() ? null : "Needs authorization to perform security-relevant functions", _ => throw new ArgumentOutOfRangeException(nameof(requirement), requirement, "Unknown value") }; + private static bool IsPrivilegedProcess() + { +#if NET462 + using WindowsIdentity currentUser = WindowsIdentity.GetCurrent(); + return new WindowsPrincipal(currentUser).IsInRole(WindowsBuiltInRole.Administrator); +#else + return Environment.IsPrivilegedProcess; +#endif + } + private static bool IsRuntime(RuntimeMoniker moniker) => BdnRuntimeInformation.GetCurrentRuntime().RuntimeMoniker == moniker; } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/dotMemory/DotMemoryTests.cs b/tests/BenchmarkDotNet.Tests/dotMemory/DotMemoryTests.cs index 7142486280..d3aec8dd96 100644 --- a/tests/BenchmarkDotNet.Tests/dotMemory/DotMemoryTests.cs +++ b/tests/BenchmarkDotNet.Tests/dotMemory/DotMemoryTests.cs @@ -3,15 +3,15 @@ using BenchmarkDotNet.Jobs; using Xunit; -namespace BenchmarkDotNet.Tests.dotMemory +namespace BenchmarkDotNet.Tests.dotMemory; + +public class DotMemoryTests { - public class DotMemoryTests + [Fact] + public void AllRuntimeMonikerAreKnown() { - [Fact] - public void AllRuntimeMonikerAreKnown() - { - foreach (RuntimeMoniker moniker in Enum.GetValues(typeof(RuntimeMoniker))) - DotMemoryDiagnoser.IsSupported(moniker); // Just check that it doesn't throw exceptions - } + var diagnoser = new DotMemoryDiagnoser(); + foreach (RuntimeMoniker moniker in Enum.GetValues(typeof(RuntimeMoniker))) + diagnoser.IsSupported(moniker); // Just check that it doesn't throw exceptions } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs b/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs index a71cae2aef..751412fd55 100644 --- a/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs +++ b/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs @@ -3,15 +3,15 @@ using BenchmarkDotNet.Jobs; using Xunit; -namespace BenchmarkDotNet.Tests.dotTrace +namespace BenchmarkDotNet.Tests.dotTrace; + +public class DotTraceTests { - public class DotTraceTests + [Fact] + public void AllRuntimeMonikerAreKnown() { - [Fact] - public void AllRuntimeMonikerAreKnown() - { - foreach (RuntimeMoniker moniker in Enum.GetValues(typeof(RuntimeMoniker))) - DotTraceDiagnoser.IsSupported(moniker); // Just check that it doesn't throw exceptions - } + var diagnoser = new DotTraceDiagnoser(); + foreach (RuntimeMoniker moniker in Enum.GetValues(typeof(RuntimeMoniker))) + diagnoser.IsSupported(moniker); // Just check that it doesn't throw exceptions } } \ No newline at end of file diff --git a/tests/runCoreTests.cmd b/tests/runCoreTests.cmd index 93c5c0f258..137ea952d9 100644 --- a/tests/runCoreTests.cmd +++ b/tests/runCoreTests.cmd @@ -12,24 +12,24 @@ if NOT %ERRORLEVEL% == 0 ( ) echo ----------------------------- -echo Running Core 2.1 Unit tests +echo Running Core 8.0 Unit tests echo ----------------------------- -call dotnet test "BenchmarkDotNet.Tests\BenchmarkDotNet.Tests.csproj" --configuration Release --framework netcoreapp2.1 +call dotnet test "BenchmarkDotNet.Tests\BenchmarkDotNet.Tests.csproj" --configuration Release --framework net8.0 if NOT %ERRORLEVEL% == 0 ( - echo CORE 2.1 Unit tests has failed + echo Core 8.0 Unit tests has failed goto end ) echo ----------------------------- -echo Running Core 2.1 Integration tests +echo Running 8.0 Integration tests echo ----------------------------- -call dotnet test "BenchmarkDotNet.IntegrationTests\BenchmarkDotNet.IntegrationTests.csproj" --configuration Release --framework netcoreapp2.1 +call dotnet test "BenchmarkDotNet.IntegrationTests\BenchmarkDotNet.IntegrationTests.csproj" --configuration Release --framework net8.0 if NOT %ERRORLEVEL% == 0 ( - echo CORE 2.1 Integration tests has failed + echo Core 8.0 Integration tests has failed goto end ) diff --git a/tests/runCoreTests.sh b/tests/runCoreTests.sh index 4e3ad59e6a..1b5bb28fa3 100755 --- a/tests/runCoreTests.sh +++ b/tests/runCoreTests.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -dotnet test BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj --configuration Release --framework netcoreapp2.1 2>&1 | tee tests.log -dotnet test BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj --configuration Release --framework netcoreapp2.1 2>&1 | tee integration-tests.log +dotnet test BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj --configuration Release --framework net8.0 2>&1 | tee tests.log +dotnet test BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj --configuration Release --framework net8.0 2>&1 | tee integration-tests.log