From 19e907194e462edc648091ef3c2bb218d6831727 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Mon, 9 Jun 2025 14:52:15 +0200 Subject: [PATCH 01/15] Set next BenchmarkDotNet version: 0.15.2 --- README.md | 2 +- build/common.props | 2 +- build/versions.txt | 3 ++- .../.template.config/template.json | 2 +- .../.template.config/template.json | 2 +- .../.template.config/template.json | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6c6b2618c2..6dea3ab02b 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ It's no harder than writing unit tests! Under the hood, it performs a lot of [magic](#automation) that guarantees [reliable and precise](#reliability) results thanks to the [perfolizer](https://github.com/AndreyAkinshin/perfolizer) statistical engine. BenchmarkDotNet protects you from popular benchmarking mistakes and warns you if something is wrong with your benchmark design or obtained measurements. The results are presented in a [user-friendly](#friendliness) form that highlights all the important facts about your experiment. -BenchmarkDotNet is already adopted by [26000+ GitHub projects](https://github.com/dotnet/BenchmarkDotNet/network/dependents) including +BenchmarkDotNet is already adopted by [26300+ GitHub projects](https://github.com/dotnet/BenchmarkDotNet/network/dependents) including [.NET Runtime](https://github.com/dotnet/runtime), [.NET Compiler](https://github.com/dotnet/roslyn), [.NET Performance](https://github.com/dotnet/performance), diff --git a/build/common.props b/build/common.props index 0e3efd410f..ee40dbf5ef 100644 --- a/build/common.props +++ b/build/common.props @@ -42,7 +42,7 @@ - 0.15.1 + 0.15.2 diff --git a/build/versions.txt b/build/versions.txt index 12ba717a87..98f8be3bf4 100644 --- a/build/versions.txt +++ b/build/versions.txt @@ -59,4 +59,5 @@ 0.13.12 0.14.0 0.15.0 -0.15.1 \ No newline at end of file +0.15.1 +0.15.2 \ No newline at end of file diff --git a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json index a4aadd866a..4f2dcda09b 100644 --- a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json +++ b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.CSharp/.template.config/template.json @@ -139,7 +139,7 @@ "type": "parameter", "datatype": "string", "description": "Version of BenchmarkDotNet that will be referenced.", - "defaultValue": "0.15.1", + "defaultValue": "0.15.2", "replaces": "$(BenchmarkDotNetVersion)" } }, diff --git a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json index c3c053896c..cd6075564f 100644 --- a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json +++ b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.FSharp/.template.config/template.json @@ -139,7 +139,7 @@ "type": "parameter", "datatype": "string", "description": "Version of BenchmarkDotNet that will be referenced.", - "defaultValue": "0.15.1", + "defaultValue": "0.15.2", "replaces": "$(BenchmarkDotNetVersion)" } }, diff --git a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json index 77346be80a..c02efa671d 100644 --- a/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json +++ b/templates/templates/BenchmarkDotNet.BenchmarkProjectTemplate.VB/.template.config/template.json @@ -139,7 +139,7 @@ "type": "parameter", "datatype": "string", "description": "Version of BenchmarkDotNet that will be referenced.", - "defaultValue": "0.15.1", + "defaultValue": "0.15.2", "replaces": "$(BenchmarkDotNetVersion)" } }, From 200260136816fa7f3f27db5710a5b805be13cb35 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Mon, 9 Jun 2025 19:05:23 +0200 Subject: [PATCH 02/15] [build] Enhance docs-fetch command --- .github/workflows/generate-changelog.yaml | 4 +-- .../Options/KnownOptions.cs | 6 +++++ build/BenchmarkDotNet.Build/Program.cs | 3 ++- .../Runners/Changelog/ChangelogBuilder.cs | 25 ++++++++++++++++--- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/.github/workflows/generate-changelog.yaml b/.github/workflows/generate-changelog.yaml index 57afef9f85..a0538e4eac 100644 --- a/.github/workflows/generate-changelog.yaml +++ b/.github/workflows/generate-changelog.yaml @@ -20,7 +20,7 @@ jobs: ref: master - name: Fetch changelog - run: ./build.cmd docs-fetch --depth 1 --preview + run: ./build.cmd docs-fetch --depth 1 --preview --force-clone env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -32,4 +32,4 @@ jobs: FOLDER: docs/_changelog GIT_CONFIG_NAME: Andrey Akinshin GIT_CONFIG_EMAIL: andrey.akinshin@gmail.com - CLEAN: true \ No newline at end of file + CLEAN: true diff --git a/build/BenchmarkDotNet.Build/Options/KnownOptions.cs b/build/BenchmarkDotNet.Build/Options/KnownOptions.cs index c5101e1d52..6da4e390ed 100644 --- a/build/BenchmarkDotNet.Build/Options/KnownOptions.cs +++ b/build/BenchmarkDotNet.Build/Options/KnownOptions.cs @@ -29,6 +29,12 @@ public static class KnownOptions Aliases = new[] { "-d" } }; + public static readonly BoolOption ForceClone = new("--force-clone") + { + Description = "Forces re-cloning of the changelog repository, deleting any existing directory.", + Aliases = new[] { "-f" } + }; + public static readonly BoolOption Help = new("--help") { Description = "Prints help information", diff --git a/build/BenchmarkDotNet.Build/Program.cs b/build/BenchmarkDotNet.Build/Program.cs index 02ccb6a789..5fd25cc298 100644 --- a/build/BenchmarkDotNet.Build/Program.cs +++ b/build/BenchmarkDotNet.Build/Program.cs @@ -172,13 +172,14 @@ public HelpInfo GetHelp() $"* Last changelog footer (if {KnownOptions.Stable.CommandLineName} is specified)\n" + $"* All changelog details in docs/_changelog\n" + $" (This dir is a cloned version of this repo from branch {Repo.ChangelogBranch})", - Options = [KnownOptions.DocsPreview, KnownOptions.DocsDepth], + Options = [KnownOptions.DocsPreview, KnownOptions.DocsDepth, KnownOptions.ForceClone], EnvironmentVariables = [EnvVar.GitHubToken], Examples = [ new Example(Name) .WithArgument(KnownOptions.DocsDepth, "3") .WithArgument(KnownOptions.DocsPreview) + .WithArgument(KnownOptions.ForceClone) ] }; } diff --git a/build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogBuilder.cs b/build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogBuilder.cs index 1fc8652358..5bfc8c91b2 100644 --- a/build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogBuilder.cs +++ b/build/BenchmarkDotNet.Build/Runners/Changelog/ChangelogBuilder.cs @@ -17,6 +17,7 @@ public class ChangelogBuilder private readonly BuildContext context; private readonly bool preview; private readonly string depth; + private readonly bool forceClone; /// /// Directory with original changelog part files from branch 'docs-changelog' @@ -33,6 +34,7 @@ public ChangelogBuilder(BuildContext context) this.context = context; preview = KnownOptions.DocsPreview.Resolve(context); depth = KnownOptions.DocsDepth.Resolve(context); + forceClone = KnownOptions.ForceClone.Resolve(context); var docsDirectory = context.RootDirectory.Combine("docs"); SrcDirectory = docsDirectory.Combine("_changelog"); @@ -43,7 +45,7 @@ public void Fetch() { EnvVar.GitHubToken.AssertHasValue(); - EnsureSrcDirectoryExist(); + EnsureSrcDirectoryExist(forceClone); var history = context.VersionHistory; var stableVersionCount = history.StableVersions.Length; @@ -224,14 +226,31 @@ private void GenerateToc() context.GenerateFile(DocfxDirectory.CombineWithFilePath("toc.yml"), content); } - private void EnsureSrcDirectoryExist(bool forceClean = false) + private void EnsureSrcDirectoryExist(bool forceClone = false) { - if (context.DirectoryExists(SrcDirectory) && forceClean) + void Log(string message) => context.Information($"[Changelog] {message}"); + + Log($"Preparing git sub-repository for changelog branch '{Repo.ChangelogBranch}'. " + + $"Target directory: '{SrcDirectory}'."); + if (context.DirectoryExists(SrcDirectory) && forceClone) + { + Log($"Directory '{SrcDirectory}' already exists and forceClean is specified. " + + $"Deleting the current directory..."); context.DeleteDirectory( SrcDirectory, new DeleteDirectorySettings { Force = true, Recursive = true }); + Log($"Directory '{SrcDirectory}' deleted successfully."); + } if (!context.DirectoryExists(SrcDirectory)) + { + Log($"Cloning branch '{Repo.ChangelogBranch}' from '{Repo.HttpsGitUrl}' to '{SrcDirectory}'."); context.GitRunner.Clone(SrcDirectory, Repo.HttpsGitUrl, Repo.ChangelogBranch); + Log($"Clone completed: '{Repo.ChangelogBranch}' -> '{SrcDirectory}'."); + } + else + { + Log($"Directory '{SrcDirectory}' already exists. Skipping clone."); + } } } \ No newline at end of file From 3df0f1bf4ede18ccf5744a550832f4ee2d651c5e Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Mon, 9 Jun 2025 19:05:55 +0200 Subject: [PATCH 03/15] Remove docs/_changelog folder from the main branch These files were extracted to docs-changelog branch --- docs/_changelog/footer/v0.10.0.md | 11 - docs/_changelog/footer/v0.10.1.md | 9 - docs/_changelog/footer/v0.10.10.md | 10 - docs/_changelog/footer/v0.10.11.md | 10 - docs/_changelog/footer/v0.10.12.md | 10 - docs/_changelog/footer/v0.10.13.md | 10 - docs/_changelog/footer/v0.10.14.md | 10 - docs/_changelog/footer/v0.10.2.md | 9 - docs/_changelog/footer/v0.10.3.md | 9 - docs/_changelog/footer/v0.10.4.md | 9 - docs/_changelog/footer/v0.10.5.md | 9 - docs/_changelog/footer/v0.10.6.md | 9 - docs/_changelog/footer/v0.10.7.md | 11 - docs/_changelog/footer/v0.10.8.md | 9 - docs/_changelog/footer/v0.10.9.md | 9 - docs/_changelog/footer/v0.11.0.md | 8 - docs/_changelog/footer/v0.11.1.md | 8 - docs/_changelog/footer/v0.11.2.md | 8 - docs/_changelog/footer/v0.11.3.md | 8 - docs/_changelog/footer/v0.11.4.md | 9 - docs/_changelog/footer/v0.11.5.md | 10 - docs/_changelog/footer/v0.12.0.md | 11 - docs/_changelog/footer/v0.12.1.md | 11 - docs/_changelog/footer/v0.13.0.md | 10 - docs/_changelog/footer/v0.13.1.md | 10 - docs/_changelog/footer/v0.13.10.md | 11 - docs/_changelog/footer/v0.13.11.md | 11 - docs/_changelog/footer/v0.13.12.md | 12 - docs/_changelog/footer/v0.13.2.md | 10 - docs/_changelog/footer/v0.13.3.md | 10 - docs/_changelog/footer/v0.13.4.md | 10 - docs/_changelog/footer/v0.13.5.md | 10 - docs/_changelog/footer/v0.13.6.md | 11 - docs/_changelog/footer/v0.13.7.md | 11 - docs/_changelog/footer/v0.13.8.md | 11 - docs/_changelog/footer/v0.13.9.md | 11 - docs/_changelog/footer/v0.14.0.md | 14 - docs/_changelog/footer/v0.15.0.md | 14 - docs/_changelog/footer/v0.8.2.md | 4 - docs/_changelog/footer/v0.9.0.md | 4 - docs/_changelog/footer/v0.9.1.md | 4 - docs/_changelog/footer/v0.9.2.md | 6 - docs/_changelog/footer/v0.9.3.md | 7 - docs/_changelog/footer/v0.9.4.md | 7 - docs/_changelog/footer/v0.9.5.md | 7 - docs/_changelog/footer/v0.9.6.md | 8 - docs/_changelog/footer/v0.9.7.md | 8 - docs/_changelog/footer/v0.9.8.md | 7 - docs/_changelog/footer/v0.9.9.md | 11 - docs/_changelog/header/v0.10.0.md | 12 - docs/_changelog/header/v0.10.1.md | 8 - docs/_changelog/header/v0.10.10.md | 10 - docs/_changelog/header/v0.10.11.md | 8 - docs/_changelog/header/v0.10.12.md | 22 - docs/_changelog/header/v0.10.13.md | 1 - docs/_changelog/header/v0.10.14.md | 4 - docs/_changelog/header/v0.10.2.md | 9 - docs/_changelog/header/v0.10.3.md | 8 - docs/_changelog/header/v0.10.4.md | 23 - docs/_changelog/header/v0.10.5.md | 5 - docs/_changelog/header/v0.10.6.md | 6 - docs/_changelog/header/v0.10.7.md | 6 - docs/_changelog/header/v0.10.8.md | 4 - docs/_changelog/header/v0.10.9.md | 14 - docs/_changelog/header/v0.11.0.md | 350 --------------- docs/_changelog/header/v0.11.1.md | 11 - docs/_changelog/header/v0.11.2.md | 657 ----------------------------- docs/_changelog/header/v0.11.3.md | 99 ----- docs/_changelog/header/v0.11.4.md | 114 ----- docs/_changelog/header/v0.11.5.md | 170 -------- docs/_changelog/header/v0.12.0.md | 469 -------------------- docs/_changelog/header/v0.12.1.md | 294 ------------- docs/_changelog/header/v0.13.0.md | 368 ---------------- docs/_changelog/header/v0.13.1.md | 15 - docs/_changelog/header/v0.13.10.md | 3 - docs/_changelog/header/v0.13.11.md | 3 - docs/_changelog/header/v0.13.12.md | 10 - docs/_changelog/header/v0.13.2.md | 254 ----------- docs/_changelog/header/v0.13.3.md | 91 ---- docs/_changelog/header/v0.13.4.md | 26 -- docs/_changelog/header/v0.13.5.md | 34 -- docs/_changelog/header/v0.13.6.md | 23 - docs/_changelog/header/v0.13.7.md | 3 - docs/_changelog/header/v0.13.8.md | 3 - docs/_changelog/header/v0.13.9.md | 3 - docs/_changelog/header/v0.14.0.md | 15 - docs/_changelog/header/v0.15.0.md | 0 docs/_changelog/header/v0.8.2.md | 19 - docs/_changelog/header/v0.9.0.md | 2 - docs/_changelog/header/v0.9.2.md | 1 - docs/_changelog/header/v0.9.3.md | 1 - docs/_changelog/header/v0.9.4.md | 9 - docs/_changelog/header/v0.9.5.md | 11 - docs/_changelog/header/v0.9.6.md | 3 - docs/_changelog/header/v0.9.7.md | 2 - docs/_changelog/header/v0.9.8.md | 8 - docs/_changelog/header/v0.9.9.md | 9 - 97 files changed, 3676 deletions(-) delete mode 100644 docs/_changelog/footer/v0.10.0.md delete mode 100644 docs/_changelog/footer/v0.10.1.md delete mode 100644 docs/_changelog/footer/v0.10.10.md delete mode 100644 docs/_changelog/footer/v0.10.11.md delete mode 100644 docs/_changelog/footer/v0.10.12.md delete mode 100644 docs/_changelog/footer/v0.10.13.md delete mode 100644 docs/_changelog/footer/v0.10.14.md delete mode 100644 docs/_changelog/footer/v0.10.2.md delete mode 100644 docs/_changelog/footer/v0.10.3.md delete mode 100644 docs/_changelog/footer/v0.10.4.md delete mode 100644 docs/_changelog/footer/v0.10.5.md delete mode 100644 docs/_changelog/footer/v0.10.6.md delete mode 100644 docs/_changelog/footer/v0.10.7.md delete mode 100644 docs/_changelog/footer/v0.10.8.md delete mode 100644 docs/_changelog/footer/v0.10.9.md delete mode 100644 docs/_changelog/footer/v0.11.0.md delete mode 100644 docs/_changelog/footer/v0.11.1.md delete mode 100644 docs/_changelog/footer/v0.11.2.md delete mode 100644 docs/_changelog/footer/v0.11.3.md delete mode 100644 docs/_changelog/footer/v0.11.4.md delete mode 100644 docs/_changelog/footer/v0.11.5.md delete mode 100644 docs/_changelog/footer/v0.12.0.md delete mode 100644 docs/_changelog/footer/v0.12.1.md delete mode 100644 docs/_changelog/footer/v0.13.0.md delete mode 100644 docs/_changelog/footer/v0.13.1.md delete mode 100644 docs/_changelog/footer/v0.13.10.md delete mode 100644 docs/_changelog/footer/v0.13.11.md delete mode 100644 docs/_changelog/footer/v0.13.12.md delete mode 100644 docs/_changelog/footer/v0.13.2.md delete mode 100644 docs/_changelog/footer/v0.13.3.md delete mode 100644 docs/_changelog/footer/v0.13.4.md delete mode 100644 docs/_changelog/footer/v0.13.5.md delete mode 100644 docs/_changelog/footer/v0.13.6.md delete mode 100644 docs/_changelog/footer/v0.13.7.md delete mode 100644 docs/_changelog/footer/v0.13.8.md delete mode 100644 docs/_changelog/footer/v0.13.9.md delete mode 100644 docs/_changelog/footer/v0.14.0.md delete mode 100644 docs/_changelog/footer/v0.15.0.md delete mode 100644 docs/_changelog/footer/v0.8.2.md delete mode 100644 docs/_changelog/footer/v0.9.0.md delete mode 100644 docs/_changelog/footer/v0.9.1.md delete mode 100644 docs/_changelog/footer/v0.9.2.md delete mode 100644 docs/_changelog/footer/v0.9.3.md delete mode 100644 docs/_changelog/footer/v0.9.4.md delete mode 100644 docs/_changelog/footer/v0.9.5.md delete mode 100644 docs/_changelog/footer/v0.9.6.md delete mode 100644 docs/_changelog/footer/v0.9.7.md delete mode 100644 docs/_changelog/footer/v0.9.8.md delete mode 100644 docs/_changelog/footer/v0.9.9.md delete mode 100644 docs/_changelog/header/v0.10.0.md delete mode 100644 docs/_changelog/header/v0.10.1.md delete mode 100644 docs/_changelog/header/v0.10.10.md delete mode 100644 docs/_changelog/header/v0.10.11.md delete mode 100644 docs/_changelog/header/v0.10.12.md delete mode 100644 docs/_changelog/header/v0.10.13.md delete mode 100644 docs/_changelog/header/v0.10.14.md delete mode 100644 docs/_changelog/header/v0.10.2.md delete mode 100644 docs/_changelog/header/v0.10.3.md delete mode 100644 docs/_changelog/header/v0.10.4.md delete mode 100644 docs/_changelog/header/v0.10.5.md delete mode 100644 docs/_changelog/header/v0.10.6.md delete mode 100644 docs/_changelog/header/v0.10.7.md delete mode 100644 docs/_changelog/header/v0.10.8.md delete mode 100644 docs/_changelog/header/v0.10.9.md delete mode 100644 docs/_changelog/header/v0.11.0.md delete mode 100644 docs/_changelog/header/v0.11.1.md delete mode 100644 docs/_changelog/header/v0.11.2.md delete mode 100644 docs/_changelog/header/v0.11.3.md delete mode 100644 docs/_changelog/header/v0.11.4.md delete mode 100644 docs/_changelog/header/v0.11.5.md delete mode 100644 docs/_changelog/header/v0.12.0.md delete mode 100644 docs/_changelog/header/v0.12.1.md delete mode 100644 docs/_changelog/header/v0.13.0.md delete mode 100644 docs/_changelog/header/v0.13.1.md delete mode 100644 docs/_changelog/header/v0.13.10.md delete mode 100644 docs/_changelog/header/v0.13.11.md delete mode 100644 docs/_changelog/header/v0.13.12.md delete mode 100644 docs/_changelog/header/v0.13.2.md delete mode 100644 docs/_changelog/header/v0.13.3.md delete mode 100644 docs/_changelog/header/v0.13.4.md delete mode 100644 docs/_changelog/header/v0.13.5.md delete mode 100644 docs/_changelog/header/v0.13.6.md delete mode 100644 docs/_changelog/header/v0.13.7.md delete mode 100644 docs/_changelog/header/v0.13.8.md delete mode 100644 docs/_changelog/header/v0.13.9.md delete mode 100644 docs/_changelog/header/v0.14.0.md delete mode 100644 docs/_changelog/header/v0.15.0.md delete mode 100644 docs/_changelog/header/v0.8.2.md delete mode 100644 docs/_changelog/header/v0.9.0.md delete mode 100644 docs/_changelog/header/v0.9.2.md delete mode 100644 docs/_changelog/header/v0.9.3.md delete mode 100644 docs/_changelog/header/v0.9.4.md delete mode 100644 docs/_changelog/header/v0.9.5.md delete mode 100644 docs/_changelog/header/v0.9.6.md delete mode 100644 docs/_changelog/header/v0.9.7.md delete mode 100644 docs/_changelog/header/v0.9.8.md delete mode 100644 docs/_changelog/header/v0.9.9.md diff --git a/docs/_changelog/footer/v0.10.0.md b/docs/_changelog/footer/v0.10.0.md deleted file mode 100644 index 823ad6954c..0000000000 --- a/docs/_changelog/footer/v0.10.0.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: November 10, 2016_ - -_Milestone: [v0.10.0](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.0)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.0 - -_Online Documentation:_ https://dotnet.github.io/BenchmarkDotNet/ \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.1.md b/docs/_changelog/footer/v0.10.1.md deleted file mode 100644 index 6a2f16d19d..0000000000 --- a/docs/_changelog/footer/v0.10.1.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: December 04, 2016_ - -_Milestone: [v0.10.1](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.1)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.10.md b/docs/_changelog/footer/v0.10.10.md deleted file mode 100644 index 9dde0a1cc3..0000000000 --- a/docs/_changelog/footer/v0.10.10.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: November 03, 2017_ - -_Milestone: [v0.10.10](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.10)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.9...v0.10.10)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.10 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.11.md b/docs/_changelog/footer/v0.10.11.md deleted file mode 100644 index 0de3715f93..0000000000 --- a/docs/_changelog/footer/v0.10.11.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: December 01, 2017_ - -_Milestone: [v0.10.11](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.11)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.10...v0.10.11)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.11 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.12.md b/docs/_changelog/footer/v0.10.12.md deleted file mode 100644 index 3106385452..0000000000 --- a/docs/_changelog/footer/v0.10.12.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: January 15, 2018_ - -_Milestone: [v0.10.12](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.12)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.11...v0.10.12)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.12 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.13.md b/docs/_changelog/footer/v0.10.13.md deleted file mode 100644 index 6eb70e1fc2..0000000000 --- a/docs/_changelog/footer/v0.10.13.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: March 02, 2018_ - -_Milestone: [v0.10.13](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.13)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.12...v0.10.13)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.13 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.13 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.13 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.13 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.14.md b/docs/_changelog/footer/v0.10.14.md deleted file mode 100644 index 2fd3ea5542..0000000000 --- a/docs/_changelog/footer/v0.10.14.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: April 09, 2018_ - -_Milestone: [v0.10.14](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.14)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.13...v0.10.14)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.14 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.14 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.14 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.14 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.2.md b/docs/_changelog/footer/v0.10.2.md deleted file mode 100644 index 46e50a038f..0000000000 --- a/docs/_changelog/footer/v0.10.2.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: January 21, 2017_ - -_Milestone: [v0.10.2](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.2)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.3.md b/docs/_changelog/footer/v0.10.3.md deleted file mode 100644 index 623439e439..0000000000 --- a/docs/_changelog/footer/v0.10.3.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: March 01, 2017_ - -_Milestone: [v0.10.3](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.3)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.3 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.4.md b/docs/_changelog/footer/v0.10.4.md deleted file mode 100644 index 9f497b878b..0000000000 --- a/docs/_changelog/footer/v0.10.4.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: April 21, 2017_ - -_Milestone: [v0.10.4](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.4)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.4 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.5.md b/docs/_changelog/footer/v0.10.5.md deleted file mode 100644 index b4d8d5b914..0000000000 --- a/docs/_changelog/footer/v0.10.5.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: April 26, 2017_ - -_Milestone: [v0.10.5](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.5)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.5 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.6.md b/docs/_changelog/footer/v0.10.6.md deleted file mode 100644 index b29dfc3939..0000000000 --- a/docs/_changelog/footer/v0.10.6.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: May 12, 2017_ - -_Milestone: [v0.10.6](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.6)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.6 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.7.md b/docs/_changelog/footer/v0.10.7.md deleted file mode 100644 index 5ee21fee21..0000000000 --- a/docs/_changelog/footer/v0.10.7.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: June 05, 2017_ - -_Milestone: [v0.10.7](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.7)_ - -_Overview post: https://aakinshin.net/posts/bdn-v0_10_7/_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.7 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.8.md b/docs/_changelog/footer/v0.10.8.md deleted file mode 100644 index 791022d425..0000000000 --- a/docs/_changelog/footer/v0.10.8.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: June 09, 2017_ - -_Milestone: [v0.10.8](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.8)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.8 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.10.9.md b/docs/_changelog/footer/v0.10.9.md deleted file mode 100644 index 8fc3fef03e..0000000000 --- a/docs/_changelog/footer/v0.10.9.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: July 28, 2017_ - -_Milestone: [v0.10.9](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.10.9)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.10.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.10.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.10.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.10.9 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.0.md b/docs/_changelog/footer/v0.11.0.md deleted file mode 100644 index f9637a79e6..0000000000 --- a/docs/_changelog/footer/v0.11.0.md +++ /dev/null @@ -1,8 +0,0 @@ -_Date: July 23, 2018_ - -_Milestone: [v0.11.0](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.0)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.10.14...v0.11.0)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.0 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.1.md b/docs/_changelog/footer/v0.11.1.md deleted file mode 100644 index f6c73f9510..0000000000 --- a/docs/_changelog/footer/v0.11.1.md +++ /dev/null @@ -1,8 +0,0 @@ -_Date: August 22, 2018_ - -_Milestone: [v0.11.1](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.1)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.0...v0.11.1)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.2.md b/docs/_changelog/footer/v0.11.2.md deleted file mode 100644 index 338170b8ae..0000000000 --- a/docs/_changelog/footer/v0.11.2.md +++ /dev/null @@ -1,8 +0,0 @@ -_Date: November 1, 2018_ - -_Milestone: [v0.11.2](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.2)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.1...v0.11.2)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.3.md b/docs/_changelog/footer/v0.11.3.md deleted file mode 100644 index befc032994..0000000000 --- a/docs/_changelog/footer/v0.11.3.md +++ /dev/null @@ -1,8 +0,0 @@ -_Date: November 20, 2018_ - -_Milestone: [v0.11.3](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.3)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.2...v0.11.3)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.3 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.4.md b/docs/_changelog/footer/v0.11.4.md deleted file mode 100644 index 5cbac89988..0000000000 --- a/docs/_changelog/footer/v0.11.4.md +++ /dev/null @@ -1,9 +0,0 @@ -_Date: February 15, 2019_ - -_Milestone: [v0.11.4](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.11.4)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.3...v0.11.4)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Tool/0.11.4 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.11.5.md b/docs/_changelog/footer/v0.11.5.md deleted file mode 100644 index be3282de78..0000000000 --- a/docs/_changelog/footer/v0.11.5.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: April 2, 2019_ - -_Milestone: [v0.11.5](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.11.5)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.4...v0.11.5)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.11.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.11.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Tool/0.11.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.11.5 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.12.0.md b/docs/_changelog/footer/v0.12.0.md deleted file mode 100644 index 565429ca6a..0000000000 --- a/docs/_changelog/footer/v0.12.0.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: October 24, 2019_ - -_Milestone: [v0.12.0](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.12.0)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.11.5...v0.12.0)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.12.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.12.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Tool/0.12.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.12.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.12.0 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.12.1.md b/docs/_changelog/footer/v0.12.1.md deleted file mode 100644 index af5fca9dc4..0000000000 --- a/docs/_changelog/footer/v0.12.1.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: April 6, 2020_ - -_Milestone: [v0.12.1](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.12.1)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.12.0...v0.12.1)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.12.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.12.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Tool/0.12.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.12.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.12.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.0.md b/docs/_changelog/footer/v0.13.0.md deleted file mode 100644 index 7a8ed327bc..0000000000 --- a/docs/_changelog/footer/v0.13.0.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: May 19, 2021_ - -_Milestone: [v0.13.0](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.0)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.12.1...v0.13.0)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.0 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.1.md b/docs/_changelog/footer/v0.13.1.md deleted file mode 100644 index eb71d8729f..0000000000 --- a/docs/_changelog/footer/v0.13.1.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: August 11, 2021_ - -_Milestone: [v0.13.1](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.1)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.0...v0.13.1)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.1 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.10.md b/docs/_changelog/footer/v0.13.10.md deleted file mode 100644 index 68ef4fd0db..0000000000 --- a/docs/_changelog/footer/v0.13.10.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: November 01, 2023_ - -_Milestone: [v0.13.10](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.10)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.9...v0.13.10)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.10 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.10 diff --git a/docs/_changelog/footer/v0.13.11.md b/docs/_changelog/footer/v0.13.11.md deleted file mode 100644 index 03dff00d30..0000000000 --- a/docs/_changelog/footer/v0.13.11.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: December 06, 2023_ - -_Milestone: [v0.13.11](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.11)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.10...v0.13.11)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.11 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.11 diff --git a/docs/_changelog/footer/v0.13.12.md b/docs/_changelog/footer/v0.13.12.md deleted file mode 100644 index 246864bb8a..0000000000 --- a/docs/_changelog/footer/v0.13.12.md +++ /dev/null @@ -1,12 +0,0 @@ -_Date: January 05, 2024_ - -_Milestone: [v0.13.12](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.12)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.11...v0.13.12)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.12 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.12 -* https://www.nuget.org/packages/BenchmarkDotNet.TestAdapter/0.13.12 diff --git a/docs/_changelog/footer/v0.13.2.md b/docs/_changelog/footer/v0.13.2.md deleted file mode 100644 index c9022e590e..0000000000 --- a/docs/_changelog/footer/v0.13.2.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: August 26, 2022_ - -_Milestone: [v0.13.2](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.2)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.1...v0.13.2)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.2 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.3.md b/docs/_changelog/footer/v0.13.3.md deleted file mode 100644 index f0fa6dff23..0000000000 --- a/docs/_changelog/footer/v0.13.3.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: December 26, 2022_ - -_Milestone: [v0.13.3](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.3)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.2...v0.13.3)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.3 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.3 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.4.md b/docs/_changelog/footer/v0.13.4.md deleted file mode 100644 index b6ac2ad6af..0000000000 --- a/docs/_changelog/footer/v0.13.4.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: January 13, 2023_ - -_Milestone: [v0.13.4](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.4)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.3...v0.13.4)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.4 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.4 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.5.md b/docs/_changelog/footer/v0.13.5.md deleted file mode 100644 index 577eeaea84..0000000000 --- a/docs/_changelog/footer/v0.13.5.md +++ /dev/null @@ -1,10 +0,0 @@ -_Date: February 17, 2023_ - -_Milestone: [v0.13.5](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.5)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.4...v0.13.5)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.5 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.5 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.13.6.md b/docs/_changelog/footer/v0.13.6.md deleted file mode 100644 index 7235cd2302..0000000000 --- a/docs/_changelog/footer/v0.13.6.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: July 11, 2023_ - -_Milestone: [v0.13.6](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.6)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.5...v0.13.6)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.6 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.6 diff --git a/docs/_changelog/footer/v0.13.7.md b/docs/_changelog/footer/v0.13.7.md deleted file mode 100644 index 1b256837d4..0000000000 --- a/docs/_changelog/footer/v0.13.7.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: August 04, 2023_ - -_Milestone: [v0.13.7](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.7)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.6...v0.13.7)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.7 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.7 diff --git a/docs/_changelog/footer/v0.13.8.md b/docs/_changelog/footer/v0.13.8.md deleted file mode 100644 index a8551cc47f..0000000000 --- a/docs/_changelog/footer/v0.13.8.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: September 08, 2023_ - -_Milestone: [v0.13.8](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.8)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.7...v0.13.8)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.8 diff --git a/docs/_changelog/footer/v0.13.9.md b/docs/_changelog/footer/v0.13.9.md deleted file mode 100644 index 962bc252cc..0000000000 --- a/docs/_changelog/footer/v0.13.9.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: October 05, 2023_ - -_Milestone: [v0.13.9](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.13.9)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.8...v0.13.9)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.13.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.13.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.13.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.13.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.13.9 diff --git a/docs/_changelog/footer/v0.14.0.md b/docs/_changelog/footer/v0.14.0.md deleted file mode 100644 index 47d421f49d..0000000000 --- a/docs/_changelog/footer/v0.14.0.md +++ /dev/null @@ -1,14 +0,0 @@ -_Date: August 06, 2024_ - -_Milestone: [v0.14.0](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.14.0)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.13.12...v0.14.0)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.14.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.14.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotMemory/0.14.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.14.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.14.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Exporters.Plotting/0.14.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.14.0 -* https://www.nuget.org/packages/BenchmarkDotNet.TestAdapter/0.14.0 diff --git a/docs/_changelog/footer/v0.15.0.md b/docs/_changelog/footer/v0.15.0.md deleted file mode 100644 index 7ad205fa59..0000000000 --- a/docs/_changelog/footer/v0.15.0.md +++ /dev/null @@ -1,14 +0,0 @@ -_Date: May 22, 2025_ - -_Milestone: [v0.15.0](https://github.com/dotnet/BenchmarkDotNet/issues?q=milestone%3Av0.15.0)_ -([List of commits](https://github.com/dotnet/BenchmarkDotNet/compare/v0.14.0...v0.15.0)) - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.15.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Annotations/0.15.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotMemory/0.15.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace/0.15.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.15.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Exporters.Plotting/0.15.0 -* https://www.nuget.org/packages/BenchmarkDotNet.Templates/0.15.0 -* https://www.nuget.org/packages/BenchmarkDotNet.TestAdapter/0.15.0 diff --git a/docs/_changelog/footer/v0.8.2.md b/docs/_changelog/footer/v0.8.2.md deleted file mode 100644 index cbd405d3ae..0000000000 --- a/docs/_changelog/footer/v0.8.2.md +++ /dev/null @@ -1,4 +0,0 @@ -_Date: January 19, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.8.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.0.md b/docs/_changelog/footer/v0.9.0.md deleted file mode 100644 index 55346f6951..0000000000 --- a/docs/_changelog/footer/v0.9.0.md +++ /dev/null @@ -1,4 +0,0 @@ -_Date: February 9, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.0 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.1.md b/docs/_changelog/footer/v0.9.1.md deleted file mode 100644 index b38f8a6436..0000000000 --- a/docs/_changelog/footer/v0.9.1.md +++ /dev/null @@ -1,4 +0,0 @@ -_Date: February 10, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.1 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.2.md b/docs/_changelog/footer/v0.9.2.md deleted file mode 100644 index 73c081a4cd..0000000000 --- a/docs/_changelog/footer/v0.9.2.md +++ /dev/null @@ -1,6 +0,0 @@ -_Milestone: [v0.9.2](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.2)_ - -_Date: March 5, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.2 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.3.md b/docs/_changelog/footer/v0.9.3.md deleted file mode 100644 index da9b45b165..0000000000 --- a/docs/_changelog/footer/v0.9.3.md +++ /dev/null @@ -1,7 +0,0 @@ -_Milestone: [v0.9.3](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.3)_ - -_Date: March 13, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.3 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.3-beta \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.4.md b/docs/_changelog/footer/v0.9.4.md deleted file mode 100644 index 0927d0a62b..0000000000 --- a/docs/_changelog/footer/v0.9.4.md +++ /dev/null @@ -1,7 +0,0 @@ -_Milestone: [v0.9.4](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.4)_ - -_Date: March 24, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.4 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.4-beta \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.5.md b/docs/_changelog/footer/v0.9.5.md deleted file mode 100644 index ea7d920994..0000000000 --- a/docs/_changelog/footer/v0.9.5.md +++ /dev/null @@ -1,7 +0,0 @@ -_Milestone: [v0.9.5](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.5)_ - -_Date: May 02, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.5 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.5-beta \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.6.md b/docs/_changelog/footer/v0.9.6.md deleted file mode 100644 index cc71d2d927..0000000000 --- a/docs/_changelog/footer/v0.9.6.md +++ /dev/null @@ -1,8 +0,0 @@ -_Milestone: [v0.9.6](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.6)_ - -_Date: May 11, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.6 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.6-beta -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.9.6 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.7.md b/docs/_changelog/footer/v0.9.7.md deleted file mode 100644 index 21f1d109e5..0000000000 --- a/docs/_changelog/footer/v0.9.7.md +++ /dev/null @@ -1,8 +0,0 @@ -_Milestone: [v0.9.7](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.7)_ - -_Date: May 29, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.7 -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.7-beta -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.9.7 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.8.md b/docs/_changelog/footer/v0.9.8.md deleted file mode 100644 index fa5bc33931..0000000000 --- a/docs/_changelog/footer/v0.9.8.md +++ /dev/null @@ -1,7 +0,0 @@ -_Milestone: [v0.9.8](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.8)_ - -_Date: July 07, 2016_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.8 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.9.8 \ No newline at end of file diff --git a/docs/_changelog/footer/v0.9.9.md b/docs/_changelog/footer/v0.9.9.md deleted file mode 100644 index 2e6b340c90..0000000000 --- a/docs/_changelog/footer/v0.9.9.md +++ /dev/null @@ -1,11 +0,0 @@ -_Date: August 18, 2016_ - -_Milestone: [v0.9.9](https://github.com/PerfDotNet/BenchmarkDotNet/issues?q=milestone%3Av0.9.9)_ - -_NuGet Packages:_ -* https://www.nuget.org/packages/BenchmarkDotNet/0.9.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Core/0.9.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Toolchains.Roslyn/0.9.9 -* https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.Windows/0.9.9 - -_Online Documentation:_ https://perfdotnet.github.io/BenchmarkDotNet/ \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.0.md b/docs/_changelog/header/v0.10.0.md deleted file mode 100644 index 48c4103b0a..0000000000 --- a/docs/_changelog/header/v0.10.0.md +++ /dev/null @@ -1,12 +0,0 @@ -* Now BenchmarkDotNet is a part of .NET Foundation -* Job and Column API refactoring (see new documentation) -* Measurement engine improvements -* Horology enhancement (see `TimeInterval` and `Frequency`) -* Introduced `RankColumn` which is based on `WelchTTest` (see [157aabc3](https://github.com/PerfDotNet/BenchmarkDotNet/commit/cf839a0d7ecfdf93da709b63fe324fd2157aabc3)) -* JsonExporters refactoring (see the Exporters/Json section in the documentation) - * Renamed JsonExporters classed and attributes - * JsonExporters with custom settings - * JsonExporters now includes information about the target type namespace (see [#246](https://github.com/PerfDotNet/BenchmarkDotNet/issues/246)). -* Add `JetBrains.Annotations` (see [#253](https://github.com/PerfDotNet/BenchmarkDotNet/pull/253)) -* RFC 4180 support in CSV exporters (see [#241](https://github.com/PerfDotNet/BenchmarkDotNet/issues/241)) -* Many bugfixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.1.md b/docs/_changelog/header/v0.10.1.md deleted file mode 100644 index 70e3be82c8..0000000000 --- a/docs/_changelog/header/v0.10.1.md +++ /dev/null @@ -1,8 +0,0 @@ -* MemoryDiagnoser got improved. The changes: - * Memory Diagnoser is now part of BenchmarkDotNet.Core.dll, and it's **enabled by default** - * MemoryDiagnoser is **100% accurate** about allocated memory when using default settings or Job.ShortRun or any longer job. (see [#284](https://github.com/dotnet/BenchmarkDotNet/pull/284)) - * Memory Diagnoser no longer includes allocations from Cleanup/Setup methods (see [#186](https://github.com/dotnet/BenchmarkDotNet/issues/186)) - * the results are now scaled so they are stable across the runs. (see [#133](https://github.com/dotnet/BenchmarkDotNet/issues/133)) -* .NET Core 1.1+ support, we no longer support 1.0, we target netcoreapp1.1 now. Reason: we wanted to use `GC.GetAllocatedBytesForCurrentThread` in MemoryDiagnoser which is available only in 1.1+ -* Improved information about environment in summary -* Minor bugfixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.10.md b/docs/_changelog/header/v0.10.10.md deleted file mode 100644 index ef34841958..0000000000 --- a/docs/_changelog/header/v0.10.10.md +++ /dev/null @@ -1,10 +0,0 @@ -Highlights: - -* Disassembly Diagnoser (read more here: [Disassembling .NET Code with BenchmarkDotNet](https://adamsitnik.com/Disassembly-Diagnoser/)) -* ParamsSources -* .NET Core x86 support -* Environment variables and Mono args support -* Better environment description -* More: additional sections in the documentation, bug fixes, build script improvements, internal refactoring. - -Overview post: [BenchmarkDotNet v0.10.10](https://aakinshin.net/posts/bdn-v0_10_10/) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.11.md b/docs/_changelog/header/v0.10.11.md deleted file mode 100644 index 3e53cf54cb..0000000000 --- a/docs/_changelog/header/v0.10.11.md +++ /dev/null @@ -1,8 +0,0 @@ -## Highlights - -* ByRef and Stack-only support ([#492](https://github.com/dotnet/BenchmarkDotNet/issues/492), [sample](https://github.com/dotnet/BenchmarkDotNet/blob/edf20758871ec621fdbfd93d862769da46c4bf15/samples/BenchmarkDotNet.Samples/IL/IL_RefReturns.cs)) -* .NET Core 2.1 support ([#587](https://github.com/dotnet/BenchmarkDotNet/issues/587)) -* Improved LINQPad support -* Smart logic for precision in ScaledColumn ([#509](https://github.com/dotnet/BenchmarkDotNet/issues/509), [#590](https://github.com/dotnet/BenchmarkDotNet/issues/590)) -* Better macOS version detection ([15d72388](https://github.com/dotnet/BenchmarkDotNet/commit/15d72388436c1060e87662b5f4519b9e7e071627)) -* Minor fixes and improvements \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.12.md b/docs/_changelog/header/v0.10.12.md deleted file mode 100644 index 43cff33827..0000000000 --- a/docs/_changelog/header/v0.10.12.md +++ /dev/null @@ -1,22 +0,0 @@ -Overview post: [BenchmarkDotNet v0.10.12](https://aakinshin.net/posts/bdn-v0_10_12/) - -### Highlights - -* **Improved DisassemblyDiagnoser:** - BenchmarkDotNet contains an embedded disassembler so that it can print assembly code for all benchmarks; - it's not easy, but the disassembler evolves in every release. -* **Improved MemoryDiagnoser:** - it has a better precision level, and it takes less time to evaluate memory allocations in a benchmark. -* **New TailCallDiagnoser:** - now you get notifications when JIT applies the tail call optimizations to your methods. -* **Better environment info:** - when your share performance results, it's very important to share information about your environment. - The library generates the environment summary for you by default. - Now it contains information about the amount of physical CPU, physical cores, and logic cores. - If you run a benchmark on a virtual machine, you will get the name of the hypervisor - (e.g., Hyper-V, VMware, or VirtualBox). -* **Better summary table:** - one of the greatest features of BenchmarkDotNet is the summary table. - It shows all important information about results in a compact and understandable form. - Now it has better customization options: you can display relative performance of different environments - (e.g., compare .NET Framework and .NET Core) and group benchmarks by categories. \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.13.md b/docs/_changelog/header/v0.10.13.md deleted file mode 100644 index 73f5aa1e53..0000000000 --- a/docs/_changelog/header/v0.10.13.md +++ /dev/null @@ -1 +0,0 @@ -Overview post: [BenchmarkDotNet v0.10.13](https://aakinshin.net/posts/bdn-v0_10_13/) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.14.md b/docs/_changelog/header/v0.10.14.md deleted file mode 100644 index aefb183459..0000000000 --- a/docs/_changelog/header/v0.10.14.md +++ /dev/null @@ -1,4 +0,0 @@ -* Per-method parameterization ([Read more](https://benchmarkdotnet.org/articles/features/parameterization.html)) -* Console histograms and multimodal disribution detection -* Many improvements for Mono disassembly support on Windows ([Read more](https://aakinshin.net/posts/dotnet-crossruntime-disasm/)) -* Many bugfixes diff --git a/docs/_changelog/header/v0.10.2.md b/docs/_changelog/header/v0.10.2.md deleted file mode 100644 index a93c8c36ae..0000000000 --- a/docs/_changelog/header/v0.10.2.md +++ /dev/null @@ -1,9 +0,0 @@ -* Closed [#307](https://github.com/dotnet/BenchmarkDotNet/issues/307): culture invariant statistics output -* Closed [#321](https://github.com/dotnet/BenchmarkDotNet/issues/321): persist optimized, auto-generated dll compiled from url/plain code -* Closed [#322](https://github.com/dotnet/BenchmarkDotNet/issues/332): always restore the console foreground color -* Closed [#337](https://github.com/dotnet/BenchmarkDotNet/issues/337): Better detection of Rscript.exe in RPlotExporter -* Closed [#345](https://github.com/dotnet/BenchmarkDotNet/issues/345): fix bug in WelchTTestPValueColumn for DryJob -* VS 2017 [compatibility fix](https://github.com/dotnet/BenchmarkDotNet/commit/f4bdae5b7e203e0f0a7d283db5faa78107674f31) -* [fix](https://github.com/dotnet/BenchmarkDotNet/commit/24dea483b8312efba669d82a6fac3603e60050f5) bold markup for Atlassian exporter -* Improved precision of nanobenchmarks -* Minor infrastructure changes and misc fixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.3.md b/docs/_changelog/header/v0.10.3.md deleted file mode 100644 index 976a78eeaf..0000000000 --- a/docs/_changelog/header/v0.10.3.md +++ /dev/null @@ -1,8 +0,0 @@ -* **New .csprojs** support for .NET Core. Also for F# ([#366](https://github.com/dotnet/BenchmarkDotNet/issues/366))! -* New plots and RPlotExporter (density plots for each job; cumulative mean plots) -* Fixed exporter order (now RPlotExporer uses the actual measurements instead of previous version) -* Xplat improvments in RuntimeInformation -* Introduced `RunStrategy.Monitoring` -* Possibility to set custom path for Mono ([#306](https://github.com/dotnet/BenchmarkDotNet/issues/306)) -* Possibility to set any .NET Core version >= 1.1 ([#336](https://github.com/dotnet/BenchmarkDotNet/issues/336)) -* **MemoryDiagnoser is now disabled by default (Breaking changes!!)** ([#369](https://github.com/dotnet/BenchmarkDotNet/issues/369)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.4.md b/docs/_changelog/header/v0.10.4.md deleted file mode 100644 index f423a6567a..0000000000 --- a/docs/_changelog/header/v0.10.4.md +++ /dev/null @@ -1,23 +0,0 @@ -* New logo -* Update to Roslyn 2.0, drop .NET 4.5 support ([#303](https://github.com/dotnet/BenchmarkDotNet/issues/303)) -* Initial support of HardwareCounters (Windows only) -* Initial experimental support of in-process benchmarks -* Optional configs for `BenchmarkSwitcher` ([#391](https://github.com/dotnet/BenchmarkDotNet/issues/391), [#392](https://github.com/dotnet/BenchmarkDotNet/pull/392)) -* Host API interface ([#356](https://github.com/dotnet/BenchmarkDotNet/pull/356)) -* Improved measurements for async benchmarks ([#415](https://github.com/dotnet/BenchmarkDotNet/issues/415)) -* Improved precision level (MinIterationTimes is 500ms instead of 200ms; introduced `AccuracyMode.MaxAbsoluteError` and `AccuracyMode.MaxRelativeError` instead of `AccuracyMode.MaxStdErrRelative`; logic which select amount of iterations uses confidence intervals instead of standard errors; the Error column (half of CI99.9%) is shown by default instead of StdErr) -* Introduced `ISummaryStyle`, raw data in CSV reports ([#118](https://github.com/dotnet/BenchmarkDotNet/issues/118), [#146](https://github.com/dotnet/BenchmarkDotNet/issues/146), [#396](https://github.com/dotnet/BenchmarkDotNet/pull/396)) -* Handle cases when report files are existed and locked ([#414](https://github.com/dotnet/BenchmarkDotNet/issues/414), [#416](https://github.com/dotnet/BenchmarkDotNet/pull/416)) -* MarkdownExporter right-justifies numeric columns ([#421](https://github.com/dotnet/BenchmarkDotNet/pull/421)) -* Better colors for console output ([#376](https://github.com/dotnet/BenchmarkDotNet/issues/376)) -* Column legends -* Add information about CPU microarchitecture for well-known processors to summary -* Fix AssemblyInformationalVersionAttribute ([#382](https://github.com/dotnet/BenchmarkDotNet/issues/382)) -* Fix incorrect method filtering in BenchmarkSwitcher ([#365](https://github.com/dotnet/BenchmarkDotNet/issues/365)) -* Fix OS Version in Summary for Windows 10 ([#351](https://github.com/dotnet/BenchmarkDotNet/issues/351)) -* Fix OS Version on Mono -* Fix --class and --method filtering ([#249](https://github.com/dotnet/BenchmarkDotNet/issues/249)) -* Fix --exporters option ([#189](https://github.com/dotnet/BenchmarkDotNet/issues/189)) -* Fix escaping logic in CsvExporter ([#294](https://github.com/dotnet/BenchmarkDotNet/issues/294), [#409](https://github.com/dotnet/BenchmarkDotNet/pull/409)) -* [Fix](https://github.com/dotnet/BenchmarkDotNet/commit/0a251b81b826876179740cc8b79c994a73a5cd51) MacOS detection -* Minor bugfixes and API improvements \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.5.md b/docs/_changelog/header/v0.10.5.md deleted file mode 100644 index 3ed3541c70..0000000000 --- a/docs/_changelog/header/v0.10.5.md +++ /dev/null @@ -1,5 +0,0 @@ -* Fixed SizeUnit presentation in the summary table ([#434](https://github.com/dotnet/BenchmarkDotNet/issues/434)) -* In MemoryDiagnoser, now 1kB = 1024B (instead of 1000 in v0.10.4) ([#434](https://github.com/dotnet/BenchmarkDotNet/issues/434)) -* Fix false allocations detection ([#436](https://github.com/dotnet/BenchmarkDotNet/pull/436) [9b44de70](https://github.com/dotnet/BenchmarkDotNet/commit/9b44de704b96e2333d762b14daa152d859b1917d)) -* Hide ScaledSD column for small values ([da857ad7](https://github.com/dotnet/BenchmarkDotNet/commit/da857ad7eda77db813692d3c3678f8ad04f5af78)) -* Autoselecting amount of digits after the decimal point ([#404](https://github.com/dotnet/BenchmarkDotNet/issues/404)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.6.md b/docs/_changelog/header/v0.10.6.md deleted file mode 100644 index 33948b2f80..0000000000 --- a/docs/_changelog/header/v0.10.6.md +++ /dev/null @@ -1,6 +0,0 @@ -* Removed buggy allocation from Engine which was spoiling the results of MemoryDiagnoser for micro benchmarks. This part of the code is now guarded with very strict integration tests, it should never happen again. We now also exclude the side effects of the Allocation Quantum. **This bug was serious, you must update to `0.10.6`** ([#439](https://github.com/dotnet/BenchmarkDotNet/issues/439)) -* Support of the `PackageTargetFallback` setting which allows to reference components that target old framework monikers (like `dotnet5.4` or `portable-net45+win8`) ([#438](https://github.com/dotnet/BenchmarkDotNet/issues/438)) -* Added `InstructionRetiredPerCycleColumn` which shows up automatically when `HardwareCounter.InstructionRetired` and `HardwareCounter.TotalCycles` are used. -* Support benchmark classes without namespace ([#446](https://github.com/dotnet/BenchmarkDotNet/issues/446)) -* Fix problem with RPlotExporter and quoted directories in %PATH% ([#446](https://github.com/dotnet/BenchmarkDotNet/issues/446)) -* Show Windows brand version in summary \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.7.md b/docs/_changelog/header/v0.10.7.md deleted file mode 100644 index e58ad7f60e..0000000000 --- a/docs/_changelog/header/v0.10.7.md +++ /dev/null @@ -1,6 +0,0 @@ -* LINQPad support (5.22.05+) ([#66](https://github.com/dotnet/BenchmarkDotNet/issues/66), [#445](https://github.com/dotnet/BenchmarkDotNet/issues/445)) -* Benchmark filters and categories ([#248](https://github.com/dotnet/BenchmarkDotNet/issues/248)) -* Updated setup/cleanup attributes: `[GlobalSetup]`, `[GlobalCleanup]`, `[IterationSetup]`, `[IterationCleanup]` ([#270](https://github.com/dotnet/BenchmarkDotNet/issues/270), [#274](https://github.com/dotnet/BenchmarkDotNet/issues/274), [#325](https://github.com/dotnet/BenchmarkDotNet/issues/325), [#456](https://github.com/dotnet/BenchmarkDotNet/issues/456)) -* Better Value Types support ([afa803d0](https://github.com/dotnet/BenchmarkDotNet/commit/afa803d0e38c0e11864b2e4394d4a85d3801d944)) -* Building Sources on Linux: it's possible to build the solution (with unloaded F#/VB projects), run samples (for both net46/netcoreapp1.1), run unit tests (for netcoreapp1.1 only) -* Fix minor bugs in `JsonExporter` ([#451](https://github.com/dotnet/BenchmarkDotNet/pull/451)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.8.md b/docs/_changelog/header/v0.10.8.md deleted file mode 100644 index c93b24ee6e..0000000000 --- a/docs/_changelog/header/v0.10.8.md +++ /dev/null @@ -1,4 +0,0 @@ -* Legend for time units ([#349](https://github.com/dotnet/BenchmarkDotNet/issues/349), [#459](https://github.com/dotnet/BenchmarkDotNet/issues/459), [f14e508e](https://github.com/dotnet/BenchmarkDotNet/commit/f14e508e44b510a26cc3ec5aed30ee7843a92baf)) -* XML exporter ([#157](https://github.com/dotnet/BenchmarkDotNet/issues/157), [#452](https://github.com/dotnet/BenchmarkDotNet/pull/452), [a0148db8](https://github.com/dotnet/BenchmarkDotNet/commit/a0148db80c518a9d255f496534a8d1666be52c69)) -* .NET Framework 4.7 support ([#461](https://github.com/dotnet/BenchmarkDotNet/issues/461), [3f2b5c3c](https://github.com/dotnet/BenchmarkDotNet/commit/3f2b5c3c134c62f34f0ecf1a9c90d91ad37f2c6a), [5513873a](https://github.com/dotnet/BenchmarkDotNet/commit/5513873ac40d07583d6136e431e3b7c8cdf6c851)) -* Public API for AllocationQuantum ([#450](https://github.com/dotnet/BenchmarkDotNet/issues/450), [#462](https://github.com/dotnet/BenchmarkDotNet/pull/462), [a0148db8](https://github.com/dotnet/BenchmarkDotNet/commit/a0148db80c518a9d255f496534a8d1666be52c69)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.10.9.md b/docs/_changelog/header/v0.10.9.md deleted file mode 100644 index 248e6b007e..0000000000 --- a/docs/_changelog/header/v0.10.9.md +++ /dev/null @@ -1,14 +0,0 @@ -* Migrate from custom build scripts to Cake (C# Make) ([#426](https://github.com/dotnet/BenchmarkDotNet/issues/426), [#475](https://github.com/dotnet/BenchmarkDotNet/pull/475), thanks [@Ky7m](https://github.com/Ky7m)) -* Target Setup methods for specific Benchmarks ([#469](https://github.com/dotnet/BenchmarkDotNet/issues/469), [#501](https://github.com/dotnet/BenchmarkDotNet/pull/501), thanks [@ipjohnson](https://github.com/ipjohnson)) -* Many improvements in XmlExporter ([#476](https://github.com/dotnet/BenchmarkDotNet/pull/476), [#488](https://github.com/dotnet/BenchmarkDotNet/pull/488), thanks [@Teknikaali](https://github.com/Teknikaali)) -* Add MemoryDiagnoser results to JsonExporter output ([#453](https://github.com/dotnet/BenchmarkDotNet/issues/453), [#478](https://github.com/dotnet/BenchmarkDotNet/pull/478), thanks [@Teknikaali](https://github.com/Teknikaali)) -* Detect correct version of .NET Core (+ improved presentation for information about runtime) ([#448](https://github.com/dotnet/BenchmarkDotNet/issues/448), [ed586585...ed586585](https://github.com/dotnet/BenchmarkDotNet/compare/dc6dc411b4d8703d0a1abafe64fb1e0b0a83af1f...cea199f74923c99f88f4bb4d53e37f86b10269b7)) -* Fix UnauthorizedAccessException ([#380](https://github.com/dotnet/BenchmarkDotNet/issues/380), [#390](https://github.com/dotnet/BenchmarkDotNet/issues/390), [#490](https://github.com/dotnet/BenchmarkDotNet/issues/490), [#491](https://github.com/dotnet/BenchmarkDotNet/issues/491), [8505abb5](https://github.com/dotnet/BenchmarkDotNet/commit/8505abb5416bad90cda03f4972b067f9ac44b304)) -* Fix app.config generation ([#499](https://github.com/dotnet/BenchmarkDotNet/issues/499), [dc6dc411](https://github.com/dotnet/BenchmarkDotNet/commit/dc6dc411b4d8703d0a1abafe64fb1e0b0a83af1f)) -* Fix incorrect order of IterationCleanup and Benchmark jitting ([#481](https://github.com/dotnet/BenchmarkDotNet/issues/481), [#503](https://github.com/dotnet/BenchmarkDotNet/pull/503)) -* Fix test scripts for MacOS+zsh ([1177c8](https://github.com/dotnet/BenchmarkDotNet/commit/1177c80e2dbe931439e44bb0ce2ce25cad8b9ba2)) -* Unix-related ProcessorAffinity fixes ([#474](https://github.com/dotnet/BenchmarkDotNet/issues/474), [26d44411](https://github.com/dotnet/BenchmarkDotNet/commit/26d44411ea47f28a9cc7df84b2df0ef89b2bbcf7)) -* Minor fixes in docs ([#465](https://github.com/dotnet/BenchmarkDotNet/pull/465), [#467](https://github.com/dotnet/BenchmarkDotNet/pull/467), [#473](https://github.com/dotnet/BenchmarkDotNet/pull/473), [#480](https://github.com/dotnet/BenchmarkDotNet/pull/480), [#483](https://github.com/dotnet/BenchmarkDotNet/pull/483), thanks [@mtschneiders](https://github.com/mtschneiders), [@davkean](https://github.com/davkean), [@aarondandy](https://github.com/aarondandy), [@AmadeusW](https://github.com/AmadeusW)) -* Temporary hacks for [appveyor connectivity incident](https://appveyor.statuspage.io/incidents/m2vdvw39kdk8) ([#497](https://github.com/dotnet/BenchmarkDotNet/pull/497), [#506](https://github.com/dotnet/BenchmarkDotNet/pull/506)) -* Additional warnings for incorrect Configs ([#482](https://github.com/dotnet/BenchmarkDotNet/issues/482), [eb84825f](https://github.com/dotnet/BenchmarkDotNet/commit/eb84825ff08aa5d23d2d512d4d4bde3e95ca0815)) -* Additional warnings for F# methods with spaces ([#479](https://github.com/dotnet/BenchmarkDotNet/issues/479), [3c2c8dec](https://github.com/dotnet/BenchmarkDotNet/commit/3c2c8dec28d6c570f2901001058cd9c6000e6ca2), [7ba1c809](https://github.com/dotnet/BenchmarkDotNet/commit/7ba1c809004e0b75eaa87724155480eaf623f8a9), [3ca39afe](https://github.com/dotnet/BenchmarkDotNet/commit/3ca39afe9f0d25359f9b092181beb02d57c5ad32)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.11.0.md b/docs/_changelog/header/v0.11.0.md deleted file mode 100644 index 9760c809ba..0000000000 --- a/docs/_changelog/header/v0.11.0.md +++ /dev/null @@ -1,350 +0,0 @@ -This is one of the biggest releases of BenchmarkDotNet ever. -There are so many improvements. -We have - new documentation, - many performance improvements, - Job Mutators, - better user experience, - correct Ctrl+C handling, - better generic benchmarks support, - more scenarios for passing arguments to benchmarks, - awesome support of console arguments, - unicode support, - LLVM support in MonoDisassembler, - and many-many other improvements and bug fixes! - -A big part of the features and bug fixes were implemented to meet the enterprise requirements of Microsoft to make it possible to port CoreCLR, CoreFX, and CoreFXLab to BenchmarkDotNet. - -The release would not be possible without many contributions from amazing community members. This release is a combined effort. We build BenchmarkDotNet together to make benchmarking .NET code easy and available to everyone for free! - -# New documentation - -We have many improvements in our documentation! -The new docs include: - -* [DocFX](https://dotnet.github.io/docfx/) under the hood -* Detailed changelogs which includes all commits, merged pull requests and resolved issues -* API references -* Code samples for main features: - we generate it automatically based on the `BenchmarkDotNet.Samples` project; - it means that all samples can always be compiled - (no more samples with outdated API) -* Better UI -* Documentation versioning: now it's possible to look at the documentation for recent BenchmarkDotNet versions - -# Performance improvements - -BenchmarkDotNet needs to be capable of running few thousands of CoreFX and CoreCLR benchmarks in an acceptable amount of time. The code itself was already optimized so we needed architectural and design changes to meet this requirement. - -## Generate one executable per runtime settings - -To ensure that the side effects of one benchmark run does not affect another benchmark run BenchmarkDotNet generates, builds and runs every benchmark in a dedicated process. So far we were generating and building one executable per benchmark, now we generate and build one executable per runtime settings. So if you want to run ten thousands of benchmarks for .NET Core 2.1 we are going to generate and build single executable, not ten thousand. If you target multiple runtimes the build is going to be executed in parallel. Moreover, if one of the parallel builds fail it's going to be repeated in a sequential way. - -Previously the time to generate and build 650 benchmarks from our Samples project was **one hour**. Now it's something around **13 seconds** which means **276 X** improvement for this particular scenario. You can see the changes [here](https://github.com/dotnet/BenchmarkDotNet/issues/699). - -## Don't execute long operations more than once per iteration - -BenchmarkDotNet was designed to allow for very accurate and stable micro-benchmarking. One of the techniques that we use is manual loop unrolling. In practice, it meant that for every iteration we were executing the benchmark at least 16 times (the default `UnrollFactor` value). It was of course not desired for the very time-consuming benchmarks. - -So far this feature was always enabled by default and users would need to configure `UnrollFactor=1` to disable it. Now BenchmarkDotNet is going to discover such scenario and don't perform manual loop unrolling for the very time-consuming benchmarks. BenchmarkDotNet uses `Job.IterationTime` setting (the default is 0.5s) in the Pilot Experiment stage to determine how many times given benchmark should be executed per iteration. - -Example: - -```cs -public class Program -{ - static void Main() => BenchmarkRunner.Run(); - - [Benchmark] - public void Sleep1s() => Thread.Sleep(TimeSpan.FromSeconds(1)); -} -``` - -Time to run with the previous version: **374 seconds**. With `0.11.0` it's **27 seconds** which gives us almost **14 X** improvement. A good example of benchmarks that are going to benefit from this change are computer game benchmarks and ML.NET benchmarks. You can see the changes [here](https://github.com/dotnet/BenchmarkDotNet/pull/760) and [here](https://github.com/dotnet/BenchmarkDotNet/pull/771). - -## Exposing more configuration settings - -The default settings were configured to work well with every scenario. Before running the benchmark, BenchmarkDotNet does not know anything about it. This is why it performs many warmup iterations before running the benchmarks. - -When you author benchmarks and run them many times you can come up with custom settings that produce similar results but in a shorter manner of time. To allow you to do that we have exposed: - -* `Job.MinIterationCount` (default value is 15) -* `Job.MaxIterationCount` (default value is 100) -* `Job.MinWarmupIterationCount` (default value is 6) -* `Job.MaxWarmupIterationCount` (default value is 50) - -# User Experience - -One of the biggest success factors of BenchmarkDotNet is a great user experience. The tool just works as expected and makes your life easy. We want to make it even better! - -## .NET Standard 2.0 - -We have ported BenchmarkDotNet to .NET Standard 2.0 and thanks to that we were able to not only simplify our code and build process but also merge `BenchmarkDotNet.Core.dll` and `BenchmarkDotNet.Toolchains.Roslyn.dll` into `BenchmarkDotNet.dll`. We still support .NET 4.6 but we have dropped .NET Core 1.1 support. More information and full discussion can be found [here](https://github.com/dotnet/BenchmarkDotNet/pull/688). - -**Note:** Our `BenchmarkDotNet.Diagnostics.Windows` package which uses `EventTrace` to implement ETW-based diagnosers was also ported to .NET Standard 2.0 and you can now use all the ETW diagnosers with .NET Core on Windows. We plan to add EventPipe support and make this page fully cross-platform and Unix compatible soon. - -## Using complex types as benchmark arguments - -So far we have required the users to implement `IParam` interface to make the custom complex types work as benchmark arguments/parameters. This has changed, now the users can use any complex types as arguments and it will just work ([more](https://github.com/dotnet/BenchmarkDotNet/pull/754)). - -```cs -public class Program -{ - static void Main(string[] args) => BenchmarkRunner.Run(); - - public IEnumerable Arguments() - { - yield return new Point2D(10, 200); - } - - [Benchmark] - [ArgumentsSource(nameof(Arguments))] - public int WithArgument(Point2D point) => point.X + point.Y; -} - -public class Point2D -{ - public int X, Y; - - public Point2D(int x, int y) - { - X = x; - Y = y; - } - - public override string ToString() => $"[{X},{Y}]"; -} -``` - -**Note**: If you want to control what will be displayed in the summary you should override `ToString`. - -## If IterationSetup is provided run benchmark once per iteration - -When Stephen Toub says that something is [buggy](https://github.com/dotnet/BenchmarkDotNet/issues/730), it most probably is. BenchmarkDotNet performs multiple invocations of benchmark per every iteration. When we have exposed the `[IterationSetup]` attribute many users were expecting that the `IterationSetup` is going to be invoked before every benchmark execution. - -It was invoked before every iteration, and iteration was more than one benchmark call if the user did not configure that explicitly. We have changed that and now if you provide an `[IterationSetup]` method it is going to be executed before every iteration and iteration will invoke the benchmark just once. - -```cs -public class Test -{ - public static void Main() => BenchmarkRunner.Run(); - - [IterationSetup] - public void MySetup() => Console.WriteLine("MySetup"); - - [Benchmark] - public void MyBenchmark() => Console.WriteLine("MyBenchmark"); -} -``` - -Before: - -```log -MySetup -MyBenchmark -MyBenchmark -MyBenchmark -MyBenchmark -(...) -``` - -After: - -```log -MySetup -MyBenchmark -MySetup -MyBenchmark -MySetup -MyBenchmark -(...) -``` - -**Note:** If you want to configure how many times benchmark should be invoked per iteration you can use the new `[InvocationCountAttribute]`. - -## Job Mutators - -`Job` represents a set of settings to run the benchmarks. We run every benchmark for every job defined by the user. The problem was that so far many jobs were just added to the config instead of being merged with other jobs. - -An example: - -```cs -[ClrJob, CoreJob] -[GcServer(true)] -public class MyBenchmarkClass -``` - -Resulted in 3 jobs and 3 benchmark executions: `ClrJob`, `CoreJob` and `GcServer(true)` for current runtime. - -Now all Jobs and their corresponding attributes marked as mutators are going to be applied to other jobs, not just added to the config. So in this particular scenario, the benchmarks from `MyBenchmarkClass` are going to be executed for .NET with Server GC enabled and .NET Core with Server GC enabled. - -Mutators are great when you want to have a single, global config for all benchmarks and apply given settings only to selected types. You can find out more about mutators [here](https://github.com/dotnet/BenchmarkDotNet/pull/800). - -## Ctrl+C - -When the user: - -* presses `Ctrl+C` -* presses `Ctrl+Break` -* logs off -* closes console window - -We are now going to close any existing ETW session created by BenchmarkDotNet and **restore console colors** ([read more](https://github.com/dotnet/BenchmarkDotNet/pull/761)). - -## Handle OutOfMemoryException more gracefully - -When our benchmark hits `OutOfMemoryException` we print some nice explanation: - -```cs -public class Program -{ - static void Main(string[] args) => BenchmarkRunner.Run(); - - private List list = new List(); - - [Benchmark] - public void AntiPattern() => list.Add(new int[int.MaxValue / 2]); -} -``` - -```log -OutOfMemoryException! -BenchmarkDotNet continues to run additional iterations until desired accuracy level is achieved. It's possible only if the benchmark method doesn't have any side-effects. -If your benchmark allocates memory and keeps it alive, you are creating a memory leak. -You should redesign your benchmark and remove the side-effects. You can use `OperationsPerInvoke`, `IterationSetup` and `IterationCleanup` to do that. -``` - -## Trimming long strings - -We used to display the values "as is" which was bad for long strings. Now the values are trimmed ([more](https://github.com/dotnet/BenchmarkDotNet/issues/748)). - -```cs -public class Long -{ - [Params("text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7")] - public string Text; - - [Benchmark] - public int HashCode() => Text.GetHashCode(); -} -``` - -| Method | Text | -|--------- |--------------------- | -| HashCode | text/(...)q=0.7 [86] | - -# More features - -## Generic benchmarks - -BenchmarkDotNet supports generic benchmarks, all you need to do is to tell it which types should be used as generic arguments ([read more](https://github.com/dotnet/BenchmarkDotNet/pull/758)). - -```cs -[GenericTypeArguments(typeof(int))] -[GenericTypeArguments(typeof(char))] -public class IntroGenericTypeArguments -{ - [Benchmark] public T Create() => Activator.CreateInstance(); -} -``` - -## Arguments - -We now support more scenarios for passing arguments to benchmarks: - -* passing arguments to asynchronous benchmarks ([more](https://github.com/dotnet/BenchmarkDotNet/issues/818)) -* passing generic types -* passing arguments by reference -* passing jagged arrays ([more](https://github.com/dotnet/BenchmarkDotNet/issues/769)) -* types with implicit cast operator to stack only types can be passed as given stack-only types to Benchmarks ([more](https://github.com/dotnet/BenchmarkDotNet/issues/774)) - -Example: - -```cs -public class WithStringToReadOnlySpan -{ - [Benchmark] - [Arguments("some string")] - public void AcceptsReadOnlySpan(ReadOnlySpan notString) -} -``` - -## Console Arguments - -`BenchmarkSwitcher` supports various console arguments ([PR](https://github.com/dotnet/BenchmarkDotNet/pull/824)), to make it work you need to pass the `args` to switcher: - -```cs -class Program -{ - static void Main(string[] args) - => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); -} -``` - -**Note:** to get the most up-to-date info about supported console arguments run the benchmarks with `--help`. - -### Filter - -The `--filter` or just `-f` allows you to filter the benchmarks by their full name (`namespace.typeName.methodName`) using glob patterns. - -Examples: - -1. Run all benchmarks from System.Memory namespace: `-f System.Memory*` -2. Run all benchmarks: `-f *` -3. Run all benchmarks from ClassA and ClassB `-f *ClassA* *ClassB*` - -**Note**: If you would like to **join** all the results into a **single summary**, you need to use `--join`. - -### Categories - -You can also filter the benchmarks by categories: - -* `--anyCategories` - runs all benchmarks that belong to **any** of the provided categories -* `--allCategories`- runs all benchmarks that belong to **all** provided categories - -### Diagnosers - -* `-m`, `--memory` - enables MemoryDiagnoser and prints memory statistics -* `-d`, `--disassm`- enables DisassemblyDiagnoser and exports diassembly of benchmarked code - -### Runtimes - -The `--runtimes` or just `-r` allows you to run the benchmarks for selected Runtimes. Available options are: Clr, Mono, Core and CoreRT. - -Example: run the benchmarks for .NET and .NET Core: - -```log -dotnet run -c Release -- --runtimes clr core -``` - -### More arguments - -* `-j`, `--job` (Default: Default) Dry/Short/Medium/Long or Default -* `-e`, `--exporters` GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML -* `-i`, `--inProcess` (Default: false) Run benchmarks in Process -* `-a`, `--artifacts` Valid path to accessible directory -* `--outliers` (Default: OnlyUpper) None/OnlyUpper/OnlyLower/All -* `--affinity` Affinity mask to set for the benchmark process -* `--allStats` (Default: false) Displays all statistics (min, max & more) -* `--attribute` Run all methods with given attribute (applied to class or method) - -## Other small improvements - -* **Unicode support:** - now you can enable support of Unicode symbols like `μ` or `±` with `[EncodingAttribute.Unicode]`, - an example: BenchmarkDotNet.Samples.IntroEncoding - (see [#735](https://github.com/dotnet/BenchmarkDotNet/pull/735)) -* **Better benchmark validation** - (see [#693](https://github.com/dotnet/BenchmarkDotNet/pull/693), [#737](https://github.com/dotnet/BenchmarkDotNet/pull/737)) -* **Improve .NET Framework version detection**: now we support .NET Framework 4.7.2 - (see [#743](https://github.com/dotnet/BenchmarkDotNet/pull/743)) -* **OutlierModes:** - now it's possible to control how to process outliers, - an example @BenchmarkDotNet.Samples.IntroOutliers - (see [#766](https://github.com/dotnet/BenchmarkDotNet/pull/766)) -* **LLVM support in MonoDisassembler** - (see [a7426e](https://github.com/dotnet/BenchmarkDotNet/commit/a7426e84fde075503f489fdf096a95f694f77b85)) -* **Grand API renaming** - we try not to change public API, but sometimes it's necessary because we want to get a consistent and understandable API in v1.0.0. - (see [#787](https://github.com/dotnet/BenchmarkDotNet/issues/787)) -* **Many-many small improvements and bug fixes** \ No newline at end of file diff --git a/docs/_changelog/header/v0.11.1.md b/docs/_changelog/header/v0.11.1.md deleted file mode 100644 index 97efdd26d2..0000000000 --- a/docs/_changelog/header/v0.11.1.md +++ /dev/null @@ -1,11 +0,0 @@ -This release includes some minor improvements and bug fixes: - -* Fixed `RPlotExporter` ([#852](https://github.com/dotnet/BenchmarkDotNet/issues/852), [#855](https://github.com/dotnet/BenchmarkDotNet/issues/855)). - In v0.11.0, the plot generation was broken because of the huge renaming in [#787](https://github.com/dotnet/BenchmarkDotNet/issues/787). -* [ArgumentsSource](xref:BenchmarkDotNet.Samples.IntroArgumentsSource) now supports additional types like - `Type` ([#840](https://github.com/dotnet/BenchmarkDotNet/issues/840)), - `BigInteger` ([#850](https://github.com/dotnet/BenchmarkDotNet/issues/850)), - `DateTime` ([#853](https://github.com/dotnet/BenchmarkDotNet/issues/853)), - and special double values like `double.NaN` ([#851](https://github.com/dotnet/BenchmarkDotNet/issues/851)) -* Generated projects ignore Directory.Build.props and Directory.Build.targets files [#854](https://github.com/dotnet/BenchmarkDotNet/pull/854) -* Now it's possible to run benchmarks with CoreRun ([de152c](https://github.com/dotnet/BenchmarkDotNet/commit/de152c7acc71eddeaa304c846cc67e6a54ca7a0f), [#857](https://github.com/dotnet/BenchmarkDotNet/pull/857)) diff --git a/docs/_changelog/header/v0.11.2.md b/docs/_changelog/header/v0.11.2.md deleted file mode 100644 index 2e6c435a92..0000000000 --- a/docs/_changelog/header/v0.11.2.md +++ /dev/null @@ -1,657 +0,0 @@ -This release includes many PRs from the Hacktoberfest. -We are very grateful to all the contributors who spent their time to help us make BenchmarkDotNet even better! - -## Highlights - -In this release, we have many improvements in different areas: - -* **Diagnosers** - * EtwProfiler (allows profiling benchmarks on Windows and exporting the data to a trace file) -* **Execution:** - * Comparing NuGet packages (now it's possible to compare different versions of the same package) - * .NET Core 3.0 support - * Deferred Execution Validator -* **Command-line:** - * `--list`: List of benchmarks - * `--info`: Print environment info - * `--runtimes`: Choosing execution runtimes (`--runtimes net472 netcoreapp2.1` will executed a benchmark on .NET 4.7.2 and .NET Core 2.1) - * Options for number of invocations and iterations - * Custom default settings for console argument parser - * Case-insensitive filter - * Benchmarking with different CoreRun instances - * Hardware counters command-line support -* **Exporters:** - * Markdown output for DisassemblyDiagnoser - * Diff view for disassembler output - * Improved LINQPad support (colored monospaced logs) - * Better CPU brand strings -* **Attributes:** - * Async `[GlobalSetup]` and `[GlobalCleanup]` support - * Introduced `[ParamsAllValues]` - * Selecting Baseline across Methods and Jobs -* **Statistics:** - * Better statistical tests (Welch's t-test and Mann-Whitney U-test) - * ZeroMeasurementAnalyser - * RatioColumn -* **Other:** - * Azure Pipelines support for internal builds - * Many minor bug fixes - * Improved documentation - * New tests - ---- - -## Diagnosers - -### EtwProfiler - -`EtwProfiler` allows to profile the benchmarked .NET code on Windows and exports the data to a trace file which can be opened with [PerfView](https://github.com/Microsoft/perfview) or [Windows Performance Analyzer](https://learn.microsoft.com/windows-hardware/test/wpt/windows-performance-analyzer). - -`EtwProfiler` uses `TraceEvent` library which internally uses Event Tracing for Windows (ETW) to capture stack traces and important .NET Runtime events. -Before the process with benchmarked code is started, EtwProfiler starts User and Kernel ETW sessions. Every session writes data to it's own file and captures different data. User session listens for the .NET Runtime events (GC, JIT etc) while the Kernel session gets CPU stacks and Hardware Counter events. After this, the process with benchmarked code is started. During the benchmark execution all the data is captured and written to a trace file. Moreover, BenchmarkDotNet Engine emits it's own events to be able to differentiate jitting, warmup, pilot and actual workload when analyzing the trace file. When the benchmarking is over, both sessions are closed and the two trace files are merged into one. - -![](https://adamsitnik.com/images/etwprofiler/flamegraph.png) - -You can find more details - in the [documentation](xref:docs.etwprofiler) and - in the [blog post](https://adamsitnik.com/ETW-Profiler/) by Adam Sitnik. - -* [#878](https://github.com/dotnet/BenchmarkDotNet/pull/878) EtwProfiler Diagnoser (by [@adamsitnik](https://github.com/adamsitnik)) -* [04a715](https://github.com/dotnet/BenchmarkDotNet/commit/04a71586206a822bca56f0abdacefdc2e5fc1b01) EtwProfiler Diagnoser (#878) (by [@adamsitnik](https://github.com/adamsitnik)) - ---- - -## Execution - -### Comparing NuGet packages - -Now it's possible to compare performance of several versions of the same NuGet package. -An example: - -```cs -[Config(typeof(Config))] -public class IntroNuGet -{ - // Specify jobs with different versions of the same NuGet package to benchmark. - // The NuGet versions referenced on these jobs must be greater or equal to the - // same NuGet version referenced in this benchmark project. - // Example: This benchmark project references Newtonsoft.Json 9.0.1 - private class Config : ManualConfig - { - public Config() - { - var baseJob = Job.MediumRun.With(CsProjCoreToolchain.Current.Value); - Add(baseJob.WithNuGet("Newtonsoft.Json", "11.0.2").WithId("11.0.2")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "11.0.1").WithId("11.0.1")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "10.0.3").WithId("10.0.3")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "10.0.2").WithId("10.0.2")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "10.0.1").WithId("10.0.1")); - Add(baseJob.WithNuGet("Newtonsoft.Json", "9.0.1").WithId("9.0.1")); - } - } - - [Benchmark] - public void SerializeAnonymousObject() - => JsonConvert.SerializeObject( - new { hello = "world", price = 1.99, now = DateTime.UtcNow }); -} -``` - -See also: @BenchmarkDotNet.Samples.IntroNuGet - -* [#290](https://github.com/dotnet/BenchmarkDotNet/issues/290) Question: Any official way to benchmark same method between different assembly versions? -* [#931](https://github.com/dotnet/BenchmarkDotNet/issues/931) Same NuGet version used when benchmarking different packages -* [#922](https://github.com/dotnet/BenchmarkDotNet/pull/922) Enables benchmarking betweeen different Nuget packages (by [@Shazwazza](https://github.com/Shazwazza)) -* [#932](https://github.com/dotnet/BenchmarkDotNet/pull/932) Partition benchmark run info based on added nuget packages (by [@blairconrad](https://github.com/blairconrad)) -* [92a786](https://github.com/dotnet/BenchmarkDotNet/commit/92a7869aaa30aeacaf1da2dcc45bc65c8333ae73) Enables benchmarking betweeen different Nuget packages (#922) fixes #290 (by [@Shazwazza](https://github.com/Shazwazza)) -* [510685](https://github.com/dotnet/BenchmarkDotNet/commit/510685f48ce2baf57682aa82e18c6486989e9625) Partition benchmark run info based on added nuget packages (#932) (by [@blairconrad](https://github.com/blairconrad)) -* [cf84a4](https://github.com/dotnet/BenchmarkDotNet/commit/cf84a44d108d5bf3860129e0a2a78cace9c95626) NuGet casing fix (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - -### .NET Core 3.0 support - -Now it's possible to run benchmarks on preview versions of .NET Core 3.0. - -* [#874](https://github.com/dotnet/BenchmarkDotNet/issues/874) .NET Core 3.0 support (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [2e398c](https://github.com/dotnet/BenchmarkDotNet/commit/2e398c89561b3b1c89ec64b94f656ae20236efd1) detect .NET Core 3.0 and use the appropriate target framework moniker, fixes ... (by [@adamsitnik](https://github.com/adamsitnik)) - -### Deferred Execution Validator - -In LINQ, execution of a query is usually [deferred](https://learn.microsoft.com/dotnet/standard/linq/deferred-execution-example) until the moment when you actually request the data. If your benchmark just returns `IEnumerable` or `IQueryable` it's not measuring the execution of the query, just the creation. - -This is why we decided to warn you about this issue whenever it happens: - -```log -Benchmark IntroDeferredExecution.Wrong returns a deferred execution result (IEnumerable). You need to either change the method declaration to return a materialized result or consume it on your own. You can use .Consume() extension method to do that. -``` - -Don't worry! We are also providing you with a `Consume` extension method which can execute given `IEnumerable` or `IQueryable` and consume its results. All you need to do is to create a [`Consumer`](xref:BenchmarkDotNet.Engines.Consumer) instance, preferably store it in a field (to exclude the cost of creating Consumer from the benchmark itself) and pass it to `Consume` extension method. - -**Do not call `.ToArray()` because it's an expensive operation and it might dominate given benchmark!** - -See also: @BenchmarkDotNet.Samples.IntroDeferredExecution - -* [#858](https://github.com/dotnet/BenchmarkDotNet/issues/858) Should the Engine iterate over and consume IEnumerable and IQueryable results? (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [cebe2a](https://github.com/dotnet/BenchmarkDotNet/commit/cebe2a0f84fa21acb6db9613fe3a4326d635f129) Deferred Execution Validator, fixes #858 (by [@adamsitnik](https://github.com/adamsitnik)) - ---- - -## Command-line - -In this release, we have tons of improvements for command-line experience. - -### `--list`: List of benchmarks - -The `--list` allows you to print all of the available benchmark names. Available options are: - -* `flat` - prints list of the available benchmarks: `--list flat` -```ini -BenchmarkDotNet.Samples.Algo_Md5VsSha256.Md5 -BenchmarkDotNet.Samples.Algo_Md5VsSha256.Sha256 -BenchmarkDotNet.Samples.IntroArguments.Benchmark -BenchmarkDotNet.Samples.IntroArgumentsSource.SingleArgument -BenchmarkDotNet.Samples.IntroArgumentsSource.ManyArguments -BenchmarkDotNet.Samples.IntroArrayParam.ArrayIndexOf -BenchmarkDotNet.Samples.IntroArrayParam.ManualIndexOf -BenchmarkDotNet.Samples.IntroBasic.Sleep -[...] -``` -* `tree` - prints tree of the available benchmarks: `--list tree` -```ini -BenchmarkDotNet - └─Samples - ├─Algo_Md5VsSha256 - │ ├─Md5 - │ └─Sha256 - ├─IntroArguments - │ └─Benchmark - ├─IntroArgumentsSource - │ ├─SingleArgument - │ └─ManyArguments - ├─IntroArrayParam - │ ├─ArrayIndexOf - │ └─ManualIndexOf - ├─IntroBasic - │ ├─Sleep -[...] -``` - -The `--list` option works with the `--filter` option. Examples: - -* `--list flat --filter *IntroSetupCleanup*` prints: -```ini -BenchmarkDotNet.Samples.IntroSetupCleanupGlobal.Logic -BenchmarkDotNet.Samples.IntroSetupCleanupIteration.Benchmark -BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkA -BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkB -BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkC -BenchmarkDotNet.Samples.IntroSetupCleanupTarget.BenchmarkD -``` -* `--list tree --filter *IntroSetupCleanup*` prints: -```ini -BenchmarkDotNet - └─Samples - ├─IntroSetupCleanupGlobal - │ └─Logic - ├─IntroSetupCleanupIteration - │ └─Benchmark - └─IntroSetupCleanupTarget - ├─BenchmarkA - ├─BenchmarkB - ├─BenchmarkC - └─BenchmarkD -``` - -* [#905](https://github.com/dotnet/BenchmarkDotNet/issues/905) Implement `--list` -* [#914](https://github.com/dotnet/BenchmarkDotNet/pull/914) Implement `--list` - fixes #905 (by [@wojtpl2](https://github.com/wojtpl2)) -* [#916](https://github.com/dotnet/BenchmarkDotNet/pull/916) Update console-args.md - add information about `--list` option (by [@wojtpl2](https://github.com/wojtpl2)) -* [330f66](https://github.com/dotnet/BenchmarkDotNet/commit/330f66c3a3d94d1369d5c0b629bbb0085d5db8eb) Implement `--list` - fixes #905 (#914) (by [@wojtpl2](https://github.com/wojtpl2)) -* [6c7521](https://github.com/dotnet/BenchmarkDotNet/commit/6c7521d4fd6776098667944321c8a65848382ae5) Update console-args.md - add information about `--list` option (#916) (by [@wojtpl2](https://github.com/wojtpl2)) - -### `--info`: Print environment info - -Some of our users really like the info we print about hardware and OS. -Now we have the `--info` console line argument which does not run the benchmarks, but simply prints the info. - -```ini -BenchmarkDotNet=v0.11.1.786-nightly, OS=Windows 10.0.17134.285 (1803/April2018Update/Redstone4) -Intel Xeon CPU E5-1650 v4 3.60GHz, 1 CPU, 12 logical and 6 physical cores -Frequency=3507500 Hz, Resolution=285.1033 ns, Timer=TSC -.NET Core SDK=3.0.100-alpha1-009642 - [Host] : .NET Core 3.0.0-preview1-27004-04 (CoreCLR 4.6.27003.04, CoreFX 4.6.27003.02), 64bit RyuJIT -``` - -* [#904](https://github.com/dotnet/BenchmarkDotNet/issues/904) Implement `--info` -* [#907](https://github.com/dotnet/BenchmarkDotNet/pull/907) fixes #904 Implement `--info` (by [@lahma](https://github.com/lahma)) -* [4be28d](https://github.com/dotnet/BenchmarkDotNet/commit/4be28d25fa9ab79ca194c615783148042738bdad) fixes #904 Implement `--info` (#907) (by [@lahma](https://github.com/lahma)) - -### `--runtimes`: Choosing execution runtimes - -The `--runtimes` or just `-r` allows you to run the benchmarks for selected Runtimes. Available options are: Mono, CoreRT, Core, Clr net46, net461, net462, net47, net471, net472, netcoreapp2.0, netcoreapp2.1, netcoreapp2.2, netcoreapp3.0. - -Example: run the benchmarks for .NET 4.7.2 and .NET Core 2.1: - -```log -dotnet run -c Release -- --runtimes net472 netcoreapp2.1 -``` - -* [#913](https://github.com/dotnet/BenchmarkDotNet/pull/913) .NET Core Toolchains improvements (by [@adamsitnik](https://github.com/adamsitnik)) -* [0f721c](https://github.com/dotnet/BenchmarkDotNet/commit/0f721c8e0e100fc951a54b6045eb7b58c55c2a1f) make it possible to specify runtimes using explicit tfms like net472 or netco... (by [@adamsitnik](https://github.com/adamsitnik)) -* [1c581e](https://github.com/dotnet/BenchmarkDotNet/commit/1c581e5bf5b4ba9f40d113ae09e0731a60523a60) .NET Core Toolchains improvements (#913) (by [@adamsitnik](https://github.com/adamsitnik)) - -### Options for number of invocations and iterations - -* `--launchCount` - how many times we should launch process with target benchmark. The default is 1. -* `--warmupCount` - how many warmup iterations should be performed. If you set it, the minWarmupCount and maxWarmupCount are ignored. By default calculated by the heuristic. -* `--minWarmupCount` - minimum count of warmup iterations that should be performed. The default is 6. -* `--maxWarmupCount` - maximum count of warmup iterations that should be performed. The default is 50. -* `--iterationTime` - desired time of execution of an iteration. Used by Pilot stage to estimate the number of invocations per iteration. 500ms by default. -* `--iterationCount` - how many target iterations should be performed. By default calculated by the heuristic. -* `--minIterationCount` - minimum number of iterations to run. The default is 15. -* `--maxIterationCount` - maximum number of iterations to run. The default is 100. -* `--invocationCount` - invocation count in a single iteration. By default calculated by the heuristic. -* `--unrollFactor` - how many times the benchmark method will be invoked per one iteration of a generated loop. 16 by default -* `--runOncePerIteration` - run the benchmark exactly once per iteration. False by default. - -Example: run single warmup iteration, from 9 to 12 actual workload iterations. - -```log -dotnet run -c Release -- --warmupCount 1 --minIterationCount 9 --maxIterationCount 12 -``` - -* [#902](https://github.com/dotnet/BenchmarkDotNet/pull/902) More command line args (by [@adamsitnik](https://github.com/adamsitnik)) -* [ba0d22](https://github.com/dotnet/BenchmarkDotNet/commit/ba0d22b41fd25022e3a945fe5ef1ae8aea697cf7) allow to configure the number of invocations and iterations from command line (by [@adamsitnik](https://github.com/adamsitnik)) - -### Custom default settings for console argument parser - -If you want to have a possibility to specify custom default Job settings programmatically and optionally overwrite it with console line arguments, then you should create a global config with single job marked as `.AsDefault` and pass it to `BenchmarkSwitcher` together with the console line arguments. - -Example: run single warmup iteration by default. - -```cs -static void Main(string[] args) - => BenchmarkSwitcher - .FromAssembly(typeof(Program).Assembly) - .Run(args, GetGlobalConfig()); - -static IConfig GetGlobalConfig() - => DefaultConfig.Instance - .With(Job.Default - .WithWarmupCount(1) - .AsDefault()); // the KEY to get it working -``` - -Now, the default settings are: `WarmupCount=1` but you might still overwrite it from console args like in the example below: - -```log -dotnet run -c Release -- --warmupCount 2 -``` - -### Case-insensitive filter - -The `--filter` or just `-f` allows you to filter the benchmarks by their full name (`namespace.typeName.methodName`) using glob patterns. - -Examples: - -1. Run all benchmarks from System.Memory namespace: `-f System.Memory*` -2. Run all benchmarks: `-f *` -3. Run all benchmarks from ClassA and ClassB `-f *ClassA* *ClassB*` - -Now this filter expression is case-insensitive. - -* [#864](https://github.com/dotnet/BenchmarkDotNet/issues/864) Make the filter case insensitive (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [106777](https://github.com/dotnet/BenchmarkDotNet/commit/106777f7f575a8535f16292f1de80e8ffba2853a) make the filter case insensitive invariant culture, fixes #864 (by [@adamsitnik](https://github.com/adamsitnik)) - -### Benchmarking with different CoreRun instances - -CoreRun is a simpler version of `dotnet run`, used for developing CoreCLR and CoreFX. - -Typically when working on the performance of .NET Core a developer has more than 1 copy of CoreRun. -Example: CoreRun before my changes, and after my changes. -This change allows to simply run same benchmark for few different CoreRuns to compare the perf in easy way. - -Sample usage: - -```log -dotnet run -c Release -f netcoreapp2.1 -- -f *Empty.method --job dry --coreRun -C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root\CoreRun.exe -C:\Projects\coreclr_upstream\bin\tests\Windows_NT.x64.Release\Tests\Core_Root_beforeMyChanges\CoreRun.exe -``` - -Sample output: - -![image](https://user-images.githubusercontent.com/6011991/47417561-860f5800-d778-11e8-8dc6-b4e3586a9c90.png) - -* [#925](https://github.com/dotnet/BenchmarkDotNet/issues/925) Make it possible to run the benchmark with multiple CoreRun.exe (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [901616](https://github.com/dotnet/BenchmarkDotNet/commit/90161654725efd5639e0190638a3383d6a49e34c) when user provides CoreRun path and runtime in explicit way, we should use th... (by [@adamsitnik](https://github.com/adamsitnik)) -* [46bebf](https://github.com/dotnet/BenchmarkDotNet/commit/46bebf1497d4e9314c6dfd2d4e10df81332aa4fa) allow the users to run the same benchmarks using few different CoreRun.exe, f... (by [@adamsitnik](https://github.com/adamsitnik)) - -### Hardware counters command-line support - -```log ---counters CacheMisses+InstructionRetired -``` - -* [1e3df7](https://github.com/dotnet/BenchmarkDotNet/commit/1e3df74b2f927f541bed723f65c2d571fa850c53) make it possible to specify hardware counters from command line (by [@adamsitnik](https://github.com/adamsitnik)) -* [a4f91a](https://github.com/dotnet/BenchmarkDotNet/commit/a4f91a392675e4851a785095af162b977d249ba3) better handling of edge cases for parsing hardware counters from the console ... (by [@adamsitnik](https://github.com/adamsitnik)) - ---- - -## Exporters - -### Markdown output for DisassemblyDiagnoser - -Now `DisassemblyDiagnoser` generates markdown version of the assembly listing. - -* [#560](https://github.com/dotnet/BenchmarkDotNet/issues/560) Suggestion: markdown output for DisassemblyDiagnoser (assignee: [@adamsitnik](https://github.com/adamsitnik)) -* [1e6235](https://github.com/dotnet/BenchmarkDotNet/commit/1e62355f209a25c7a33f9ab7e7e03b0afe7d851f) github markdown exporter for Disassembler, fixes #560 (by [@adamsitnik](https://github.com/adamsitnik)) - -### Diff view for disassembler output - -Now we have `PrettyGithubMarkdownDiffDisassemblyExporter` which can generates - nice diffs between assembly listings. -This mode can be activated via the `--disasmDiff` command line argument or - the `printDiff: true` argument of `DisassemblyDiagnoserConfig`. -An output example (Diff between SumLocal and SumField on .NET Core 2.1.4 (CoreCLR 4.6.26814.03, CoreFX 4.6.26814.02), 64bit RyuJIT) - -```diff --; BenchmarkDotNet.Samples.IntroDisassemblyRyuJit.SumLocal() -- var local = field; // we use local variable that points to the field -- ^^^^^^^^^^^^^^^^^^ -- mov rax,qword ptr [rcx+8] -+; BenchmarkDotNet.Samples.IntroDisassemblyRyuJit.SumField() - int sum = 0; - ^^^^^^^^^^^^ -- xor edx,edx -- for (int i = 0; i < local.Length; i++) -+ xor eax,eax -+ for (int i = 0; i < field.Length; i++) - ^^^^^^^^^ -- xor ecx,ecx -- for (int i = 0; i < local.Length; i++) -+ xor edx,edx -+ for (int i = 0; i < field.Length; i++) - ^^^^^^^^^^^^^^^^ -- mov r8d,dword ptr [rax+8] -- test r8d,r8d -+ mov rcx,qword ptr [rcx+8] -+ cmp dword ptr [rcx+8],0 - jle M00_L01 -- sum += local[i]; -+ sum += field[i]; - ^^^^^^^^^^^^^^^^ - M00_L00: -- movsxd r9,ecx -- add edx,dword ptr [rax+r9*4+10h] -- for (int i = 0; i < local.Length; i++) -+ mov r8,rcx -+ cmp edx,dword ptr [r8+8] -+ jae 00007ff9`0c412c1f -+ movsxd r9,edx -+ add eax,dword ptr [r8+r9*4+10h] -+ for (int i = 0; i < field.Length; i++) - ^^^ -- inc ecx -- cmp r8d,ecx -+ inc edx -+ cmp dword ptr [rcx+8],edx - jg M00_L00 - return sum; - ^^^^^^^^^^^ - M00_L01: -- mov eax,edx --; Total bytes of code 34 -+ add rsp,28h -+; Total bytes of code 42 -``` - -* [#544](https://github.com/dotnet/BenchmarkDotNet/issues/544) Diff view for disassembler output (assignee: [@wojtpl2](https://github.com/wojtpl2)) -* [#927](https://github.com/dotnet/BenchmarkDotNet/pull/927) Improve Disassembly exporters and add PrettyGithubMarkdownDiffDisassemblyExporter (by [@wojtpl2](https://github.com/wojtpl2)) -* [#936](https://github.com/dotnet/BenchmarkDotNet/issues/936) Producing the asm diff reports on demand -* [#937](https://github.com/dotnet/BenchmarkDotNet/pull/937) Producing the asm diff reports on demand - fix for #936 (by [@wojtpl2](https://github.com/wojtpl2)) -* [1903a1](https://github.com/dotnet/BenchmarkDotNet/commit/1903a1bd96d207ed51611d1dc546920f5bfb0d86) Improve Disassembly exporters and add PrettyGithubMarkdownDiffDisassemblyExpo... (by [@wojtpl2](https://github.com/wojtpl2)) -* [dd103b](https://github.com/dotnet/BenchmarkDotNet/commit/dd103b60a4af0d3b9e7efb523c0923e7cbd8b62d) Producing the asm diff reports on demand - fixes #936 (#937) (by [@wojtpl2](https://github.com/wojtpl2)) - -### Improved LINQPad support - -If you run BenchmarkDotNet v0.11.2+ in LINQPad, your logs will be colored and monospaced: - -![linqpad](https://user-images.githubusercontent.com/2259237/47839043-4ec92880-ddc2-11e8-838d-960f6d5449e5.png) - -* [#447](https://github.com/dotnet/BenchmarkDotNet/issues/447) Implement ColoredLogger for LinqPad -* [#903](https://github.com/dotnet/BenchmarkDotNet/pull/903) Add LINQPad logging (by [@bgrainger](https://github.com/bgrainger)) -* [#915](https://github.com/dotnet/BenchmarkDotNet/pull/915) Use a monospaced font for LINQPad logging output (by [@bgrainger](https://github.com/bgrainger)) -* [c3b609](https://github.com/dotnet/BenchmarkDotNet/commit/c3b6095b933b132c1773ced3af126f282465b980) Add LINQPad logging (#903) (by [@bgrainger](https://github.com/bgrainger)) -* [10fdd0](https://github.com/dotnet/BenchmarkDotNet/commit/10fdd0998b46c4358f6fa38aacc21e57a7730724) Use a monospaced font for LINQPad logging output. (#915) (by [@bgrainger](https://github.com/bgrainger)) - -### Better CPU brand strings - -We did a lot of changes which improve the presentation form of the CPU brand string. -Here is an example of such string in the previous version of BenchmarkDotNet: - -```log -AMD Ryzen 7 2700X Eight-Core Processor (Max: 4.10GHz), 1 CPU, 16 logical and 8 physical cores -``` - -Now it becomes: - -```log -AMD Ryzen 7 2700X 4.10GHz, 1 CPU, 16 logical and 8 physical cores -``` - -As you can see, "Eight-Core Processor" was removed (because we already have "8 physical cores"); - "(Max: 4.10GHz)" was replaced by 4.10GHz (because the original CPU brand string doesn't contain the nominal frequency). - -* [#859](https://github.com/dotnet/BenchmarkDotNet/issues/859) Strange max frequency values on Windows (assignee: [@Rizzen](https://github.com/Rizzen)) -* [#909](https://github.com/dotnet/BenchmarkDotNet/issues/909) Improve CPU Brand Strings without frequency -* [#860](https://github.com/dotnet/BenchmarkDotNet/pull/860) Fix strange CPU Frequency values (by [@Rizzen](https://github.com/Rizzen)) -* [#910](https://github.com/dotnet/BenchmarkDotNet/pull/910) Simplify AMD Ryzen CPU brand info (by [@lahma](https://github.com/lahma)) -* [a78b38](https://github.com/dotnet/BenchmarkDotNet/commit/a78b38b0e89d04ad3fe8934162c7adb42f81eabe) Fix strange CPU Frequency values (#860) (by [@Rizzen](https://github.com/Rizzen)) -* [5df1e6](https://github.com/dotnet/BenchmarkDotNet/commit/5df1e6434b791eb5da6f6ef42505fc6a94ebd008) Simplify AMD Ryzen CPU brand info (#910) (by [@lahma](https://github.com/lahma)) - ---- - -## Attributes - -### Async GlobalSetup and GlobalCleanup - -Now GlobalSetup and GlobalCleanup methods can be async. - -See also: docs.setup-and-cleanup - -* [#521](https://github.com/dotnet/BenchmarkDotNet/issues/521) Support async Setup/Cleanup -* [#892](https://github.com/dotnet/BenchmarkDotNet/pull/892) Added support for async GlobalSetup. (by [@dlemstra](https://github.com/dlemstra)) -* [#923](https://github.com/dotnet/BenchmarkDotNet/pull/923) async GlobalCleanup support (by [@dlemstra](https://github.com/dlemstra)) -* [#926](https://github.com/dotnet/BenchmarkDotNet/pull/926) Added support for async GlobalCleanup. (by [@dlemstra](https://github.com/dlemstra)) -* [e0f7a6](https://github.com/dotnet/BenchmarkDotNet/commit/e0f7a67681860ead87cef76fa0db349460b34eb0) Added support for async GlobalSetup. (#892) (by [@dlemstra](https://github.com/dlemstra)) -* [a971a4](https://github.com/dotnet/BenchmarkDotNet/commit/a971a435ce6e6ca25d246e5e2cd56c5b2cf4739d) async GlobalCleanup support (#923) (by [@dlemstra](https://github.com/dlemstra)) -* [e4c7b8](https://github.com/dotnet/BenchmarkDotNet/commit/e4c7b852e5593bb280881e28ece51d26687c5ba9) Added support for async GlobalCleanup. (#926), fixes #521 (by [@dlemstra](https://github.com/dlemstra)) - -### Introduced ParamsAllValues - -If you want to use all possible values of an `enum` or another type with a small number of values, you can use the [`[ParamsAllValues]`](xref:BenchmarkDotNet.Attributes.ParamsAllValuesAttribute) attribute, instead of listing all the values by hand. The types supported by the attribute are: - -* `bool` -* any `enum` that is not marked with `[Flags]` -* `Nullable`, where `T` is an enum or boolean - -An example: - -```cs -public class IntroParamsAllValues -{ - public enum CustomEnum - { - A, - BB, - CCC - } - - [ParamsAllValues] - public CustomEnum E { get; set; } - - [ParamsAllValues] - public bool? B { get; set; } - - [Benchmark] - public void Benchmark() - { - Thread.Sleep( - E.ToString().Length * 100 + - (B == true ? 20 : B == false ? 10 : 0)); - } -} -``` - -Output: - -```markdown - Method | E | B | Mean | Error | ----------- |---- |------ |---------:|------:| - Benchmark | A | ? | 101.9 ms | NA | - Benchmark | A | False | 111.9 ms | NA | - Benchmark | A | True | 122.3 ms | NA | - Benchmark | BB | ? | 201.5 ms | NA | - Benchmark | BB | False | 211.8 ms | NA | - Benchmark | BB | True | 221.4 ms | NA | - Benchmark | CCC | ? | 301.8 ms | NA | - Benchmark | CCC | False | 312.3 ms | NA | - Benchmark | CCC | True | 322.2 ms | NA | - -// * Legends * - E : Value of the 'E' parameter - B : Value of the 'B' parameter -``` - -* [#658](https://github.com/dotnet/BenchmarkDotNet/issues/658) [Params] for enums should include all values by default -* [#908](https://github.com/dotnet/BenchmarkDotNet/pull/908) Added [ParamsAllValues] (by [@gsomix](https://github.com/gsomix)) -* [922dff](https://github.com/dotnet/BenchmarkDotNet/commit/922dfff62d6cf6fd808865e705a09eee63690a2e) Added [ParamsAllValues] (#908), fixes #658 (by [@gsomix](https://github.com/gsomix)) -* [846d08](https://github.com/dotnet/BenchmarkDotNet/commit/846d0863b6456d3e1e6ccab06d8e61c5cd064194) ParamsAllValuesValidator fixes (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - -### Selecting Baseline across Methods and Jobs - -Now it's possible to mark a method and a job as baselines at the same time: - -```cs -public class TheBaselines -{ - [Benchmark(Baseline = true)] - public void Sleep100ms() => Thread.Sleep(TimeSpan.FromMilliseconds(100)); - - [Benchmark] - public void Sleep50ms() => Thread.Sleep(TimeSpan.FromMilliseconds(50)); -} - -static void Main(string[] args) - => BenchmarkSwitcher - .FromTypes(new[] { typeof(TheBaselines) }) - .Run(args, - DefaultConfig.Instance - .With(Job.Core.AsBaseline()) - .With(Job.Clr.WithId("CLR 4.7.2"))); -``` - -* [#880](https://github.com/dotnet/BenchmarkDotNet/issues/880) Select Baseline across Methods and Jobs (assignee: [@AndreyAkinshin](https://github.com/AndreyAkinshin)) -* [21a007](https://github.com/dotnet/BenchmarkDotNet/commit/21a0073cc8b486f41b2e84deafacd00a1303013a) Support method-job baseline pairs, fixes #880 (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - ---- - -## Statistics - -### Statistical tests - -In this release, statistical testing was significantly improved. -Now it's possible to compare all benchmarks against baseline with the help - of Welch's t-test or Mann–Whitney U test. - -An example: - -```cs -[StatisticalTestColumn( - StatisticalTestKind.Welch, ThresholdUnit.Microseconds, 1, true)] -[StatisticalTestColumn( - StatisticalTestKind.MannWhitney, ThresholdUnit.Microseconds, 1, true)] -[StatisticalTestColumn( - StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.03, true)] -[StatisticalTestColumn( - StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.03, true)] -[SimpleJob(warmupCount: 0, targetCount: 5)] -public class IntroStatisticalTesting -{ - [Benchmark] public void Sleep50() => Thread.Sleep(50); - [Benchmark] public void Sleep97() => Thread.Sleep(97); - [Benchmark] public void Sleep99() => Thread.Sleep(99); - [Benchmark(Baseline = true)] public void Sleep100() => Thread.Sleep(100); - [Benchmark] public void Sleep101() => Thread.Sleep(101); - [Benchmark] public void Sleep103() => Thread.Sleep(103); - [Benchmark] public void Sleep150() => Thread.Sleep(150); -} -``` - -Output: - -| Method | Mean | Error | StdDev | Ratio | Welch(1us)/p-values | Welch(3%)/p-values | MannWhitney(1us)/p-values | MannWhitney(3%)/p-values | -|--------- |----------:|----------:|----------:|------:|---------------------- |---------------------- |-------------------------- |------------------------- | -| Sleep50 | 53.13 ms | 0.5901 ms | 0.1532 ms | 0.51 | Faster: 1.0000/0.0000 | Faster: 1.0000/0.0000 | Faster: 1.0000/0.0040 | Faster: 1.0000/0.0040 | -| Sleep97 | 100.07 ms | 0.9093 ms | 0.2361 ms | 0.97 | Faster: 1.0000/0.0000 | Same: 1.0000/0.1290 | Faster: 1.0000/0.0040 | Same: 1.0000/0.1111 | -| Sleep99 | 102.23 ms | 2.4462 ms | 0.6353 ms | 0.99 | Faster: 0.9928/0.0072 | Same: 1.0000/0.9994 | Faster: 0.9960/0.0079 | Same: 1.0000/1.0000 | -| Sleep100 | 103.34 ms | 0.8180 ms | 0.2124 ms | 1.00 | Base: 0.5029/0.5029 | Base: 1.0000/1.0000 | Base: 0.7262/0.7262 | Base: 1.0000/1.0000 | -| Sleep101 | 103.73 ms | 2.1591 ms | 0.5607 ms | 1.00 | Same: 0.1041/0.8969 | Same: 0.9999/1.0000 | Same: 0.1111/0.9246 | Same: 1.0000/1.0000 | -| Sleep103 | 106.21 ms | 1.2511 ms | 0.3249 ms | 1.03 | Slower: 0.0000/1.0000 | Same: 0.9447/1.0000 | Slower: 0.0040/1.0000 | Same: 0.9246/1.0000 | -| Sleep150 | 153.16 ms | 3.4929 ms | 0.9071 ms | 1.48 | Slower: 0.0000/1.0000 | Slower: 0.0000/1.0000 | Slower: 0.0040/1.0000 | Slower: 0.0040/1.0000 | - -```log -// * Legends * - Mean : Arithmetic mean of all measurements - Error : Half of 99.9% confidence interval - StdDev : Standard deviation of all measurements - Ratio : Mean of the ratio distribution ([Current]/[Baseline]) - Welch(1us)/p-values : Welch-based TOST equivalence test with 1 us threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' - Welch(3%)/p-values : Welch-based TOST equivalence test with 3% threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' - MannWhitney(1us)/p-values : MannWhitney-based TOST equivalence test with 1 us threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' - MannWhitney(3%)/p-values : MannWhitney-based TOST equivalence test with 3% threshold. Format: 'Result: p-value(Slower)|p-value(Faster)' - 1 ms : 1 Millisecond (0.001 sec) -``` - -The statistical testing is a work-in-progress feature. -In future versions of BenchmarkDotNet, we are planning to improve API, fill missed docs, and introduce more parameters for customization. - -See also: @BenchmarkDotNet.Samples.IntroStatisticalTesting - -* [60eca0](https://github.com/dotnet/BenchmarkDotNet/commit/60eca005326970202a33891e5aecd2ef6b7e4cd0) Threshold API for WelchTTest; Improve Student accuracy for small n (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) -* [05cc8d](https://github.com/dotnet/BenchmarkDotNet/commit/05cc8d15ef88e382bbb1827d766d7275c3e42abd) Statistical testing improvements (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - -### ZeroMeasurementAnalyser - -When you have an empty benchmark like this - -```cs -[Benchmark] -public void Empty() { } -``` - -The expected duration of this method is zero. -However, you can get the mean value like `0.0023ns` because of the natural noise. -It's a pretty confusing result for many developers. -Since v0.11.2, we have `ZeroMeasurementAnalyser` which warn you about such methods. -By default, BenchmarkDotNet automatically evaluate overhead. -In this case, `ZeroMeasurementAnalyser` runs Welch's t-test and compare actual and overhead measurements. -If the overhead evaluation is disabled, it runs one-sample Student's t-test against a half of CPU cycle. - -* [#906](https://github.com/dotnet/BenchmarkDotNet/pull/906) Zero measurement analyser (by [@Rizzen](https://github.com/Rizzen)) -* [48d193](https://github.com/dotnet/BenchmarkDotNet/commit/48d193e30c780eb43e65b21f892c48db5dab6f6b) Zero measurement analyser (#906) (by [@Rizzen](https://github.com/Rizzen)) - -### RatioColumn - -The `Ratio` column was formerly known as `Scaled`. -The old title was a source of misunderstanding and confusion because - many developers interpreted it as the ratio of means (e.g., `50.46`/`100.39` for `Time50`). -The ratio of distribution means and the mean of the ratio distribution are pretty close to each other in most cases, - but they are not equal. - -See also: - @BenchmarkDotNet.Samples.IntroBenchmarkBaseline, - @BenchmarkDotNet.Samples.IntroRatioSD, - @docs.baselines. - -* [4e64c9](https://github.com/dotnet/BenchmarkDotNet/commit/4e64c94cfe7b49bbdc06aabb6ee1f262bd370862) Ratio/RatioSD columns (by [@AndreyAkinshin](https://github.com/AndreyAkinshin)) - diff --git a/docs/_changelog/header/v0.11.3.md b/docs/_changelog/header/v0.11.3.md deleted file mode 100644 index f3be5d81af..0000000000 --- a/docs/_changelog/header/v0.11.3.md +++ /dev/null @@ -1,99 +0,0 @@ - - -## Highlights - -This release is focused mainly on bug fixes that were affecting user experience. But don't worry, we have some new features too! - -* **Diagnosers** - * ConcurrencyVisualizerProfiler (allows profiling benchmarks on Windows and exporting the data to a trace file which can be opened with Concurrency Visualizer) -* **Command-line:** - * `--stopOnFirstError`: Stops the benchmarks execution on first error. [#947](https://github.com/dotnet/BenchmarkDotNet/pull/947) - * `--statisticalTest`: Performs a Mann–Whitney Statistical Test for identifying regressions and improvements. [#960](https://github.com/dotnet/BenchmarkDotNet/pull/960) -* **Bug fixes:** - * Dry mode doesn't work because of the ZeroMeasurementHelper [#943](https://github.com/dotnet/BenchmarkDotNet/issues/943) - * MannWhitneyTest fails when comparing statistics of different sample size [#948](https://github.com/dotnet/BenchmarkDotNet/issues/948) and [#950](https://github.com/dotnet/BenchmarkDotNet/issues/950) - * Improve the dynamic loading of Diagnostics package [#955](https://github.com/dotnet/BenchmarkDotNet/issues/955) - * BenchmarkRunner.RunUrl throws NRE when Config is not provided [#961](https://github.com/dotnet/BenchmarkDotNet/issues/961) - * Don't require the users to do manual installation of TraceEvent when using Diagnostics package [#962](https://github.com/dotnet/BenchmarkDotNet/issues/962) - * Stop benchmark after closing application + Flush log after stopping benchmark [#963](https://github.com/dotnet/BenchmarkDotNet/issues/963) - ---- - -## Diagnosers - -### ConcurrencyVisualizerProfiler - -`ConcurrencyVisualizerProfiler` allows to profile the benchmarked .NET code on Windows and exports the data to a CVTrace file which can be opened with [Concurrency Visualizer](https://learn.microsoft.com/visualstudio/profiling/concurrency-visualizer). - -`ConcurrencyVisualizerProfiler` uses `EtwProfiler` to get a `.etl` file which still can be opened with [PerfView](https://github.com/Microsoft/perfview) or [Windows Performance Analyzer](https://learn.microsoft.com/windows-hardware/test/wpt/windows-performance-analyzer). The difference is that it also enables all Task and Thread related ETW Providers and exports a simple `xml` which can be opened with Visual Studio if you install [Concurrency Visualizer plugin](https://marketplace.visualstudio.com/items?itemName=Diagnostics.ConcurrencyVisualizer2017) - -![open trace](https://user-images.githubusercontent.com/6011991/48638184-2b13fe00-e9d0-11e8-8a94-0e951e4606ae.png) - -![Utilization](https://user-images.githubusercontent.com/6011991/48638108-f6a04200-e9cf-11e8-8f7c-feda0a99138c.png) - -![Threads](https://user-images.githubusercontent.com/6011991/48638120-fb64f600-e9cf-11e8-9e2d-53b615bfe9a8.png) - - -* [#964](https://github.com/dotnet/BenchmarkDotNet/issues/964) Concurrency Visualizer Profiler Diagnoser (by [@adamsitnik](https://github.com/adamsitnik)) -* [dfb3c89](https://github.com/dotnet/BenchmarkDotNet/commit/dfb3c8912505799a76b0eb5ae0c082bb44599fa7) ConcurrencyVisualizerProfiler diagnoser! (by [@adamsitnik](https://github.com/adamsitnik)) - ---- - -## Command-line - -In this release, we have some new command-line arguments! - -### `--stopOnFirstError`: Stops the benchmarks execution on first error - -When provided, BenchmarkDotNet is going to stop the benchmarks execution on first error. - -* [#947](https://github.com/dotnet/BenchmarkDotNet/pull/947) Add option to stop running when the first benchmark fails (by [@wojtpl2](https://github.com/wojtpl2)) - -### `--statisticalTest`: Statistical Test - -To perform a Mann–Whitney U Test and display the results in a dedicated column you need to provide the Threshold via -`--statisticalTest`. Examples: 5%, 10ms, 100ns, 1s. - -Example: run Mann–Whitney U test with relative ratio of 1% for all benchmarks for .NET 4.6 (base), .NET Core 2.0 and .NET Core 2.1. - -```cs -class Program -{ - static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); -} - -public class MySample -{ - [Benchmark] - public void Sleep() - { -#if NETFRAMEWORK - Thread.Sleep(50); -#elif NETCOREAPP2_0 - Thread.Sleep(45); -#elif NETCOREAPP2_1 - Thread.Sleep(55); -#endif - } - - [Benchmark] - public void Same() => Thread.Sleep(50); -} -``` - -```log -dotnet run -c Release -f netcoreapp2.1 --filter * --runtimes net46 netcoreapp2.0 netcoreapp2.1 --statisticalTest 1% -``` - -**Note:** .NET 4.6 will be our baseline because it was provided as first on the runtimes list. - - -| Method | Runtime | Toolchain | Mean | Error | StdDev | Ratio | MannWhitney(1%) | -|------- |-------- |-------------- |---------:|----------:|----------:|------:|---------------- | -| Sleep | Clr | net46 | 50.51 ms | 0.1833 ms | 0.1714 ms | 1.00 | Base | -| Sleep | Core | netcoreapp2.0 | 45.53 ms | 0.1262 ms | 0.1181 ms | 0.90 | Faster | -| Sleep | Core | netcoreapp2.1 | 55.50 ms | 0.1217 ms | 0.1138 ms | 1.10 | Slower | -| | | | | | | | | -| Same | Clr | net46 | 50.47 ms | 0.1795 ms | 0.1679 ms | 1.00 | Base | -| Same | Core | netcoreapp2.0 | 50.55 ms | 0.1873 ms | 0.1752 ms | 1.00 | Same | -| Same | Core | netcoreapp2.1 | 50.55 ms | 0.2162 ms | 0.2022 ms | 1.00 | Same | diff --git a/docs/_changelog/header/v0.11.4.md b/docs/_changelog/header/v0.11.4.md deleted file mode 100644 index 3cf8b79f55..0000000000 --- a/docs/_changelog/header/v0.11.4.md +++ /dev/null @@ -1,114 +0,0 @@ -It's been few months since our last release, but we have been working hard and have some new features for you! - -## Highlights - -* **Features** - * **BenchmarkDotNet as a global tool**: a new global tool which allows you to run benchmarks from given library. - Now you can run benchmarks from the command line via `dotnet benchmark`. - You can find more information about it in the documentation - [#1006](https://github.com/dotnet/BenchmarkDotNet/pull/1006) - [#213](https://github.com/dotnet/BenchmarkDotNet/issues/213) - * **InProcessEmitToolchain**: new, full-featured InProcess toolchain which allows executing benchmarks in the current process - without spawning additional process per benchmark. - It supports `[Arguments]`, `[ArgumentsSource]`, passing the arguments by `out`, `ref` and returning stack-only types like `Span`. - [#919](https://github.com/dotnet/BenchmarkDotNet/issues/919), - [#921](https://github.com/dotnet/BenchmarkDotNet/pull/921) - [#843](https://github.com/dotnet/BenchmarkDotNet/issues/843) - * **ARM support**: BenchmarkDotNet supports now ARM and ARM64. - [#780](https://github.com/dotnet/BenchmarkDotNet/issues/780), - [#979](https://github.com/dotnet/BenchmarkDotNet/pull/979) - [#385](https://github.com/dotnet/BenchmarkDotNet/issues/385) - * **Mono AOT support**: a new toolchain which allows running benchmarks using AOT version of Mono - [#940](https://github.com/dotnet/BenchmarkDotNet/pull/940) - * **NuGet symbol server support**: BenchmarkDotNet publishes now the symbols to NuGet.org symbol server and you can easily debug it. - [#967](https://github.com/dotnet/BenchmarkDotNet/issues/967) - [#968](https://github.com/dotnet/BenchmarkDotNet/pull/968) - * **Experimental support for .NET Core 3.0 WPF benchmarks** - [#1066](https://github.com/dotnet/BenchmarkDotNet/pull/1066) - For a working example please go to [https://github.com/dotMorten/WPFBenchmarkTests](https://github.com/dotMorten/WPFBenchmarkTests) -* **Improvements:** - * CoreRT Toolchain improvements - thanks to help from CoreRT Team we were able to make - the CoreRT Toolchain work with the latest version of CoreRT - [#1001](https://github.com/dotnet/BenchmarkDotNet/pull/1001), - [#1057](https://github.com/dotnet/BenchmarkDotNet/pull/1057) - * Display the number of benchmarks to run: we now display how many benchmarks are going to be executed - before running them and how many remained after running each of them - [#1048](https://github.com/dotnet/BenchmarkDotNet/issues/1048) - * Better list of suggested benchmarks for wrong filter - [#834](https://github.com/dotnet/BenchmarkDotNet/issues/834) - [#957](https://github.com/dotnet/BenchmarkDotNet/pull/957) - * Invalid assembly binding redirects generated by VS were a pain to many of our users, - we have now implemented an approach that tries to work around this issue. - [#895](https://github.com/dotnet/BenchmarkDotNet/issues/895), - [#667](https://github.com/dotnet/BenchmarkDotNet/issues/667), - [#896](https://github.com/dotnet/BenchmarkDotNet/issues/896), - [#942](https://github.com/dotnet/BenchmarkDotNet/issues/942) - * Handling duplicates in IConfig - [#912](https://github.com/dotnet/BenchmarkDotNet/pull/912), - [#938](https://github.com/dotnet/BenchmarkDotNet/issues/938), - [#360](https://github.com/dotnet/BenchmarkDotNet/issues/360), - [#463](https://github.com/dotnet/BenchmarkDotNet/issues/463) - * Disassembly diagnoser should be kept in a separate directory to avoid dependency conflicts - [#1059](https://github.com/dotnet/BenchmarkDotNet/issues/1059) - * Give a warning when the `[Benchmark]` method is static - we now produce an error when users fail into this common issue - [#983](https://github.com/dotnet/BenchmarkDotNet/issues/983) - [#985](https://github.com/dotnet/BenchmarkDotNet/pull/985) - * C# keywords are prohibited as benchmark names - [#849](https://github.com/dotnet/BenchmarkDotNet/issues/849) - * File names should be consistent across all OSes - `<` and `>` are valid on Unix, but not on Windows. - We have unified that and now files produced on Unix and Windows have the same names. - [#981](https://github.com/dotnet/BenchmarkDotNet/issues/981) - * Improve restore, build and publish projects - [#1002](https://github.com/dotnet/BenchmarkDotNet/issues/1002), - [#1013](https://github.com/dotnet/BenchmarkDotNet/pull/1013) - * Make it possible to disable OptimizationsValidator - [#988](https://github.com/dotnet/BenchmarkDotNet/issues/988) - * Sort enum parameters by value instead of name - [#977](https://github.com/dotnet/BenchmarkDotNet/pull/977) - * Detect .NET Core benchmark failures from LINQPad - [#980](https://github.com/dotnet/BenchmarkDotNet/pull/980) - * Improved error logging - [#1008](https://github.com/dotnet/BenchmarkDotNet/pull/1008) - * Improved disassembly diff - [#1022](https://github.com/dotnet/BenchmarkDotNet/pull/1022) - * Using invariant culture for Roslyn Toolchain error messages - [#1042](https://github.com/dotnet/BenchmarkDotNet/pull/1042) - * Use only full names in the auto-generated code to avoid any possible conflicts with user code - [#1007](https://github.com/dotnet/BenchmarkDotNet/issues/1007), - [#1009](https://github.com/dotnet/BenchmarkDotNet/pull/1009) - [#1010](https://github.com/dotnet/BenchmarkDotNet/issues/1010) - * Write the GitHub table format to the console by default - [#1062](https://github.com/dotnet/BenchmarkDotNet/issues/1062) - * Proper cleanup on Ctrl+C/console Window exit - [#1061](https://github.com/dotnet/BenchmarkDotNet/pull/1061) - * Introduce StoppingCriteria - the first step to writing your own heuristic that determines when benchmarking should be stopped - [#984](https://github.com/dotnet/BenchmarkDotNet/pull/984) -* **Breaking changes:** - * .NET Standard 2.0 only - BenchmarkDotNet has a single target now, which should help with some assembly resolving issues. - We had to drop .NET 4.6 support because of that and .NET 4.6.1 is now the oldest supported .NET Framework. - [#1032](https://github.com/dotnet/BenchmarkDotNet/pull/1032) - * CustomCoreClrToolchain has been removed, it's recommended to use CoreRunToolchain instead - [#928](https://github.com/dotnet/BenchmarkDotNet/issues/928) -* **Bug fixes:** - * NRE in `Summary` ctor - [#986](https://github.com/dotnet/BenchmarkDotNet/issues/986) - [#987](https://github.com/dotnet/BenchmarkDotNet/pull/987) - * ArgumentNullException when running benchmarks from published .NET Core app - [#1018](https://github.com/dotnet/BenchmarkDotNet/issues/1018) - * Dry jobs can eat iteration failures - [#1045](https://github.com/dotnet/BenchmarkDotNet/issues/1045) - * NullReferenceException in BenchmarkDotNet.Reports.SummaryTable after iteration failure - [#1046](https://github.com/dotnet/BenchmarkDotNet/issues/1046) - * Running the example throws NullReference - [#1049](https://github.com/dotnet/BenchmarkDotNet/issues/1049) - * Fix race condition in process output reader - [#1051](https://github.com/dotnet/BenchmarkDotNet/issues/1051) - [#1053](https://github.com/dotnet/BenchmarkDotNet/pull/1053) - * Fix a rare but really annoying bug where for some reason we were sometimes setting ForegroundColor - to the same color as BackgroundColor and some parts of the logged output were invisible - [commit](https://github.com/dotnet/BenchmarkDotNet/commit/ea3036810ef60b483d766a097e6f3edfde28a834) - * StopOnFirstError must be respected - [commit](https://github.com/dotnet/BenchmarkDotNet/commit/87d281d7dbf52036819efff52e6661e436648b73) - ---- - diff --git a/docs/_changelog/header/v0.11.5.md b/docs/_changelog/header/v0.11.5.md deleted file mode 100644 index 98e31c0195..0000000000 --- a/docs/_changelog/header/v0.11.5.md +++ /dev/null @@ -1,170 +0,0 @@ -## Highlights - -* **Features and noticeable improvements** - * **Power plan management** - Now BenchmarkDotNet executes all benchmarks with enabled High-Performance power plan (configurable, Windows-only). - You can find some details below. - [#68](https://github.com/dotnet/BenchmarkDotNet/issues/68) - [#952](https://github.com/dotnet/BenchmarkDotNet/pull/952) - * **Better Environment Variables API** - Now we have some additional extension methods which allow defining environment variables in user jobs. - In the previous version, users always had to set an array of environment variables like this: - `job.With(new[] { new EnvironmentVariable("a", "b") })`. - Now it's possible to define an environment variable like - `job.With(new EnvironmentVariable("a", "b"))` or - `job.WithEnvironmentVariable("a", "b")`. - Also, it's possible to clear the list of environment variables via - `job.WithoutEnvironmentVariables()`. - [#1069](https://github.com/dotnet/BenchmarkDotNet/issues/1069) - [#1080](https://github.com/dotnet/BenchmarkDotNet/pull/1080) - * **Better outlier messages** - The previous version of BenchmarkDotNet printed information about detected or removed outliers like this: - "3 outliers were detected". - It was nice, but it didn't provide additional information about these outliers - (users had to read the full log to find the outliers values). - Now BenchmarkDotNet prints additional information about outlier values like this: - "3 outliers were detected (2.50 us..2.70 us)". - [e983cd31](https://github.com/dotnet/BenchmarkDotNet/commit/e983cd3126e64f82fe59bc1bc45d1a870a615e87) - * **Support modern CPU architecture names** - In the environment information section, BenchmarkDotNet prints not only the processor brand string, but also its architecture - (e.g., "Intel Core i7-4770K CPU 3.50GHz (Haswell)"). - However, it failed to recognize some recent processors. - Now it's able to detect the architecture for modern Intel processors correctly - (Kaby Lake, Kaby Lake R, Kaby Lake G, Amber Lake Y, Coffee Lake, Cannon Lake, Whiskey Lake). - [995e053d](https://github.com/dotnet/BenchmarkDotNet/commit/995e053d14a61cdadc417149480f23ebf679bcb7) - * **Introduce BenchmarkDotNet.Annotations** - Currently, BenchmarkDotNet targets .NET Standard 2.0. - It makes some users unhappy because they want to define benchmarks in projects with lower target framework. - We decided to start working on the `BenchmarkDotNet.Annotations` NuGet package which targets .NET Standard 1.0 - and contains classes that users need to define their benchmarks. - However, it's not easy to refactor the full source code base and move all relevant public APIs to this package. - In v0.11.5, we did the first step and moved some of these APIs to `BenchmarkDotNet.Annotations`. - We want to continue moving classes to this package and get full-featured annotation package in the future. - [#1084](https://github.com/dotnet/BenchmarkDotNet/pull/1084) - [#1096](https://github.com/dotnet/BenchmarkDotNet/pull/1096) - * **Use InProcessEmitToolchain by default in InProcess benchmarks** - In BenchmarkDotNet 0.11.4, we introduced `InProcessEmitToolchain`. - It's a new, full-featured InProcess toolchain which allows executing benchmarks in the current process - without spawning additional process per benchmark. - It supports `[Arguments]`, `[ArgumentsSource]`, passing the arguments by `out`, `ref` and returning stack-only types like `Span`. - However, in v0.11.4, it can be activated only if `InProcessEmitToolchain` is declared explicitly. - Now it's enabled by default when `[InProcessAttribute]` is used. - [#1093](https://github.com/dotnet/BenchmarkDotNet/pull/1093) - * **Introduce an option which prevents overwriting results** - Currently, BenchmarkDotNet overwrites results each time when the benchmarks are executed. - It allows avoiding tons of obsolete files in the `BenchmarkDotNet.Artifacts` folder. - However, the behavior doesn't fit all use cases: - sometimes users want to keep results for old benchmark runs. - Now we have a special option for it. - The option can be activated via `--noOverwrite` console line argument or - `DontOverwriteResults` extension method for `IConfig` - [#1074](https://github.com/dotnet/BenchmarkDotNet/issues/1074) - [#1083](https://github.com/dotnet/BenchmarkDotNet/pull/1083) -* **Other improvements and bug fixes** - * **Diagnostics and validation** - * **Better benchmark declaration error processing** - In the previous version, BenchmarkDotNet threw an exception when some benchmark methods had an invalid declaration - (e.g., invalid signature or invalid access modifiers). - Now it prints a nice error message without ugly stack traces. - [#1107](https://github.com/dotnet/BenchmarkDotNet/issues/1107) - * **Better error message for users who want to debug benchmarks** - [#1073](https://github.com/dotnet/BenchmarkDotNet/issues/1073) - * **Don't show the same validation error multiple times** - Now each error will be printed only once. - [#1079](https://github.com/dotnet/BenchmarkDotNet/issues/1079) - * **Restrict MemoryDiagnoserAttribute usage to class** - Now it's impossible to accidentally mark a method with this attribute. - [#1119](https://github.com/dotnet/BenchmarkDotNet/issues/1119) - [#1122](https://github.com/dotnet/BenchmarkDotNet/pull/1122) - * **Export** - * **Better indentation in disassembly listings** - Now DissassemblyDiagnoser correctly process source code which contains tab as the indentation symbol - [#1110](https://github.com/dotnet/BenchmarkDotNet/issues/1110) - * **Fix incorrect indentation for StackOverflow exporter** - Previously, StackOverflow exporter doesn't have a proper indent for job runtimes in the environment information. - Now it's fixed. - [#826](https://github.com/dotnet/BenchmarkDotNet/issues/826) - [#1104](https://github.com/dotnet/BenchmarkDotNet/pull/1104) - * **Fix StackOverflowException in XmlExporter.Full** - [#1086](https://github.com/dotnet/BenchmarkDotNet/issues/1086) - [#1090](https://github.com/dotnet/BenchmarkDotNet/pull/1090) - * **Shortify MemoryDiagnoser column titles** - Now we use the following column titles: - "Allocated" instead of "Allocated Memory/Op", - "Gen 0" instead of "Gen 0/1k Op". - The full description of each column can be found in the legend section below the summary table. - [#1081](https://github.com/dotnet/BenchmarkDotNet/pull/1081) - * **Benchmark generation and execution** - * **Fixed broken Orderers** - The previous version has a nasty bug with custom orderers. - Now it's fixed. - [#1070](https://github.com/dotnet/BenchmarkDotNet/issues/1070) - [#1109](https://github.com/dotnet/BenchmarkDotNet/issues/1109) - * **Better overhead evaluation** - In the previous version, BenchmarkDotNet evaluated the benchmark overhead as a mean value of all overhead iteration. - It was fine in most cases, but in some cases, the mean value can be spoiled by outliers. - Now BenchmarkDotNet uses the median value. - [#1116](https://github.com/dotnet/BenchmarkDotNet/issues/1116) - * **Respect CopyLocalLockFileAssemblies** - Now BenchmarkDotNet respect the `CopyLocalLockFileAssemblies` value and copies it - to the generated benchmark project. - [#1068](https://github.com/dotnet/BenchmarkDotNet/issues/1068) - [#1108](https://github.com/dotnet/BenchmarkDotNet/pull/1108) - * **Disable CodeAnalysisRuleSet for generated benchmarks** - Previously, generated benchmarks may fail if the `CodeAnalysisRuleSet` is defined in `Directory.Build.Props`. - [#1082](https://github.com/dotnet/BenchmarkDotNet/pull/1082) - * **Supported undefined enum values** - [#1020](https://github.com/dotnet/BenchmarkDotNet/issues/1020) - [#1071](https://github.com/dotnet/BenchmarkDotNet/issues/1071) - * **Other minor improvements and bug fixes** - -## Power plans - -In [#952](https://github.com/dotnet/BenchmarkDotNet/pull/952), power plan management was implemented. -It resolves a pretty old issue [#68](https://github.com/dotnet/BenchmarkDotNet/issues/68) which was created more than three years ago. -Now BenchmarkDotNet forces OS to execute a benchmark on the High-Performance power plan. -You can disable this feature by modifying PowerPlanMode property. -Here is an example where we are playing with this value: - -```cs -[Config(typeof(Config))] -public class IntroPowerPlan -{ - private class Config : ManualConfig - { - public Config() - { - Add(Job.MediumRun.WithPowerPlan(PowerPlan.HighPerformance)); - Add(Job.MediumRun.WithPowerPlan(PowerPlan.UserPowerPlan)); - } - } - - [Benchmark] - public int IterationTest() - { - int j = 0; - for (int i = 0; i < short.MaxValue; ++i) - j = i; - return j; - } - - [Benchmark] - public int SplitJoin() - => string.Join(",", new string[1000]).Split(',').Length; -} -``` - -And here is an example of the summary table on plugged-off laptop: - -```md - Method | PowerPlan | Mean | Error | StdDev | --------------- |---------------- |---------:|----------:|----------:| - IterationTest | HighPerformance | 40.80 us | 0.4168 us | 0.6109 us | - SplitJoin | HighPerformance | 13.24 us | 0.2514 us | 0.3763 us | - IterationTest | UserPowerPlan | 79.72 us | 2.5623 us | 3.8352 us | - SplitJoin | UserPowerPlan | 24.54 us | 2.1062 us | 3.1525 us | -``` - -As you can see, the power plan produces a noticeable effect on the benchmark results. - -This feature is available on Windows only. \ No newline at end of file diff --git a/docs/_changelog/header/v0.12.0.md b/docs/_changelog/header/v0.12.0.md deleted file mode 100644 index 0e6a00582f..0000000000 --- a/docs/_changelog/header/v0.12.0.md +++ /dev/null @@ -1,469 +0,0 @@ -It's been several months since our last release, but we have been working hard and have some new features for you! - -## Highlights - -* **Features and major improvements** - * **Advanced multiple target frameworks support** - Now BenchmarkDotNet supports .NET Framework 4.8, .NET Core 3.1, and .NET Core 5.0. - Also, we reworked our API that allows targeting several runtimes from the same config: - the new API is more consistent, flexible, and powerful. - For example, if you want to execute your benchmarking using .NET Framework 4.8 and .NET Core 3.1, - you can use the `SimpleJob(RuntimeMoniker.Net48)`, `[SimpleJob(RuntimeMoniker.NetCoreApp31)]` attributes or - `Job.Default.With(ClrRuntime.Net48)`, `Job.Default.With(CoreRuntime.Core31)` jobs in a manual config. - You can find more details below. - [#1188](https://github.com/dotnet/BenchmarkDotNet/pull/1188), - [#1186](https://github.com/dotnet/BenchmarkDotNet/issues/1186), - [#1236](https://github.com/dotnet/BenchmarkDotNet/issues/1236) - * **Official templates for BenchmarkDotNet-based projects** - With the help of the [BenchmarkDotNet.Templates](https://www.nuget.org/packages/BenchmarkDotNet.Templates/) NuGet package, - you can easily create new projects from the command line via `dotnet new benchmark`. - This command has a lot of useful options, so you can customize your new project as you want. - [#1044](https://github.com/dotnet/BenchmarkDotNet/pull/1044) - * **New NativeMemoryProfiler** - `NativeMemoryProfiler` measure the native memory traffic and adds the extra columns `Allocated native memory` and `Native memory leak` to the summary table. - Internally, it uses `EtwProfiler` to profile the code using ETW. - [#457](https://github.com/dotnet/BenchmarkDotNet/issues/457), - [#1131](https://github.com/dotnet/BenchmarkDotNet/pull/1131), - [#1208](https://github.com/dotnet/BenchmarkDotNet/pull/1208), - [#1214](https://github.com/dotnet/BenchmarkDotNet/pull/1214), - [#1218](https://github.com/dotnet/BenchmarkDotNet/pull/1218), - [#1219](https://github.com/dotnet/BenchmarkDotNet/pull/1219) - * **New ThreadingDiagnoser** - `ThreadingDiagnoser` also adds two extra columns to the summary table: - `Completed Work Items` (the number of work items that have been processed in ThreadPool per single operation) and - `Lock Contentions` (the number of times there *was contention* upon trying to take a Monitor's lock per single operation). - Internally, it uses [new APIs](https://github.com/dotnet/corefx/issues/35500) exposed in .NET Core 3.0. - [#1154](https://github.com/dotnet/BenchmarkDotNet/issues/1154), - [#1227](https://github.com/dotnet/BenchmarkDotNet/pull/1227) - * **Improved MemoryDiagnoser** - Now `MemoryDiagnoser` includes memory allocated by *all threads* that were live during benchmark execution: a new GC API was exposed in .NET Core 3.0 preview6+. - It allows to get the number of allocated bytes for all threads. - [#1155](https://github.com/dotnet/BenchmarkDotNet/pull/1155), - [#1153](https://github.com/dotnet/BenchmarkDotNet/issues/1153), - [#723](https://github.com/dotnet/BenchmarkDotNet/issues/723) - * **LINQPad 6 support** - Now both LINQPad 5 and LINQPad 6 are supported! - [#1241](https://github.com/dotnet/BenchmarkDotNet/issues/1241), - [#1245](https://github.com/dotnet/BenchmarkDotNet/pull/1245) - * **Fast documentation search** - We continue to improve the usability of our documentation. - In this release, we improved the search experience in the documentation: now it works almost instantly with the help of [Algolia](https://www.algolia.com/) engine! - [#1148](https://github.com/dotnet/BenchmarkDotNet/pull/1148), - [#1158](https://github.com/dotnet/BenchmarkDotNet/issues/1158) -* **Minor summary and exporter improvements** - * **Improved presentation of the current architecture in the environment information** - In the previous version of BenchmarkDotNet, the reports always contained "64bit" or "32bit" which did not tell if it was ARM or not. - Now it prints the full architecture name (`x64`, `x86`, `ARM`, or `ARM64`). - For example, instead of `.NET Framework 4.8 (4.8.3815.0), 64bit RyuJIT` you will get `.NET Framework 4.8 (4.8.3815.0), X64 RyuJIT` or `.NET Framework 4.8 (4.8.3815.0), ARM64 RyuJIT`. - [#1213](https://github.com/dotnet/BenchmarkDotNet/pull/1213) - * **Simplified reports for Full .NET Framework version** - Previous version: `.NET Framework 4.7.2 (CLR 4.0.30319.42000), 64bit RyuJIT-v4.7.3324.0`. - Current version: `.NET Framework 4.7.2 (4.7.3362.0), 64bit RyuJIT`. - [#1114](https://github.com/dotnet/BenchmarkDotNet/pull/1114), - [#1111](https://github.com/dotnet/BenchmarkDotNet/issues/1111) - * **More reliable CPU info on Windows** - We added a workaround to for a bug in wmic that uses `\r\r\n` as a line separator. - [#1144](https://github.com/dotnet/BenchmarkDotNet/issues/1144), - [#1145](https://github.com/dotnet/BenchmarkDotNet/pull/1145) - * **Better naming for generated plots** - When `[RPlotExporter]` is used, BenchmarkDotNet generates a lot of useful plots in the `BenchmarkDotNet.Artifacts` folder. - The naming of the plot files was improved: benchmarks without `Params` doesn't include a double dash (`--`) in their names anymore. - [1183](https://github.com/dotnet/BenchmarkDotNet/issues/1183), - [1212](https://github.com/dotnet/BenchmarkDotNet/pull/1212) - * **Better density plot precision** - The previous version of BenchmarkDotNet used the rule-of-thumb bandwidth selector in `RPlotExporter` density plots. - It was fine for unimodal distributions, but sometimes it produced misleading plots for multimodal distributions. - Now, RPlotExporter uses the Sheather&Jones bandwidth selector that significantly improves the presentation of the density plots for complex distributions. - [58fde64](https://github.com/dotnet/BenchmarkDotNet/commit/58fde64c809ceadb3fca9d677a7cec83071b833f) - * **Better alignment in `HtmlExporter`** - Now BenchmarkDotNet aligns the content exported by `HtmlExporter` to the right. - [#1189](https://github.com/dotnet/BenchmarkDotNet/pull/1189) - [dfa074](https://github.com/dotnet/BenchmarkDotNet/commit/dfa074b024cfa8bbe4fe175d3d65a3d9f85127ff) - * **Better precision calculation in SummaryTable** - [4e9eb43](https://github.com/dotnet/BenchmarkDotNet/commit/4e9eb4335eee05a95a3766f2c81ae260508021af) - * **Better summary analysis** - BenchmarkDotNet warns the user when benchmark baseline value is too close to zero and the columns derived from BaselineCustomColumn cannot be computed. - [#1161](https://github.com/dotnet/BenchmarkDotNet/pull/1161), - [#600](https://github.com/dotnet/BenchmarkDotNet/issues/600) - * **Make log file datetime format 24-hour** - [#1149](https://github.com/dotnet/BenchmarkDotNet/issues/1149) - * **Improve AskUser prompt message** - The error messages will surround `*` by quotes on Linux and macOS. - [#1147](https://github.com/dotnet/BenchmarkDotNet/issues/1147) -* **Minor API improvements** - * **ED-PELT algorithm for changepoint detection is now available** - You can find details in [this blog post](https://aakinshin.net/posts/edpelt/). - [f89091](https://github.com/dotnet/BenchmarkDotNet/commit/f89091a2a9c1a4058dd8e32d5aa01271910dd7dc) - * **Improved OutlierMode API** - BenchmarkDotNet performs measurement postprocessing that may remove some of the outlier values (it can be useful to remove upper outliers that we get because of the natural CPU noise). - In the previous version, naming for the `OutlierMode` values was pretty confusing: `None/OnlyUpper/OnlyLower/All`. - Now, these values were renamed to `DontRemove/RemoveUpper/RemoveLower/RemoveAll`. - For example, if you want to remove all the outliers, you can annotate your benchmark with the `[Outliers(OutlierMode.RemoveAll)]` attribute. - The old names still exist (to make sure that the changes are backward compatible), but they are marked as obsolete, and they will be removed in the future versions of the library. - [#1199](https://github.com/dotnet/BenchmarkDotNet/pull/1199), - [0e4b8e](https://github.com/dotnet/BenchmarkDotNet/commit/0e4b8e69d10e73a83fce3ce3980497ee7798bc87) - * **Add the possibility to pass `Config` to `BenchmarkSwitcher.RunAll` and `RunAllJoined`** - [#1194](https://github.com/dotnet/BenchmarkDotNet/issues/1194), - [ae23bd](https://github.com/dotnet/BenchmarkDotNet/commit/ae23bddebe49cd66a9627790c073b7bc45ccbf5c) - * **Improved command line experience** - When user uses `--packages $path`, the `$path` will be sent to the dotnet build command as well. - [1187](https://github.com/dotnet/BenchmarkDotNet/issues/1187) - * **Extend the list of supported power plans.** - Now it supports "ultimate", "balanced", and "power saver" plans. - [#1132](https://github.com/dotnet/BenchmarkDotNet/issues/1132), - [#1139](https://github.com/dotnet/BenchmarkDotNet/pull/1139) - * **Make it possible to not enforce power plan on Windows.** - [1578c5c](https://github.com/dotnet/BenchmarkDotNet/commit/1578c5c60c3f59f9128f680e35d1db219aa60d8d) - * **`Guid` support in benchmark arguments** - Now you can use `Guid` instances as benchmark arguments. - [04ec20b](https://github.com/dotnet/BenchmarkDotNet/commit/04ec20b5e0c0a514e8d158684864e4f9934ae8cc) - * **Make `ArgumentsSource` support `IEnumerable` for benchmarks accepting a single argument to mimic `MemberData` behaviour.** - [ec296dc](https://github.com/dotnet/BenchmarkDotNet/commit/ec296dc45de798f7407852d5ab7febe2b457eca4) - * **Make `FullNameProvider` public** - So it can be reused by the `dotnet/performance` repository. - [6d71308](https://github.com/dotnet/BenchmarkDotNet/commit/6d71308f4c1d96bcf8a526bfcc61bb23307b4041) - * **Extend `Summary` with `LogFilePath`** - [#1135](https://github.com/dotnet/BenchmarkDotNet/issues/1135), - [6e6559](https://github.com/dotnet/BenchmarkDotNet/commit/6e6559499652c312369fd223d0cf4747f65512d6) - * **Allow namespace filtering for `InliningDiagnoser`** - [#1106](https://github.com/dotnet/BenchmarkDotNet/issues/1106), - [#1130](https://github.com/dotnet/BenchmarkDotNet/pull/1130) - * **Option to configure `MaxParameterColumnWidth`** - [#1269](https://github.com/dotnet/BenchmarkDotNet/issues/1269), - [4ec888](https://github.com/dotnet/BenchmarkDotNet/commit/4ec88844547507474ccd0303b31f935b3463318c) -* **Other improvements** - * **Misc improvements in the documentation** - [#1175](https://github.com/dotnet/BenchmarkDotNet/pull/1175), - [#1173](https://github.com/dotnet/BenchmarkDotNet/pull/1173), - [#1180](https://github.com/dotnet/BenchmarkDotNet/pull/1180), - [#1203](https://github.com/dotnet/BenchmarkDotNet/pull/1203), - [#1204](https://github.com/dotnet/BenchmarkDotNet/pull/1204), - [#1206](https://github.com/dotnet/BenchmarkDotNet/pull/1206), - [#1209](https://github.com/dotnet/BenchmarkDotNet/pull/1209), - [#1219](https://github.com/dotnet/BenchmarkDotNet/pull/1219), - [#1225](https://github.com/dotnet/BenchmarkDotNet/pull/1225), - [#1279](https://github.com/dotnet/BenchmarkDotNet/pull/1279) - * **Copy `PreserveCompilationContext` MSBuild setting from the project that defines benchmarks** - [#1152](https://github.com/dotnet/BenchmarkDotNet/issues/1152), - [063d1a](https://github.com/dotnet/BenchmarkDotNet/commit/063d1a56152fd5812cb6e9dd5095dc6e647e6938) - * **Add `System.Buffers.ArrayPoolEventSource` to the list of default .NET Providers of `EtwProfiler`** - [#1179](https://github.com/dotnet/BenchmarkDotNet/issues/1179), - [a106b1](https://github.com/dotnet/BenchmarkDotNet/commit/a106b114b1f04fa1024be84a8969f5a168fa1c8b) - * **Consume CoreRT from the new NuGet feed** - Because CoreRT no longer publishes to MyGet. - [#1129](https://github.com/dotnet/BenchmarkDotNet/pull/1129) -* **Breaking changes:** - * The `[ClrJob]`, `[CoreJob]` and `[CoreRtJob]` attributes got obsoleted and replaced by a `[SimpleJob]` which requires the user to provide target framework moniker in an explicit way. - (See the "Advanced multiple target frameworks support" section for details.) - [#1188](https://github.com/dotnet/BenchmarkDotNet/pull/1188), - [#1182](https://github.com/dotnet/BenchmarkDotNet/issues/1182), - [#1115](https://github.com/dotnet/BenchmarkDotNet/issues/1115), - [#1056](https://github.com/dotnet/BenchmarkDotNet/issues/1056), - [#993](https://github.com/dotnet/BenchmarkDotNet/issues/993), - * The old `InProcessToolchain` is now obsolete. It's recommended to use `InProcessEmitToolchain`. If you want to use the old one on purpose, you have to use `InProcessNoEmitToolchain`. - [#1123](https://github.com/dotnet/BenchmarkDotNet/pull/1123) -* **Bug fixes:** - * Invalid arg passing in StreamLogger constructor. The `append` arg was not passed to the `StreamWriter` .ctor. - [#1185](https://github.com/dotnet/BenchmarkDotNet/pull/1185) - * Improve the output path of `.etl` files produced by `EtwProfiler`. `EtwProfiler` was throwing NRE for users who were using `[ClrJob]` and `[CoreJob]` attributes. - [#1156](https://github.com/dotnet/BenchmarkDotNet/issues/1156), - [#1072](https://github.com/dotnet/BenchmarkDotNet/issues/1072) - * Flush custom loggers at the end of benchmark session. - [#1134](https://github.com/dotnet/BenchmarkDotNet/issues/1134) - * Make ids for tag columns unique - when using multiple `TagColumns` only one `TagColumn` was printed in the results. - [#1146](https://github.com/dotnet/BenchmarkDotNet/issues/1146) - -## Advanced multiple target frameworks support - -Now BenchmarkDotNet supports .NET Framework 4.8, .NET Core 3.1, and .NET Core 5.0. -Also, we reworked our API that allows targeting several runtimes from the same config: - the new API is more consistent, flexible, and powerful. -For example, if you want to execute your benchmarking using .NET Framework 4.8 and .NET Core 3.1, - you can use the `SimpleJob(RuntimeMoniker.Net48)`, `[SimpleJob(RuntimeMoniker.NetCoreApp31)]` attributes or - `Job.Default.With(ClrRuntime.Net48)`, `Job.Default.With(CoreRuntime.Core31)` jobs in a manual config. - -Now let's discuss how to use it in detail. -If you want to test multiple frameworks, your project file **MUST target all of them** and you **MUST install the corresponding SDKs**: - -```xml -netcoreapp3.0;netcoreapp2.1;net48 -``` - -If you run your benchmarks without specifying any custom settings, BenchmarkDotNet is going to run the benchmarks **using the same framework as the host process** (it corresponds to `RuntimeMoniker.HostProcess`): - -```cmd -dotnet run -c Release -f netcoreapp2.1 # is going to run the benchmarks using .NET Core 2.1 -dotnet run -c Release -f netcoreapp3.0 # is going to run the benchmarks using .NET Core 3.0 -dotnet run -c Release -f net48 # is going to run the benchmarks using .NET 4.8 -mono $pathToExe # is going to run the benchmarks using Mono from your PATH -``` - -To run the benchmarks for multiple runtimes with a single command from the command line, you need to specify the runtime moniker names via `--runtimes|-r` console argument: - -```cmd -dotnet run -c Release -f netcoreapp2.1 --runtimes netcoreapp2.1 netcoreapp3.0 # is going to run the benchmarks using .NET Core 2.1 and .NET Core 3.0 -dotnet run -c Release -f netcoreapp2.1 --runtimes netcoreapp2.1 net48 # is going to run the benchmarks using .NET Core 2.1 and .NET 4.8 -``` - -What is going to happen if you provide multiple Full .NET Framework monikers? -Let's say: - -```cmd -dotnet run -c Release -f net461 net472 net48 -``` - -Full .NET Framework always runs every .NET executable using the latest .NET Framework available on a given machine. -If you try to run the benchmarks for a few .NET TFMs, they are all going to be executed using the latest .NET Framework from your machine. -The only difference is that they are all going to have different features enabled depending on the target version they were compiled for. -You can read more about this - [here](https://learn.microsoft.com/dotnet/framework/migration-guide/version-compatibility) and - [here](https://learn.microsoft.com/dotnet/framework/migration-guide/application-compatibility). -This is **.NET Framework behavior which can not be controlled by BenchmarkDotNet or any other tool**. - -**Note:** Console arguments support works only if you pass the `args` to `BenchmarkSwitcher`: - -```cs -class Program -{ - static void Main(string[] args) - => BenchmarkSwitcher - .FromAssembly(typeof(Program).Assembly) - .Run(args); // crucial to make it work -} -``` - -You can achieve the same thing using `[SimpleJobAttribute]`: - -```cs -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Jobs; - -namespace BenchmarkDotNet.Samples -{ - [SimpleJob(RuntimeMoniker.Net48)] - [SimpleJob(RuntimeMoniker.Mono)] - [SimpleJob(RuntimeMoniker.NetCoreApp21)] - [SimpleJob(RuntimeMoniker.NetCoreApp30)] - public class TheClassWithBenchmarks -``` - -Or using a custom config: - -```cs -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Running; - -namespace BenchmarkDotNet.Samples -{ - class Program - { - static void Main(string[] args) - { - var config = DefaultConfig.Instance - .With(Job.Default.With(CoreRuntime.Core21)) - .With(Job.Default.With(CoreRuntime.Core30)) - .With(Job.Default.With(ClrRuntime.Net48)) - .With(Job.Default.With(MonoRuntime.Default)); - - BenchmarkSwitcher - .FromAssembly(typeof(Program).Assembly) - .Run(args, config); - } - } -} -``` - -The recommended way of running the benchmarks for multiple runtimes is to use the `--runtimes` console line argument. -By using the console line argument, you don't need to edit the source code anytime you want to change the list of runtimes. -Moreover, if you share the source code of the benchmark, other people can run it even if they don't have the exact same framework version installed. - -## Official templates for BenchmarkDotNet-based projects - -Since v0.12.0, BenchmarkDotNet provides project templates to setup your benchmarks easily. -The template exists for each major .NET language ([C#](https://learn.microsoft.com/dotnet/csharp/), [F#](https://learn.microsoft.com/dotnet/fsharp/) and [VB](https://learn.microsoft.com/dotnet/visual-basic/)) with equivalent features and structure. -The templates require the [.NET Core SDK](https://www.microsoft.com/net/download). Once installed, run the following command to install the templates: - -```log -dotnet new -i BenchmarkDotNet.Templates -``` - -If you want to uninstall all BenchmarkDotNet templates: - -```log -dotnet new -u BenchmarkDotNet.Templates -``` - -The template is a NuGet package distributed over nuget.org: [BenchmarkDotNet.Templates](https://www.nuget.org/packages/BenchmarkDotNet.Templates/). -To create a new C# benchmark library project from the template, run: - -```log -dotnet new benchmark -``` - -If you'd like to create F# or VB project, you can specify project language with `-lang` option: - -```log -dotnet new benchmark -lang F# -dotnet new benchmark -lang VB -``` - -The template projects have five additional options - all of them are optional. -By default, a class library project targeting netstandard2.0 is created. -You can specify `-f` or `--frameworks` to change target to one or more frameworks: - -```log -dotnet new benchmark -f netstandard2.0;net472 -``` - -The option `--console-app` creates a console app project targeting `netcoreapp3.0` with an entry point: - -```log -dotnet new benchmark --console-app -``` - -This lets you run the benchmarks from a console (`dotnet run`) or from your favorite IDE. -The option `-f` or `--frameworks` will be ignored when `--console-app` is set. -The option `-b` or `--benchmarkName` sets the name of the benchmark class: - -```log -dotnet new benchmark -b Md5VsSha256 -``` - -BenchmarkDotNet lets you create a dedicated configuration class (see [Configs](xref:docs.configs)) to customize the execution of your benchmarks. -To create a benchmark project with a configuration class, use the option `-c` or `--config`: - -```log -dotnet new benchmark -c -``` - -The option `--no-restore` if specified, skips the automatic NuGet restore after the project is created: - -```log -dotnet new benchmark --no-restore -``` - -Use the `-h` or `--help` option to display all possible arguments with a description and the default values: - -```log -dotnet new benchmark --help -``` - -The version of the template NuGet package is synced with the [BenchmarkDotNet](https://www.nuget.org/packages/BenchmarkDotNet/) package. -For instance, the template version `0.12.0` is referencing [BenchmarkDotnet 0.12.0](https://www.nuget.org/packages/BenchmarkDotNet/0.12.0) - there is no floating version behavior. -For more info about the `dotnet new` CLI, please read [the documentation](https://learn.microsoft.com/dotnet/core/tools/dotnet). - -## New NativeMemoryProfiler - -`NativeMemoryProfiler` measure the native memory traffic and adds the extra columns `Allocated native memory` and `Native memory leak` to the summary table. -Internally, it uses `EtwProfiler` to profile the code using ETW. - -Consider the following benchmark: - -```cs -[ShortRunJob] -[NativeMemoryProfiler] -[MemoryDiagnoser] -public class IntroNativeMemory -{ - [Benchmark] - public void BitmapWithLeaks() - { - var flag = new Bitmap(200, 100); - var graphics = Graphics.FromImage(flag); - var blackPen = new Pen(Color.Black, 3); - graphics.DrawLine(blackPen, 100, 100, 500, 100); - } - - [Benchmark] - public void Bitmap() - { - using (var flag = new Bitmap(200, 100)) - { - using (var graphics = Graphics.FromImage(flag)) - { - using (var blackPen = new Pen(Color.Black, 3)) - { - graphics.DrawLine(blackPen, 100, 100, 500, 100); - } - } - } - } - - private const int Size = 20; // Greater value could cause System.OutOfMemoryException for test with memory leaks. - private int ArraySize = Size * Marshal.SizeOf(typeof(int)); - - [Benchmark] - public unsafe void AllocHGlobal() - { - IntPtr unmanagedHandle = Marshal.AllocHGlobal(ArraySize); - Span unmanaged = new Span(unmanagedHandle.ToPointer(), ArraySize); - Marshal.FreeHGlobal(unmanagedHandle); - } - - [Benchmark] - public unsafe void AllocHGlobalWithLeaks() - { - IntPtr unmanagedHandle = Marshal.AllocHGlobal(ArraySize); - Span unmanaged = new Span(unmanagedHandle.ToPointer(), ArraySize); - } -} -``` - -It will produce the summary table like this one: - -| Method | Mean | Error | StdDev | Gen 0 | Gen 1 | Gen 2 | Allocated | Allocated native memory | Native memory leak | -|---------------------- |-------------:|--------------:|-------------:|------:|------:|------:|----------:|------------------------:|-------------------:| -| BitmapWithLeaks | 73,456.43 ns | 22,498.10 ns | 1,233.197 ns | - | - | - | 177 B | 13183 B | 11615 B | -| Bitmap | 91,590.08 ns | 101,468.12 ns | 5,561.810 ns | - | - | - | 180 B | 12624 B | - | -| AllocHGlobal | 79.91 ns | 43.93 ns | 2.408 ns | - | - | - | - | 80 B | - | -| AllocHGlobalWithLeaks | 103.50 ns | 153.21 ns | 8.398 ns | - | - | - | - | 80 B | 80 B | - -As you can see, we have two additional columns `Allocated native memory` and `Native memory leak` that contain some very useful numbers! - -## New ThreadingDiagnoser - -`ThreadingDiagnoser` also adds two extra columns to the summary table: - -* `Completed Work Items`: The number of work items that have been processed in ThreadPool (per single operation) -* `Lock Contentions`: The number of times there **was contention** upon trying to take a Monitor's lock (per single operation) - -Internally, it uses [new APIs](https://github.com/dotnet/corefx/issues/35500) exposed in .NET Core 3.0. - -It can be activated with the help of the `[ThreadingDiagnoser]` attribute: - -```cs -[ThreadingDiagnoser] -public class IntroThreadingDiagnoser -{ - [Benchmark] - public void CompleteOneWorkItem() - { - ManualResetEvent done = new ManualResetEvent(initialState: false); - ThreadPool.QueueUserWorkItem(m => (m as ManualResetEvent).Set(), done); - done.WaitOne(); - } -} -``` - -The above example will print a summary table like this one: - -| Method | Mean | StdDev | Median | Completed Work Items | Lock Contentions | -|-------------------- |--------------:|-----------:|--------------:|---------------------:|-----------------:| -| CompleteOneWorkItem | 8,073.5519 ns | 69.7261 ns | 8,111.6074 ns | 1.0000 | - | - - -## LINQPad 6 support - -Now both LINQPad 5 and LINQPad 6 are supported: - -![](https://user-images.githubusercontent.com/6011991/64907554-5d77b000-d6f4-11e9-9684-50ea678647eb.png) - -## Fast documentation search - -We continue to improve the usability of our documentation. -In this release, we improved the search experience in the documentation: now it works almost instantly with the help of [Algolia](https://www.algolia.com/) engine! -That's how it looks: - -![](https://user-images.githubusercontent.com/2259237/67472874-701dc700-f65a-11e9-85f1-d4f3160f78cf.gif) diff --git a/docs/_changelog/header/v0.12.1.md b/docs/_changelog/header/v0.12.1.md deleted file mode 100644 index 5c02ea5c0a..0000000000 --- a/docs/_changelog/header/v0.12.1.md +++ /dev/null @@ -1,294 +0,0 @@ -## Highlights - -* **.NET 5 support** - As you probably know, .NET Core 5 was officially [rebranded](https://github.com/dotnet/runtime/pull/33694) to .NET 5. - The new version of BenchmarkDotNet supports the new runtime after rebranding. - [#1399](https://github.com/dotnet/BenchmarkDotNet/pull/1399) - [465ebf](https://github.com/dotnet/BenchmarkDotNet/commit/465ebf3fdbf20f0e9219c4c957fb33c13256fdcd) -* **Perfolizer adoption** - The internal statistical engine of BenchmarkDotNet became mature enough to be transformed into an independent project. - Meet [perfolizer](https://github.com/AndreyAkinshin/perfolizer) — a toolkit for performance analysis! - While BenchmarkDotNet focuses on obtaining reliable measurements, perfolizer focuses on the decent analysis of measured data. - You still can use all the statistical algorithms from BenchmarkDotNet, - but you can also install perfolizer as a [standalone NuGet package](https://www.nuget.org/packages/Perfolizer/). - You can find more details in the [official announcement](https://aakinshin.net/posts/introducing-perfolizer/). - [#1386](https://github.com/dotnet/BenchmarkDotNet/pull/1386) - [54a061](https://github.com/dotnet/BenchmarkDotNet/commit/54a06102a6e0cc4169d23c6f9cd2779ee408d2bf) -* **Cross-platform disassembler** - Now the `DisassemblyDiagnoser` is cross-platform! - The disassembling logic was also improved, now it handles runtime helper methods and references to method tables properly. - Internally, it uses the [Iced](https://github.com/0xd4d/iced) library for formatting assembly code. - Special thanks to [@adamsitnik](https://github.com/adamsitnik) for the implementation and [@0xd4d](https://github.com/0xd4d) for Iced! - [#1332](https://github.com/dotnet/BenchmarkDotNet/pull/1332) - [#899](https://github.com/dotnet/BenchmarkDotNet/issues/899) - [#1316](https://github.com/dotnet/BenchmarkDotNet/issues/1316) - [#1364](https://github.com/dotnet/BenchmarkDotNet/issues/1364) - [294320](https://github.com/dotnet/BenchmarkDotNet/commit/294320be9525b0ecfefd0351381756d3a3b77211) -* **EventPipe-based cross-platform profiler** - Now you can easily profiler your benchmarks on Windows, Linux, and macOS! - Just mark your class with the `[EventPipeProfiler(...)]` attribute and get a `.speedscope.json` file that you can browse in [SpeedScope](https://www.speedscope.app/). - Special thanks to [@WojciechNagorski](https://github.com/WojciechNagorski) for the implementation! - [#1321](https://github.com/dotnet/BenchmarkDotNet/pull/1321) - [#1315](https://github.com/dotnet/BenchmarkDotNet/issues/1315) - [c648ff](https://github.com/dotnet/BenchmarkDotNet/commit/c648ff956662abae512c579ffa7f1dc12178f6c3) -* **New fluent API** - We continue to improve our API and make it easier for reading and writing. - Special thanks to [@WojciechNagorski](https://github.com/WojciechNagorski) for the implementation! - [#1273](https://github.com/dotnet/BenchmarkDotNet/pull/1273) - [#1234](https://github.com/dotnet/BenchmarkDotNet/issues/1234) - [640d88](https://github.com/dotnet/BenchmarkDotNet/commit/640d885ae0daddcee7c9ba9b5f1bf5790b9b5ae3) -* **Ref readonly support** - Now you can use `ref readonly` in benchmark signatures. - Special thanks to [@adamsitnik](https://github.com/adamsitnik) for the implementation! - [#1389](https://github.com/dotnet/BenchmarkDotNet/pull/1389) - [#1388](https://github.com/dotnet/BenchmarkDotNet/issues/1388) - [9ac777](https://github.com/dotnet/BenchmarkDotNet/commit/9ac7770682a45afb6cf4ec353f9fa3c69ece67ce) - -## Cross-platform disassembler - -Just mark your benchmark class with the `[DisassemblyDiagnoser]` attribute - and you will get the disassembly listings for all the benchmarks. -The formatting looks pretty nice thanks to [Iced](https://github.com/0xd4d/iced). -It works like a charm on Windows, Linux, and macOS. - -```cs -[DisassemblyDiagnoser] -public class IntroDisassembly -{ - private int[] field = Enumerable.Range(0, 100).ToArray(); - - [Benchmark] - public int SumLocal() - { - var local = field; // we use local variable that points to the field - - int sum = 0; - for (int i = 0; i < local.Length; i++) - sum += local[i]; - - return sum; - } - - [Benchmark] - public int SumField() - { - int sum = 0; - for (int i = 0; i < field.Length; i++) - sum += field[i]; - - return sum; - } -} -``` - -**.NET Core 2.1.16 (CoreCLR 4.6.28516.03, CoreFX 4.6.28516.10), X64 RyuJIT** - -```x86asm -; BenchmarkDotNet.Samples.IntroDisassembly.SumLocal() - mov rax,[rcx+8] - xor edx,edx - xor ecx,ecx - mov r8d,[rax+8] - test r8d,r8d - jle short M00_L01 -M00_L00: - movsxd r9,ecx - add edx,[rax+r9*4+10] - inc ecx - cmp r8d,ecx - jg short M00_L00 -M00_L01: - mov eax,edx - ret -; Total bytes of code 35 -``` - -**.NET Core 2.1.16 (CoreCLR 4.6.28516.03, CoreFX 4.6.28516.10), X64 RyuJIT** - -```x86asm -; BenchmarkDotNet.Samples.IntroDisassembly.SumField() - sub rsp,28 - xor eax,eax - xor edx,edx - mov rcx,[rcx+8] - cmp dword ptr [rcx+8],0 - jle short M00_L01 -M00_L00: - mov r8,rcx - cmp edx,[r8+8] - jae short M00_L02 - movsxd r9,edx - add eax,[r8+r9*4+10] - inc edx - cmp [rcx+8],edx - jg short M00_L00 -M00_L01: - add rsp,28 - ret -M00_L02: - call CORINFO_HELP_RNGCHKFAIL - int 3 -; Total bytes of code 53 -``` - -Now we handle runtime helper methods and references to method tables properly. Example: - -Before: - -```x86asm -; MicroBenchmarks.WithCallsAfter.Benchmark(Int32) - push rsi - sub rsp,20h - mov rsi,rcx - cmp edx,7FFFFFFFh - jne M00_L00 - call MicroBenchmarks.WithCallsAfter.Static() - mov rcx,rsi - call MicroBenchmarks.WithCallsAfter.Instance() - mov rcx,rsi - call MicroBenchmarks.WithCallsAfter.Recursive() - mov rcx,rsi - mov rax,qword ptr [rsi] - mov rax,qword ptr [rax+40h] - call qword ptr [rax+20h] - mov rcx,rsi - mov edx,1 - mov rax,7FF8F4217050h - add rsp,20h - pop rsi - jmp rax -M00_L00: - mov rcx,offset System_Private_CoreLib+0xa31d48 - call coreclr!MetaDataGetDispenser+0x322a0 - mov rsi,rax - mov ecx,0ACFAh - mov rdx,7FF8F42F4680h - call coreclr!MetaDataGetDispenser+0x17140 - mov rdx,rax - mov rcx,rsi - call System.InvalidOperationException..ctor(System.String) - mov rcx,rsi - call coreclr!coreclr_shutdown_2+0x39f0 - int 3 - add byte ptr [rax],al - sbb dword ptr [00007ff9`26284e30],eax - add dword ptr [rax+40h],esp - add byte ptr [rax],al - add byte ptr [rax],al - add byte ptr [rax],al - add byte ptr [rax-70BC4CCh],ah -; Total bytes of code 157 -``` - -After: - -```x86asm -; BenchmarkDotNet.Samples.WithCallsAfter.Benchmark(Int32) - push rsi - sub rsp,20 - mov rsi,rcx - cmp edx,7FFFFFFF - jne M00_L00 - call BenchmarkDotNet.Samples.WithCallsAfter.Static() - mov rcx,rsi - call BenchmarkDotNet.Samples.WithCallsAfter.Instance() - mov rcx,rsi - call BenchmarkDotNet.Samples.WithCallsAfter.Recursive() - mov rcx,rsi - mov rax,[rsi] - mov rax,[rax+40] - call qword ptr [rax+20] - mov rcx,rsi - mov edx,1 - mov rax BenchmarkDotNet.Samples.WithCallsAfter.Benchmark(Boolean) - add rsp,20 - pop rsi - jmp rax -M00_L00: - mov rcx MT_System.InvalidOperationException - call CORINFO_HELP_NEWSFAST - mov rsi,rax - mov ecx,12D - mov rdx,7FF954FF83F0 - call CORINFO_HELP_STRCNS - mov rdx,rax - mov rcx,rsi - call System.InvalidOperationException..ctor(System.String) - mov rcx,rsi - call CORINFO_HELP_THROW - int 3 -; Total bytes of code 134 -``` - -See also: [Cross-runtime .NET disassembly with BenchmarkDotNet](https://aakinshin.net/posts/dotnet-crossruntime-disasm/). - -Special thanks to [@adamsitnik](https://github.com/adamsitnik) for the implementation and [@0xd4d](https://github.com/0xd4d) for Iced! - -## EventPipe-based cross-platform profiler - -Now you can easily profiler your benchmarks on Windows, Linux, and macOS! - -If you want to use the new profiler, you should just mark your benchmark class with the `[EventPipeProfiler(...)]` attribute: - -``` -[EventPipeProfiler(EventPipeProfile.CpuSampling)] // <-- Enables new profiler -public class IntroEventPipeProfiler -{ - [Benchmark] - public void Sleep() => Thread.Sleep(2000); -} -``` - -Once the benchmark run is finished, you get a `.speedscope.json` file that can be opened in [SpeedScope](https://www.speedscope.app/): - -![](https://wojciechnagorski.github.io/images/EventPipeProfiler/SpeedScope.png#mid) - -The new profiler supports several modes: - -* `CpuSampling` - Useful for tracking CPU usage and general .NET runtime information. This is the default option. -* `GcVerbose` - Tracks GC collections and samples object allocations. -* `GcCollect` - Tracks GC collections only at very low overhead. -* `Jit` - Logging when Just in time (JIT) compilation occurs. Logging of the internal workings of the Just In Time compiler. This is fairly verbose. It details decisions about interesting optimization (like inlining and tail call) - -Please see Wojciech Nagórski's [blog post](https://wojciechnagorski.github.io/2020/04/cross-platform-profiling-.net-code-with-benchmarkdotnet/) for all the details. - -Special thanks to [@WojciechNagorski](https://github.com/WojciechNagorski) for the implementation! - -## New fluent API - -We continue to improve our API and make it easier for reading and writing. -The old API is still existing, but it is marked as obsolete and will be removed in the further library versions. -The most significant changes: - -**Changes in Job configuration** - -![](https://user-images.githubusercontent.com/17333903/66208963-b49af000-e6b6-11e9-87b3-1e4bb5519273.PNG) - -**Changes in IConfig/ManualConfig** - -![](https://user-images.githubusercontent.com/17333903/66208975-b6fd4a00-e6b6-11e9-94cd-85c73ae0ec27.PNG) - -**Full fluent API** - -![](https://user-images.githubusercontent.com/17333903/66208977-b8c70d80-e6b6-11e9-8e43-6dac3921c121.PNG) - -Special thanks to [@WojciechNagorski](https://github.com/WojciechNagorski) for the implementation! - -## Ref readonly support - -Now you can use `ref readonly` in benchmark signatures. -Here is an example: - -```cs -public class RefReadonlyBenchmark -{ - static readonly int[] array = { 1 }; - - [Benchmark] - public ref readonly int RefReadonly() => ref RefReadonlyMethod(); - - static ref readonly int RefReadonlyMethod() => ref array[0]; -} -``` - -Special thanks to [@adamsitnik](https://github.com/adamsitnik) for the implementation! \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.0.md b/docs/_changelog/header/v0.13.0.md deleted file mode 100644 index daf8ed8ea6..0000000000 --- a/docs/_changelog/header/v0.13.0.md +++ /dev/null @@ -1,368 +0,0 @@ -It's been a year since our last release. BenchmarkDotNet has been downloaded more than seven million times from [nuget.org](https://www.nuget.org/packages/BenchmarkDotNet/). It's more than we could have ever possibly imagined! Some could say, that it's also more than we can handle ;) That is why we wanted to once again thank [all the contributors](https://github.com/dotnet/BenchmarkDotNet/graphs/contributors) who helped us with `0.13.0` release! - -## Highlights - -In BenchmarkDotNet v0.13.0, we have supported various technologies: - -* .NET 5 and .NET 6 target framework monikers -* .NET SDK installed via snap -* SingleFile deployment -* Xamarin applications -* WASM applications -* Mono AOT - -We have also introduced new features and improvements including: - -* Memory randomization -* Method-specific job attributes -* Sortable parameter columns -* Customizable ratio column -* Improved CoreRun and CoreRT support -* Improved Hardware Counters support - -Of course, this release includes dozens of other improvements and bug fixes! - -## Supported technologies - -### .NET 5, .NET 6, SingleFile and snap - -At some point in time, `netcoreapp5.0` moniker was changed to `net5.0`, which required a fix on our side ([#1479](https://github.com/dotnet/BenchmarkDotNet/pull/1479), btw we love this kind of changes). Moreover, .NET 5 introduced platform-specific TMFs (example: `net5.0-windows10.0.19041.0`) which also required some extra work: [#1560](https://github.com/dotnet/BenchmarkDotNet/pull/1560), [#1691](https://github.com/dotnet/BenchmarkDotNet/pull/1691). - - - In [#1523](https://github.com/dotnet/BenchmarkDotNet/pull/1523) support for .NET 6 was added. - -```xml -net5.0;net5.0-windows10.0.19041.0;net6.0 -``` - -In [#1686](https://github.com/dotnet/BenchmarkDotNet/pull/1686) [@am11](https://github.com/am11) has implemented support for **single file deployment** (supported in .NET 5 onwards). - -Last, but not least in [#1652](https://github.com/dotnet/BenchmarkDotNet/pull/1652) **snap** support has been implemented. - -```log -adam@adsitnik-ubuntu:~/projects/BenchmarkDotNet/samples/BenchmarkDotNet.Samples$ dotnet50 run -c Release -f net5.0 --filter BenchmarkDotNet.Samples.IntroColdStart.Foo -// Validating benchmarks: -// ***** BenchmarkRunner: Start ***** -// ***** Found 1 benchmark(s) in total ***** -// ***** Building 1 exe(s) in Parallel: Start ***** -// start /snap/dotnet-sdk/112/dotnet restore /p:UseSharedCompilation=false /p:BuildInParallel=false /m:1 /p:Deterministic=true /p:Optimize=true in /home/adam/projects/BenchmarkDotNet/samples/BenchmarkDotNet.Samples/bin/Release/net5.0/9a018ee4-0f33-46dd-9093-01d3bf31233b -// command took 1.49s and exited with 0 -// start /snap/dotnet-sdk/112/dotnet build -c Release --no-restore /p:UseSharedCompilation=false /p:BuildInParallel=false /m:1 /p:Deterministic=true /p:Optimize=true in /home/adam/projects/BenchmarkDotNet/samples/BenchmarkDotNet.Samples/bin/Release/net5.0/9a018ee4-0f33-46dd-9093-01d3bf31233b -// command took 2.78s and exited with 0 -// ***** Done, took 00:00:04 (4.37 sec) ***** -// Found 1 benchmarks: -// IntroColdStart.Foo: Job-NECTOD(IterationCount=5, RunStrategy=ColdStart) - -// ************************** -// Benchmark: IntroColdStart.Foo: Job-NECTOD(IterationCount=5, RunStrategy=ColdStart) -// *** Execute *** -// Launch: 1 / 1 -// Execute: /snap/dotnet-sdk/112/dotnet "9a018ee4-0f33-46dd-9093-01d3bf31233b.dll" --benchmarkName "BenchmarkDotNet.Samples.IntroColdStart.Foo" --job "IterationCount=5, RunStrategy=ColdStart" --benchmarkId 0 in /home/adam/projects/BenchmarkDotNet/samples/BenchmarkDotNet.Samples/bin/Release/net5.0/9a018ee4-0f33-46dd-9093-01d3bf31233b/bin/Release/net5.0 -``` - -### Xamarin support - -Thanks to the contributions of the amazing [@jonathanpeppers](https://github.com/jonathanpeppers) BenchmarkDotNet supports Xamarin! The examples can be found in our repo: [iOS](https://github.com/dotnet/BenchmarkDotNet/tree/master/samples/BenchmarkDotNet.Samples.iOS), [Android](https://github.com/dotnet/BenchmarkDotNet/tree/master/samples/BenchmarkDotNet.Samples.Android). - -![](https://user-images.githubusercontent.com/840039/73184210-b105fb80-40e1-11ea-9efb-d29fe4d08c79.png#mid) - -[#1360](https://github.com/dotnet/BenchmarkDotNet/pull/1360), [#1429](https://github.com/dotnet/BenchmarkDotNet/pull/1429), [#1434](https://github.com/dotnet/BenchmarkDotNet/pull/1434), [#1509](https://github.com/dotnet/BenchmarkDotNet/pull/1509) - -### WASM support - -Thanks to the work of [@naricc](https://github.com/naricc) you can now benchmark WASM using Mono Runtime! For more details, please refer to our [docs](https://benchmarkdotnet.org/articles/configs/toolchains.html#Wasm). - -![](https://user-images.githubusercontent.com/6011991/87671577-dd860080-c771-11ea-922f-e26a32a3f831.png#mid) - -[#1483](https://github.com/dotnet/BenchmarkDotNet/pull/1483), [#1498](https://github.com/dotnet/BenchmarkDotNet/pull/1498), [#1500](https://github.com/dotnet/BenchmarkDotNet/pull/1500), [#1501](https://github.com/dotnet/BenchmarkDotNet/pull/1501), [#1507](https://github.com/dotnet/BenchmarkDotNet/pull/1507), [#1592](https://github.com/dotnet/BenchmarkDotNet/pull/1592), [#1689](https://github.com/dotnet/BenchmarkDotNet/pull/1689). - -### Mono AOT support - -In another awesome contribution ([#1662](https://github.com/dotnet/BenchmarkDotNet/pull/1662)) [@naricc](https://github.com/naricc) has implemented Mono AOT support. The new toolchain supports doing Mono AOT runs with both the Mono-Mini compiler and the Mono-LLVM compiler (which uses LLVM on the back end). - - For more details, please go to our [docs](https://benchmarkdotnet.org/articles/configs/toolchains.html#MonoAotLLVM). - -## New features and improvements - -### Memory randomization - -In [#1587](https://github.com/dotnet/BenchmarkDotNet/pull/1587) [@adamsitnik](https://github.com/adamsitnik) has introduced a new, **experimental** feature called "Memory Randomization". - -This feature allows you to ask BenchmarkDotNet to randomize the memory alignment by allocating random-sized byte arrays between iterations and **call [GlobalSetup] before every benchmark iteration** and `[GlobalCleanup]` after every benchmark iteration. - -Sample benchmark: - -```cs -public class IntroMemoryRandomization -{ - [Params(512 * 4)] - public int Size; - - private int[] _array; - private int[] _destination; - - [GlobalSetup] - public void Setup() - { - _array = new int[Size]; - _destination = new int[Size]; - } - - [Benchmark] - public void Array() => System.Array.Copy(_array, _destination, Size); -} -``` - -Without asking for the randomization, the objects are allocated in `[GlobalSetup]` and their unmodified addresses (and alignment) are used for all iterations (as long as they are not promoted to an older generation by the GC). This is typically the desired behavior, as it gives you very nice and flat distributions: - -```cmd -dotnet run -c Release --filter IntroMemoryRandomization -``` - -```ini --------------------- Histogram -------------------- -[502.859 ns ; 508.045 ns) | @@@@@@@@@@@@@@@ ---------------------------------------------------- -``` - -But if for some reason you are interested in getting a distribution that is better reflecting the "real-life" performance you can enable the randomization: - -```cmd -dotnet run -c Release --filter IntroMemoryRandomization --memoryRandomization true -``` - -```ini --------------------- Histogram -------------------- -[108.803 ns ; 213.537 ns) | @@@@@@@@@@@@@@@ -[213.537 ns ; 315.458 ns) | -[315.458 ns ; 446.853 ns) | @@@@@@@@@@@@@@@@@@@@ -[446.853 ns ; 559.259 ns) | @@@@@@@@@@@@@@@ ---------------------------------------------------- -``` - -### Method-specific job attributes - -From now, all attributes that derive from `JobMutatorConfigBaseAttribute` ([full list](https://benchmarkdotnet.org/api/BenchmarkDotNet.Attributes.JobMutatorConfigBaseAttribute.html#BenchmarkDotNet_Attributes_JobMutatorConfigBaseAttribute)) can be applied to methods. You no longer have to move a method to a separate type to customize config for it. - -```cs -[Benchmark] -[WarmupCount(1)] -public void SingleWarmupIteration() - -[Benchmark] -[WarmupCount(9)] -public void NineWarmupIterations() -``` - -### Sortable parameter columns - -In order to sort columns of parameters in the results table, you can use the Property `Priority` inside the params attribute. The priority range is `[Int32.MinValue;Int32.MaxValue]`, lower priorities will appear earlier in the column order. The default priority is set to `0`. - -```cs -public class IntroParamsPriority -{ - [Params(100)] - public int A { get; set; } - - [Params(10, Priority = -100)] - public int B { get; set; } - - [Benchmark] - public void Benchmark() => Thread.Sleep(A + B + 5); -} -``` - -| Method | B | A | Mean | Error | StdDev | -|---------- |--- |---- |---------:|--------:|--------:| -| Benchmark | 10 | 100 | 115.4 ms | 0.12 ms | 0.11 ms | - - -This feature got implemented by [@JohannesDeml](https://github.com/JohannesDeml) in [#1612](https://github.com/dotnet/BenchmarkDotNet/pull/1612). - -### Customizable ratio column - -Now it's possible to customize the format of the ratio column. - -```cs -[Config(typeof(Config))] -public class IntroRatioStyle -{ - [Benchmark(Baseline = true)] - public void Baseline() => Thread.Sleep(1000); - - [Benchmark] - public void Bar() => Thread.Sleep(150); - - [Benchmark] - public void Foo() => Thread.Sleep(1150); - - private class Config : ManualConfig - { - public Config() - { - SummaryStyle = SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend); - } - } -} -``` - -``` -| Method | Mean | Error | StdDev | Ratio | RatioSD | -|--------- |-----------:|--------:|--------:|-------------:|--------:| -| Baseline | 1,000.6 ms | 2.48 ms | 0.14 ms | baseline | | -| Bar | 150.9 ms | 1.30 ms | 0.07 ms | 6.63x faster | 0.00x | -| Foo | 1,150.4 ms | 5.17 ms | 0.28 ms | 1.15x slower | 0.00x | -``` - -This feature was implemented in [#731](https://github.com/dotnet/BenchmarkDotNet/issues/721). - -### Improved CoreRun support - -BenchmarkDotNet was reporting invalid .NET Core version number when comparing performance using CoreRuns built from `dotnet/corefx` and `dotnet/runtime`. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1580](https://github.com/dotnet/BenchmarkDotNet/pull/1580) - -In [#1552](https://github.com/dotnet/BenchmarkDotNet/pull/1552) [@stanciuadrian](https://github.com/stanciuadrian) has implemented support for all `GcMode` characteristics for `CoreRunToolchain`. Previously the settings were just ignored, now they are being translated to corresponding `COMPlus_*` env vars. - -### Improved CoreRT support - -CoreRT has moved from https://github.com/dotnet/corert/ to https://github.com/dotnet/runtimelab/tree/feature/NativeAOT and we had to update the default compiler version and nuget feed address. Moreover, there was a bug in `CoreRtToolchain` which was causing any additional native dependencies to not work. - -Big thanks to [@MichalStrehovsky](https://github.com/MichalStrehovsky), [@jkotas](https://github.com/jkotas) and [@kant2002](https://github.com/kant2002) for their help and support! - -[#1606](https://github.com/dotnet/BenchmarkDotNet/pull/1606), [#1643](https://github.com/dotnet/BenchmarkDotNet/pull/1643), [#1679](https://github.com/dotnet/BenchmarkDotNet/pull/1679) - -### Command-line argument support in BenchmarkRunner - -So far only `BenchmarkSwitcher` was capable of handling console line arguments. Thanks to [@chan18](https://github.com/chan18) `BenchmarkRunner` supports them as well ([#1292](https://github.com/dotnet/BenchmarkDotNet/pull/1292)): - -```cs -public class Program -{ - public static void Main(string[] args) => BenchmarkRunner.Run(typeof(Program).Assembly, args: args); -} -``` - -### New API: ManualConfig.CreateMinimumViable - -`ManualConfig.CreateEmpty` creates a completely empty config. Without adding a column provider and a logger to the config the users won't see any results being printed. In [#1582](https://github.com/dotnet/BenchmarkDotNet/pull/1582) [@adamsitnik](https://github.com/adamsitnik) has introduced a new method that creates minimum viable config: - -```cs -IConfig before = ManualConfig.CreateEmpty() - .AddColumnProvider(DefaultColumnProviders.Instance) - .AddLogger(ConsoleLogger.Default); - -IConfig after = ManualConfig.CreateMinimumViable(); -``` - -### Benchmarking NuGet packages from custom feeds - -In [#1659](https://github.com/dotnet/BenchmarkDotNet/pull/1659/) [@workgroupengineering](https://github.com/workgroupengineering) added the possibility to indicate the source of the tested nuget package and whether it is a pre-release version. - -### Deterministic benchmark builds - -BenchmarkDotNet is now always enforcing Deterministic builds ([#1489](https://github.com/dotnet/BenchmarkDotNet/pull/1489)) and Optimizations enabled ([#1494](https://github.com/dotnet/BenchmarkDotNet/pull/1494)) which is a must-have if you are using custom build configurations. MSBuild enforces optimizations **only** for configurations that are named `Release` (the comparison is case-insensitive). - -```xml - - - - - - -``` - -```cs -var config = DefaultConfig.Instance - .AddJob(Job.Default.WithCustomBuildConfiguration("X").WithId("X").AsBaseline()) - .AddJob(Job.Default.WithCustomBuildConfiguration("Y").WithId("Y")); -``` - -[#1489](https://github.com/dotnet/BenchmarkDotNet/pull/1489), [#1494](https://github.com/dotnet/BenchmarkDotNet/pull/1494) - -### Improved Hardware Counters support - -BenchmarkDotNet is being used by the .NET Team to ensure that .NET is not regressing. More than three thousand benchmarks (they can be found [here](https://github.com/dotnet/performance/tree/main/src/benchmarks/micro)) are being executed multiple times a day on multiple hardware configs. Recently, .NET Team started to use `InstructionsRetired` to help to filter unstable benchmarks that report regressions despite not changing the number of instructions retired. This has exposed few bugs in Hardware Counters support in BenchmarkDotNet, which all got fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1547](https://github.com/dotnet/BenchmarkDotNet/pull/1547) and [#1550](https://github.com/dotnet/BenchmarkDotNet/pull/1550). Moreover, we have **removed** the old [PmcDiagnoser](https://adamsitnik.com/Hardware-Counters-Diagnoser/) and extended [EtwProfiler](https://adamsitnik.com/ETW-Profiler/) with the hardware counters support. It's just much more accurate and futureproof. For details, please go to [#1548](https://github.com/dotnet/BenchmarkDotNet/pull/1548). - -How stable was `PmcDiagnoser` (same benchmarks run twice in a row on the same machine): - -| Method | Runtime | InstructionRetired/Op | -|---------- |--------------------- |----------------------:| -| Burgers_0 | .NET 5.0 | 845,746 | -| Burgers_0 | .NET Core 2.1 | 30,154,151 | -| Burgers_0 | .NET Framework 4.6.1 | 4,230,848 | - -| Method | Runtime | InstructionRetired/Op | -|---------- |--------------------- |----------------------:| -| Burgers_0 | .NET 5.0 | 34,154,524 | -| Burgers_0 | .NET Core 2.1 | 246,534,203 | -| Burgers_0 | .NET Framework 4.6.1 | 2,607,686 | - -How stable is `EtwProfiler`: - -| Method | Runtime | InstructionRetired/Op | -|---------- |--------------------- |----------------------:| -| Burgers_0 | .NET 5.0 | 3,069,978,261 | -| Burgers_0 | .NET Core 2.1 | 3,676,000,000 | -| Burgers_0 | .NET Framework 4.6.1 | 3,468,866,667 | - -| Method | Runtime | InstructionRetired/Op | -|---------- |--------------------- |----------------------:| -| Burgers_0 | .NET 5.0 | 3,066,810,000 | -| Burgers_0 | .NET Core 2.1 | 3,674,666,667 | -| Burgers_0 | .NET Framework 4.6.1 | 3,468,600,000 | - -Moreover, in [#1540](https://github.com/dotnet/BenchmarkDotNet/pull/1540) [@WojciechNagorski](https://github.com/WojciechNagorski) has added the removal of temporary files created by `EtwProfiler`. - -### Improved Troubleshooting - -We have the possibility to ask BDN to stop on the first error: `--stopOnFirstError true`. - -The problem was when the users had not asked for that, tried to run `n` benchmarks, all of them failed to build, and BDN was printing the same build error `n` times. - -In [#1672](https://github.com/dotnet/BenchmarkDotNet/pull/1672) [@adamsitnik](https://github.com/adamsitnik) has changed that, so when all the build fails, BDN stops after printing the first error. - -Moreover, we have also changed the default behavior for the failed builds of the boilerplate code. If the build fails, we don't remove the files. Previously we have required the users to pass `--keepFiles` to keep them. See [#1567](https://github.com/dotnet/BenchmarkDotNet/pull/1567) for more details and don't forget about the [Troubleshooting](https://benchmarkdotnet.org/articles/guides/troubleshooting.html) docs! - -### Docs and Samples improvements - -Big thanks to [@lukasz-pyrzyk](https://github.com/lukasz-pyrzyk), [@fleckert](https://github.com/fleckert), [@MarecekF](https://github.com/MarecekF), [@joostas](https://github.com/joostas), [@michalgalecki](https://github.com/michalgalecki), [@WojciechNagorski](https://github.com/WojciechNagorski), [@MendelMonteiro](https://github.com/MendelMonteiro), [@kevinsalimi](https://github.com/kevinsalimi), [@cedric-cf](https://github.com/cedric-cf), [@YohDeadfall](https://github.com/YohDeadfall), [@jeffhandley](https://github.com/jeffhandley) and [@JohannesDeml](https://github.com/JohannesDeml) who have improved our docs and samples! - -[#1463](https://github.com/dotnet/BenchmarkDotNet/pull/1463), [#1465](https://github.com/dotnet/BenchmarkDotNet/pull/1465), [#1508](https://github.com/dotnet/BenchmarkDotNet/pull/1508), [#1518](https://github.com/dotnet/BenchmarkDotNet/pull/1518), [#1554](https://github.com/dotnet/BenchmarkDotNet/pull/1554), [#1568](https://github.com/dotnet/BenchmarkDotNet/pull/1568), [#1601](https://github.com/dotnet/BenchmarkDotNet/pull/1601), [#1633](https://github.com/dotnet/BenchmarkDotNet/pull/1633), [#1645](https://github.com/dotnet/BenchmarkDotNet/pull/1645), [#1647](https://github.com/dotnet/BenchmarkDotNet/pull/1647), [#1657](https://github.com/dotnet/BenchmarkDotNet/pull/1657), [#1675](https://github.com/dotnet/BenchmarkDotNet/pull/1675), [#1676](https://github.com/dotnet/BenchmarkDotNet/pull/1676), [#1690](https://github.com/dotnet/BenchmarkDotNet/pull/1690). - -### Template improvements - -* Projects created out of our official templates might have been unexpectedly packed when running `dotnet pack` on the entire solution. In [#1584](https://github.com/dotnet/BenchmarkDotNet/pull/1584) [@kendaleiv](https://github.com/kendaleiv) has explicitly disabled packing for the template. -* The template had `netcoreapp3.0` TFM hardcoded. This got fixed by [@https://github.com/ExceptionCaught](https://github.com/ExceptionCaught) in [#1630](https://github.com/dotnet/BenchmarkDotNet/pull/1630) and [#1632](https://github.com/dotnet/BenchmarkDotNet/pull/1632). -* In [#1667](https://github.com/dotnet/BenchmarkDotNet/pull/1667) [@YohDeadfall](https://github.com/YohDeadfall) has changed the default debug type from `portable` to `pdbonly` (required by `DisassemblyDiagnoser`). - -## Bug fixes - - * Very long string `[Arguments]` and `[Params]` were causing BenchmarkDotNet to crash. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1248](https://github.com/dotnet/BenchmarkDotNet/pull/1248) and [#1545](https://github.com/dotnet/BenchmarkDotNet/pull/1545/). So far trace file names were containing full benchmark names and arguments. Now if the name is too long, the trace file name is a hash (consistent for multiple runs of the same benchmark). The same goes for passing benchmark name by the host process to the benchmark process via command-line arguments. -* `LangVersion` set to a non-numeric value like `latest` was crashing the build. Fixed by [@martincostello](https://github.com/martincostello) in [#1420](https://github.com/dotnet/BenchmarkDotNet/pull/1420). -* Windows 10 November 201**9** was being recognized as 201**8**. Fixed by [@kapsiR](https://github.com/kapsiR) in [#1437](https://github.com/dotnet/BenchmarkDotNet/pull/1437). -* Assemblies loaded via streams were not supported. Fixed by [@jeremyosterhoudt](https://github.com/jeremyosterhoudt) in [#1443](https://github.com/dotnet/BenchmarkDotNet/pull/1443). -* [NativeMemoryProfiler](https://wojciechnagorski.github.io/2019/08/analyzing-native-memory-allocation-with-benchmarkdotnet/) was detecting small leaks that were false positives. Fixed by [@WojciechNagorski](https://github.com/WojciechNagorski) in [#1451](https://github.com/dotnet/BenchmarkDotNet/pull/1451) and [#1600](https://github.com/dotnet/BenchmarkDotNet/pull/1600). -* [DisassemblyDiagnoser](https://adamsitnik.com/Disassembly-Diagnoser/) was crashing on Linux. Fixed by [@damageboy](https://github.com/damageboy) in [#1459](https://github.com/dotnet/BenchmarkDotNet/pull/1459). -* Target framework moniker was being printed as toolchain name for Full .NET Framework benchmarks. Fixed by [@svick](https://github.com/svick) in [#1471](https://github.com/dotnet/BenchmarkDotNet/pull/1471). -* `[ParamsSource]` returning `IEnumerable` was not working properly when combined with `[Arguments]`. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1478](https://github.com/dotnet/BenchmarkDotNet/pull/1478). -* `NullReferenceException` in `MultimodalDistributionAnalyzer`. Fixed by [@suslovk](https://github.com/suslovk) in [#1488](https://github.com/dotnet/BenchmarkDotNet/pull/1488). -* `NotSupportedException` was being thrown if there was an encoding mismatch between host and benchmark process. Diagnosed by [@ChristophLindemann](https://github.com/ChristophLindemann) in [#1487](https://github.com/dotnet/BenchmarkDotNet/issues/1487), fixed by [@lovettchris](https://github.com/lovettchris) in [#1491](https://github.com/dotnet/BenchmarkDotNet/pull/1491). -* `MissingMethodException` was being thrown in projects that referenced a newer version of [Iced](https://github.com/icedland/iced). Fixed by [@Symbai](https://github.com/Symbai) in [#1497](https://github.com/dotnet/BenchmarkDotNet/pull/1497) and in [#1502](https://github.com/dotnet/BenchmarkDotNet/pull/1502). -* `AppendTargetFrameworkToOutputPath` set to `false` was not supported. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1563](https://github.com/dotnet/BenchmarkDotNet/pull/1563) -* A locking finalizer could have hanged benchmark process which would just print `// AfterAll` and never quit. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1571](https://github.com/dotnet/BenchmarkDotNet/pull/1571). To prevent other hangs from happening, a timeout of `250ms` was added. If the process does not quit after running the benchmarks and waiting `250ms`, it's being force killed. -* In some cases, `JsonExporter` was reporting `NaN` for some of the Statistics. This was breaking non-.NET JSON deserializers. Fixed by [@marcnet80](https://github.com/marcnet80) in [#1581](https://github.com/dotnet/BenchmarkDotNet/pull/1581). -* `UnitType.Size` metrics were not using the provided number format. Fixed by [@jodydonetti](https://github.com/jodydonetti) in [#1569](https://github.com/dotnet/BenchmarkDotNet/pull/1569) and [#1618](https://github.com/dotnet/BenchmarkDotNet/pull/1618). -* `MaxColumnWidth` setting was not used for type names. Fixed by [@JohannesDeml](https://github.com/JohannesDeml) in [#1609](https://github.com/dotnet/BenchmarkDotNet/pull/1609). -* Current culture was not respected when formatting `Ratio` column values. Fixed by [@JohannesDeml](https://github.com/JohannesDeml) in [#1610](https://github.com/dotnet/BenchmarkDotNet/pull/1610). -* BenchmarkDotNet was redirecting Standard Error of the benchmark process, which was causing deadlocks for benchmarks that were writing to it. Fixed by [@adamstinik](https://github.com/adamsitnik) in [#1631](https://github.com/dotnet/BenchmarkDotNet/pull/1631) -* `DisassemblyDiagnoser` was failing to disassemble multiline source code. [@YohDeadfall](https://github.com/YohDeadfall) fixed that in [#1674](https://github.com/dotnet/BenchmarkDotNet/pull/1674). -* In [#1644](https://github.com/dotnet/BenchmarkDotNet/pull/1644) [@adamstinik](https://github.com/adamsitnik) has fixed the inconsistency between benchmark filter and hint. - -## Removal of the dotnet global tool - -In [#1006](https://github.com/dotnet/BenchmarkDotNet/pull/1006) (0.11.4) we have introduced a new dotnet global tool. - -By looking at the number of reported bugs we got to the conclusion that the tool has not passed the test of time. - -Why? Because it was based entirely on dynamic assembly loading which is very hard to get right in .NET and the fact that we support all existing .NET Runtimes (.NET, .NET Core, Mono, CoreRT) made it even harder (if not impossible). - -**We have removed it and the old versions are no longer supported**. For more details, please refer to [#1572](https://github.com/dotnet/BenchmarkDotNet/pull/1572). \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.1.md b/docs/_changelog/header/v0.13.1.md deleted file mode 100644 index 5744a4f2e2..0000000000 --- a/docs/_changelog/header/v0.13.1.md +++ /dev/null @@ -1,15 +0,0 @@ -BenchmarkDotNet v0.13.1 is a service update with various bug fixes and improvements. - -## Highlights - -* S390x architecture support ([#1712](https://github.com/dotnet/BenchmarkDotNet/pull/1712)) -* Various WASM toolchain improvements ([#1719](https://github.com/dotnet/BenchmarkDotNet/pull/1719), [#1722](https://github.com/dotnet/BenchmarkDotNet/pull/1722), [#1729](https://github.com/dotnet/BenchmarkDotNet/pull/1729), [#1742](https://github.com/dotnet/BenchmarkDotNet/pull/1742), [#1743](https://github.com/dotnet/BenchmarkDotNet/pull/1743), [#1744](https://github.com/dotnet/BenchmarkDotNet/pull/1744), [#1746](https://github.com/dotnet/BenchmarkDotNet/pull/1746), [#1757](https://github.com/dotnet/BenchmarkDotNet/pull/1757), [#1763](https://github.com/dotnet/BenchmarkDotNet/pull/1763)) -* Support of CoreRT on 5.0.3XX SDK ([#1725](https://github.com/dotnet/BenchmarkDotNet/pull/1725)) -* Using Utf8 for reading from standard input (fixes a nasty encoding-related bug) ([#1735](https://github.com/dotnet/BenchmarkDotNet/pull/1735)) -* Adjusting WaitForExit timeouts ([#1745](https://github.com/dotnet/BenchmarkDotNet/pull/1745)) -* Support for returning unmanaged types from benchmarks with InProcessToolchain ([#1750](https://github.com/dotnet/BenchmarkDotNet/pull/1750)) -* Disabled Tiered JIT ([#1751](https://github.com/dotnet/BenchmarkDotNet/pull/1751)) -* A MemoryDiagnoser bug fix ([#1762](https://github.com/dotnet/BenchmarkDotNet/pull/1762)) -* Allow users to hide Gen X columns ([#1764](https://github.com/dotnet/BenchmarkDotNet/pull/1764)) -* Copy GC settings from host process to the benchmark process ([#1765](https://github.com/dotnet/BenchmarkDotNet/pull/1765)) -* Do not split surrogates in shortified parameter values ([#1705](https://github.com/dotnet/BenchmarkDotNet/pull/1705)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.10.md b/docs/_changelog/header/v0.13.10.md deleted file mode 100644 index 9f55725d19..0000000000 --- a/docs/_changelog/header/v0.13.10.md +++ /dev/null @@ -1,3 +0,0 @@ -## Highlights - -Initial support of .NET 9 and minor bug fixes. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.11.md b/docs/_changelog/header/v0.13.11.md deleted file mode 100644 index ef2d1b9579..0000000000 --- a/docs/_changelog/header/v0.13.11.md +++ /dev/null @@ -1,3 +0,0 @@ -## Highlights - -Small improvements. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.12.md b/docs/_changelog/header/v0.13.12.md deleted file mode 100644 index c1d788166c..0000000000 --- a/docs/_changelog/header/v0.13.12.md +++ /dev/null @@ -1,10 +0,0 @@ -## Highlights - -The biggest highlight of this release if our new VSTest Adapter, - which allows to run benchmarks as unit tests in your favorite IDE! -The detailed guide can be found [here](xref:docs.vstest). - -This release also includes to a minor bug fix that caused incorrect job id generation: - fixed job id generation ([#2491](https://github.com/dotnet/BenchmarkDotNet/pull/2491)). - -Also, the target framework in the BenchmarkDotNet templates was bumped to .NET 8.0. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.2.md b/docs/_changelog/header/v0.13.2.md deleted file mode 100644 index dd5365e56e..0000000000 --- a/docs/_changelog/header/v0.13.2.md +++ /dev/null @@ -1,254 +0,0 @@ -## Highlights - -In BenchmarkDotNet v0.13.2, we have implemented support for: - -* .NET 7 -* NativeAOT -* .NET Framework 4.8.1 - -We have also introduced new features and improvements including: - -* Possibility to hide selected columns, -* Allocation Ratio column, -* Logging progress and estimated finish time, -* ARM64 support for `BenchmarkDotNet.Diagnostics.Windows` package, -* Printing Hardware Intrinsics information, -* Glob filters support for DisassemblyDiagnoser. - -Of course, this release includes dozens of other improvements and bug fixes! - -Our special thanks go to [@mawosoft](https://github.com/mawosoft), [@YegorStepanov](https://github.com/YegorStepanov) and [@radical](https://github.com/radical) who fixed a LOT of really nasty bugs. - -## Supported technologies - -### .NET 7 and .NET Framework 4.8.1 - -.NET 4.8.1 has been [released](https://devblogs.microsoft.com/dotnet/announcing-dotnet-framework-481/) earlier this month, while .NET 7 should land in autumn this year. Now you can use BenchmarkDotNet to compare both! - -```ini -BenchmarkDotNet=v0.13.1.1845-nightly, OS=Windows 11 (10.0.22622.575) -Microsoft SQ1 3.0 GHz, 1 CPU, 8 logical and 8 physical cores -.NET SDK=7.0.100-preview.6.22352.1 - [Host] : .NET 7.0.0 (7.0.22.32404), Arm64 RyuJIT AdvSIMD - Job-QJVIDT : .NET 7.0.0 (7.0.22.32404), Arm64 RyuJIT AdvSIMD - Job-FNNXOY : .NET Framework 4.8.1 (4.8.9032.0), Arm64 RyuJIT -``` - -| Method | Runtime | Mean | Allocated | -|-------------- |--------------------- |---------:|----------:| -| BinaryTrees_2 | .NET 7.0 | 193.6 ms | 227.33 MB | -| BinaryTrees_2 | .NET Framework 4.8.1 | 192.8 ms | 228.01 MB | - -Credit for adding .NET 7 support in [#1816](https://github.com/dotnet/BenchmarkDotNet/pull/1816) goes to [@am11](https://github.com/am11). [@adamsitnik](https://github.com/adamsitnik) implemented .NET 4.8.1 support in [#2044](https://github.com/dotnet/BenchmarkDotNet/pull/2044) and [#2067](https://github.com/dotnet/BenchmarkDotNet/pull/2067). Big thanks to [@MichalPetryka](https://github.com/MichalPetryka) who was using preview versions of BenchmarkDotNet and reported a bug related to .NET 4.8.1 support: [#2059](https://github.com/dotnet/BenchmarkDotNet/issues/2059) that got fixed before we released a new version. - -### NativeAOT - -We are really excited to see the experimental CoreRT project grow and become officially supported by Microsoft (under new name: NativeAOT)! You can read more about it [here](https://devblogs.microsoft.com/dotnet/announcing-dotnet-7-preview-3/#what-is-native-aot). Implementing and improving the support was a combined effort of multiple contributors that spawned across multiple repositories: -* [@MichalStrehovsky](https://github.com/MichalStrehovsky): [#66290 in dotnet/runtime](https://github.com/dotnet/runtime/pull/66290), [#2020 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2020), [#2046 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2046) -* [@hez2010](https://github.com/hez2010): [#66650 in dotnet/runtime](https://github.com/dotnet/runtime/pull/66650) -* [@Beau-Gosse-dev](https://github.com/Beau-Gosse-dev): [#1955 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1955) -* [@adamsitnik](https://github.com/adamsitnik): [#1960 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1960), [#1965 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1965), [#1972 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1972), [#1973 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1973), [#1994 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1994), [#1997 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1997), [#2045 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2045), [#2068 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2068) -* [@kant2002](https://github.com/kant2002): [#1976 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1976), [#1979 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/1979) -* [@jkotas](https://github.com/jkotas): [#68038 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68038), [#68142 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68142), [#68249 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68249), [#68308 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68308), [#68375 in dotnet/runtime](https://github.com/dotnet/runtime/pull/68375) -* [@MichalPetryka](https://github.com/MichalPetryka): [#2065 in dotnet/BenchmarkDotNet](https://github.com/dotnet/BenchmarkDotNet/pull/2065) - - -As every AOT solution, NativeAOT has some [limitations](https://github.com/dotnet/runtime/blob/main/src/coreclr/nativeaot/docs/limitations.md) like limited reflection support or lack of dynamic assembly loading. Because of that, the host process (what you run from command line) is never an AOT process, but just a regular .NET process. This process (called Host process) uses reflection to read benchmarks metadata (find all `[Benchmark]` methods etc.), generates a new project that references the benchmarks and compiles it using ILCompiler. The boilerplate code is not using reflection, so the project is built with `TrimmerDefaultAction=link` (we have greatly reduced build time thanks to that). Such compilation produces a native executable, which is later started by the Host process. This process (called Benchmark or Child process) performs the actual benchmarking and reports the results back to the Host process. By default BenchmarkDotNet uses the latest version of `Microsoft.DotNet.ILCompiler` to build the NativeAOT benchmark according to [this instructions](https://github.com/dotnet/runtime/blob/main/src/coreclr/nativeaot/docs/compiling.md). Moreover, BenchmarkDotNet by default uses current machines CPU features (change: [#1994](https://github.com/dotnet/BenchmarkDotNet/pull/1994), discussion: [#2061](https://github.com/dotnet/BenchmarkDotNet/issues/2061)) and if you don't like this behavior, you can [disable it](https://github.com/dotnet/BenchmarkDotNet/issues/2061#issuecomment-1203602177). - -This is why you need to: -- install [pre-requisites](https://learn.microsoft.com/dotnet/core/deploying/native-aot/#prerequisites) required by NativeAOT compiler -- target .NET to be able to run NativeAOT benchmarks (example: `net7.0` in the .csproj file) -- run the app as a .NET process (example: `dotnet run -c Release -f net7.0`). -- specify the NativeAOT runtime in an explicit way, either by using console line arguments `--runtimes nativeaot7.0` (the recommended approach), or by using`[SimpleJob]` attribute or by using the fluent Job config API `Job.ShortRun.With(NativeAotRuntime.Net70)`: - -```cmd -dotnet run -c Release -f net7.0 --runtimes nativeaot7.0 -``` - -For more examples please go to [docs](https://benchmarkdotnet.org/articles/configs/toolchains.html#nativeaot). - -```ini -BenchmarkDotNet=v0.13.1.1845-nightly, OS=Windows 11 (10.0.22000.856/21H2) -AMD Ryzen Threadripper PRO 3945WX 12-Cores, 1 CPU, 24 logical and 12 physical cores -.NET SDK=7.0.100-rc.1.22423.16 - [Host] : .NET 7.0.0 (7.0.22.42223), X64 RyuJIT AVX2 - Job-KDVXET : .NET 7.0.0 (7.0.22.42223), X64 RyuJIT AVX2 - Job-HFRAGK : .NET 7.0.0-rc.1.22424.9, X64 NativeAOT AVX2 -``` - -| Method | Runtime | Mean | Ratio | Allocated | -|-------------- |-------------- |---------:|------:|----------:| -| BinaryTrees_2 | .NET 7.0 | 95.06 ms | 1.00 | 227.33 MB | -| BinaryTrees_2 | NativeAOT 7.0 | 90.32 ms | 0.96 | 227.33 MB | - - -Some of .NET features are not supported by Native AOT, that is why you may want to filter them out using new `[AotFilter]` attribute: - -```cs -[AotFilter("Currently not supported due to missing metadata.")] -public class Xml_FromStream -``` - -## New features and improvements - -### Hiding Columns - -In [#1621](https://github.com/dotnet/BenchmarkDotNet/pull/1621) [@marcnet80](https://github.com/marcnet80) has reduced the number of columns displayed when multiple runtimes are being compared. - -![](https://user-images.githubusercontent.com/6011991/99952390-defab180-2d7f-11eb-851c-ffef6fbb0647.png#mid) - -In [#1890](https://github.com/dotnet/BenchmarkDotNet/pull/1890) [@YegorStepanov](https://github.com/YegorStepanov) has implemented a set of new APIs that allow for hiding columns. It's also exposed via `-h` and `--hide` command line arguments. - -```cs -[MemoryDiagnoser] // adds Gen0, Gen1, Gen2 and Allocated Bytes columns -[HideColumns(Column.Gen0, Column.Gen1, Column.Gen2)] // dont display GenX columns -public class IntroHidingColumns -{ - [Benchmark] - public byte[] AllocateArray() => new byte[100_000]; -} -``` - -Sample results without `[HideColumns]`: - - -| Method | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------------- |---------:|----------:|----------:|--------:|--------:|--------:|----------:| -| AllocateArray | 3.303 us | 0.0465 us | 0.0435 us | 31.2462 | 31.2462 | 31.2462 | 97.69 KB | - -With: - -| Method | Mean | Error | StdDev | Allocated | -|-------------- |---------:|----------:|----------:|----------:| -| AllocateArray | 3.489 us | 0.0662 us | 0.0763 us | 97.69 KB | - -Imagine how much time [@YegorStepanov](https://github.com/YegorStepanov) has saved to all the people who so far were removing the columns manually from the results before publishing them on GitHub! - -### Allocation Ratio Column - -In [#1859](https://github.com/dotnet/BenchmarkDotNet/pull/1859) [@YegorStepanov](https://github.com/YegorStepanov) has added Allocation Ratio Column. It's enabled by default when `MemoryDiagnoser` is used and one of the benchmarks is marked as `[Benchmark(Baseline = true)]` or when there are multuple jobs defined and one of them is marked as baseline. - -```cs -[MemoryDiagnoser] -public class AllocationColumnSample -{ - [Benchmark(Baseline = true)] - [Arguments("test")] - public string Builder(string value) - { - StringBuilder sb = new (value); - - for (int i = 0; i < 10; i++) - sb.Append(value); - - return sb.ToString(); - } - - [Benchmark] - [Arguments("test")] - public string Concatenation(string value) - { - string result = value; - - for (int i = 0; i < 10; i++) - result += value; - - return result; - } -} -``` - -| Method | value | Mean | Error | StdDev | Ratio | Gen 0 | Allocated | Alloc Ratio | -|-------------- |------ |---------:|--------:|--------:|------:|-------:|----------:|------------:| -| Builder | test | 127.9 ns | 0.49 ns | 0.43 ns | 1.00 | 0.0544 | 456 B | 1.00 | -| Concatenation | test | 120.2 ns | 0.94 ns | 0.88 ns | 0.94 | 0.0908 | 760 B | 1.67 | - -### Progress and estimated finish time - -In [#1909](https://github.com/dotnet/BenchmarkDotNet/pull/1909) [@adamsitnik](https://github.com/adamsitnik) has added logging of progress and estimated finish time. - -```log -// ** Remained 5211 (99.9%) benchmark(s) to run. Estimated finish 2022-08-25 22:26 (9h 7m from now) ** -``` - -### arm64 support for BenchmarkDotNet.Diagnostics.Windows package - -Due to the [update](https://github.com/dotnet/BenchmarkDotNet/pull/2030) to [TraceEvent 3.0](https://www.nuget.org/packages/Microsoft.Diagnostics.Tracing.TraceEvent) `BenchmarkDotNet.Diagnostics.Windows` package has now arm64 support. Which means that you can use `EtwProfiler` and other ETW-based diagnosers on Windows arm64. - -It would not be possible without [@brianrob](https://github.com/brianrob) who implemented arm64 support for TraceEvent in [#1533](https://github.com/microsoft/perfview/pull/1533)! - -### Hardware Intrinsics information - -In [#2051](https://github.com/dotnet/BenchmarkDotNet/pull/2051) [@adamsitnik](https://github.com/adamsitnik) has extended the hardware information printed in the Summary table with Hardware Intrinsics information. - -![](https://user-images.githubusercontent.com/6011991/180951306-bf8c9f8f-469a-4d7c-8ee1-d7fa47acc0e1.png#mid) - -Sine the space in Summary table is quite limited, we full information is printed only in the log: - -![](https://user-images.githubusercontent.com/6011991/180951531-38c6b6d0-ba6f-4766-a305-b2e306e8420b.png#mid) - -Special thanks to [@tannergooding](https://github.com/tannergooding) who provided a lot of [very valuable feedback](https://github.com/dotnet/BenchmarkDotNet/pull/2051#issuecomment-1194368152) and [@MichalPetryka](https://github.com/MichalPetryka) who contributed an improvement [#2066](https://github.com/dotnet/BenchmarkDotNet/pull/2066) for older runtimes. - -### Other improvements - -* WASM toolchain has received a lot of improvements from various .NET Team members: [#1769](https://github.com/dotnet/BenchmarkDotNet/pull/1769), [#1936](https://github.com/dotnet/BenchmarkDotNet/pull/1936), [#1938](https://github.com/dotnet/BenchmarkDotNet/pull/1938), [#1982](https://github.com/dotnet/BenchmarkDotNet/pull/1982). -* Dependencies and TFMs updates: [#1805](https://github.com/dotnet/BenchmarkDotNet/pull/1805), [#1978](https://github.com/dotnet/BenchmarkDotNet/pull/1978), [#2012](https://github.com/dotnet/BenchmarkDotNet/pull/2012), [#2019](https://github.com/dotnet/BenchmarkDotNet/pull/2019), [#2035](https://github.com/dotnet/BenchmarkDotNet/pull/2035). -* Ensure proper SummaryStyle handling implemented by [@mawosoft](https://github.com/mawosoft) in [#1828](https://github.com/dotnet/BenchmarkDotNet/pull/1828). -* Preserving `EnablePreviewFeatures` project setting which gives the possibility to benchmark preview .NET features. Implemented by [@kkokosa](https://github.com/kkokosa) in [#1842](https://github.com/dotnet/BenchmarkDotNet/pull/1842). -* CI: Using non-deprecated macOS pool on Azure Pipelines, implemented by [@akoeplinger](https://github.com/akoeplinger) in [#1847](https://github.com/dotnet/BenchmarkDotNet/pull/1847) -* CI: Updating Cake to 2.0.0, adopting frosting project style. Implemented by [@AndreyAkinshin](https://github.com/AndreyAkinshin) in [#1865](https://github.com/dotnet/BenchmarkDotNet/pull/1865). -* Detecting ReSharper's Dynamic Program Analysis. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1874](https://github.com/dotnet/BenchmarkDotNet/pull/1874). -* Preventing benchmark failure when some of the exporters fail. Implemented by [@epeshk](https://github.com/epeshk) in [#1902](https://github.com/dotnet/BenchmarkDotNet/pull/1902). -* Don't use the diagnosers when benchmarking has failed. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1903](https://github.com/dotnet/BenchmarkDotNet/pull/1903). -* Ensuring the default order of benchmarks is the same as declared in source code. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1907](https://github.com/dotnet/BenchmarkDotNet/pull/1907). -* Making `BuildTimeout` configurable. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1906](https://github.com/dotnet/BenchmarkDotNet/pull/1906). -* Notify users about private methods with Setup/Cleanup attributes. Implemented by [@epeshk](https://github.com/epeshk) in [#1912](https://github.com/dotnet/BenchmarkDotNet/pull/1912). -* Don't run Roslyn Analyzers for the generated code. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1917](https://github.com/dotnet/BenchmarkDotNet/pull/1917). -* Ensure `WorkloadActionUnroll` and similar are optimized if possible. Implemented by [@AndyAyersMS](https://github.com/AndyAyersMS) in [#1935](https://github.com/dotnet/BenchmarkDotNet/pull/1935). -* Don't use blocking acknowledgments when there is no need to. Implemented by [@adamsitnik](https://github.com/adamsitnik) in [#1940](https://github.com/dotnet/BenchmarkDotNet/pull/1940). -* Executor: Don't use Process.ExitCode, unless the process has exited. Implemented by [@radical](https://github.com/radical) in [#1947](https://github.com/dotnet/BenchmarkDotNet/pull/1947). -* Revise heuristic for initial jitting. Implemented by [@AndyAyersMS](https://github.com/AndyAyersMS) in [#1949](https://github.com/dotnet/BenchmarkDotNet/pull/1949). -* Allow logging build commands output. Implemented by [@radical](https://github.com/radical) in [#1950](https://github.com/dotnet/BenchmarkDotNet/pull/1950). -* Change Mono AOT mode to Normal AOT with LLVM JIT fall back. Implemented by [@fanyang-mono](https://github.com/fanyang-mono) in [#1990](https://github.com/dotnet/BenchmarkDotNet/pull/1990). - -### Glob filters support for DisassemblyDiagnoser - -So far, the disassembler was always loading the type that was generated by BDN, searching for the benchmark method, disassembling it and when encountered direct method calls, disassembling the called methods as well (if their depth was lesser or equal to max configured depth). - -This was working fine, but only for direct method calls. For indirect, the disassembly was incomplete. - -In [#2072](https://github.com/dotnet/BenchmarkDotNet/pull/2072) [@adamsitnik](https://github.com/adamsitnik) has added the possibility to filter methods disassembled by the `DisassemblyDiagnoser`. - -The users can now pass `--disasmFilter $globPattern` and it's going to be applied to full signatures of all methods available for disassembling. Examples: -* `--disasmFilter *System.Text*` - disassemble all `System.Text` methods. -* `--disasmFilter *` - disassemble all possible methods. - -Moreover, [ClrMD](https://github.com/microsoft/clrmd) was updated to v2 ([#2040](https://github.com/dotnet/BenchmarkDotNet/pull/2040)) and few disassembler bugs have been fixed ([#2075](https://github.com/dotnet/BenchmarkDotNet/pull/2075), [#2078](https://github.com/dotnet/BenchmarkDotNet/pull/2078)). We are expecting that the disassembler will be more reliable now. - -### Docs and Samples improvements - -Big thanks to [@SnakyBeaky](https://github.com/SnakyBeaky), [@Distinctlyminty](https://github.com/Distinctlyminty), [@asaf92](https://github.com/asaf92), [@adamsitnik](https://github.com/adamsitnik) and [@eiriktsarpalis](https://github.com/eiriktsarpalis) who have improved our docs, samples and error messages! - -[#1776](https://github.com/dotnet/BenchmarkDotNet/pull/1776), [#1797](https://github.com/dotnet/BenchmarkDotNet/pull/1797), [#1850](https://github.com/dotnet/BenchmarkDotNet/pull/1850), [#1861](https://github.com/dotnet/BenchmarkDotNet/pull/1861), [#1939](https://github.com/dotnet/BenchmarkDotNet/pull/1939), [#1974](https://github.com/dotnet/BenchmarkDotNet/pull/1974), [#1997](https://github.com/dotnet/BenchmarkDotNet/pull/1997), [#2042](https://github.com/dotnet/BenchmarkDotNet/pull/2042), [#2050](https://github.com/dotnet/BenchmarkDotNet/pull/2050), [#2068](https://github.com/dotnet/BenchmarkDotNet/pull/2068). - -## Bug fixes - -* WASM: [#1811](https://github.com/dotnet/BenchmarkDotNet/pull/1811), [#1846](https://github.com/dotnet/BenchmarkDotNet/pull/1846), [#1916](https://github.com/dotnet/BenchmarkDotNet/pull/1916), [#1926](https://github.com/dotnet/BenchmarkDotNet/pull/1926), [#1932](https://github.com/dotnet/BenchmarkDotNet/pull/1932). -* Diagnoser-provided Analysers weren't automatically added to Config. Fixed by [@mawosoft](https://github.com/mawosoft) in [#1790](https://github.com/dotnet/BenchmarkDotNet/pull/1790). -* Exportes could been duplicated. Fixed by [@workgroupengineering](https://github.com/workgroupengineering) in [#1796](https://github.com/dotnet/BenchmarkDotNet/pull/1796). -* Small bug in SummaryStyle. Fixed by [@mawosoft](https://github.com/mawosoft) in [#1801](https://github.com/dotnet/BenchmarkDotNet/pull/1801). -* `InvalidOperationException/NullReferenceException` in `SmartParaemter`. Fixed by [@mawosoft](https://github.com/mawosoft) in [#1810](https://github.com/dotnet/BenchmarkDotNet/pull/1810). -* Failures caused by colons in benchmark name. Fixed by [@ronbrogan](https://github.com/ronbrogan) in [#1823](https://github.com/dotnet/BenchmarkDotNet/pull/1823). -* Some benchmark arugments were not properly escaped and were causing process launcher to crush. Fixed by [@adamsitnik](https://github.com/adamsitnik) in [#1841](https://github.com/dotnet/BenchmarkDotNet/pull/1841) -* Invalid size specifiers for Memory and Disassembly diagnosers. Fixed by [@YegorStepanov](https://github.com/YegorStepanov) in [#1854](https://github.com/dotnet/BenchmarkDotNet/pull/1854) and [#1855](https://github.com/dotnet/BenchmarkDotNet/pull/1855). -* Respect LogicalGroup order in DefaultOrderer. Fixed by [@AndreyAkinshin](https://github.com/AndreyAkinshin) in [#1866](https://github.com/dotnet/BenchmarkDotNet/pull/1866). -* Endless loop in user interaction with redirected input. Fixed by [@tmds](https://github.com/tmds) in [#](https://github.com/dotnet/BenchmarkDotNet/pull/1870). -* Broken power plan support. Fixed by [@YegorStepanov](https://github.com/YegorStepanov) in [#1885](https://github.com/dotnet/BenchmarkDotNet/pull/1885). -* `BytesAllocatedPerOperation` was not being output by the JSON and XML exporters. Fixed by [#martincostello](https://github.com/martincostello) in [#1919](https://github.com/dotnet/BenchmarkDotNet/pull/1919). -* Incorrect default InvocationCount in the summary table. Fixed by [@AndreyAkinshin](https://github.com/AndreyAkinshin) in [#1929](https://github.com/dotnet/BenchmarkDotNet/issues/1929). -* Failed build output was printed in reverse order. Fixed by [@radical](https://github.com/radical) in [#1945](https://github.com/dotnet/BenchmarkDotNet/pull/1945). -* Build failures due to `NETSDK1150`. Fixed by [@OlegOLK](https://github.com/OlegOLK) in [#1981](https://github.com/dotnet/BenchmarkDotNet/pull/1981). -* `MetricCoumn` was not respecting provided units when formatting values. Fixed by [@mawosoft](https://github.com/mawosoft) in [#2033](https://github.com/dotnet/BenchmarkDotNet/pull/2033). -* Generating invalid code that was causing benchmark failures. Fixed by [@mawosoft](https://github.com/mawosoft) in [#2041](https://github.com/dotnet/BenchmarkDotNet/pull/2041). -* CI: non-master build branches were publishing artifacts to the CI feed. Fixed by [@mawosoft](https://github.com/mawosoft) in [#2047](https://github.com/dotnet/BenchmarkDotNet/pull/2047). -* Comments in the project files were causing build failures. Fixed by [@mawosoft](https://github.com/mawosoft) in [#2056](https://github.com/dotnet/BenchmarkDotNet/pull/2056). diff --git a/docs/_changelog/header/v0.13.3.md b/docs/_changelog/header/v0.13.3.md deleted file mode 100644 index db1514edf6..0000000000 --- a/docs/_changelog/header/v0.13.3.md +++ /dev/null @@ -1,91 +0,0 @@ -## Highlights - -* New supported technologies - * Add arm64 disassembler - [#1422](https://github.com/dotnet/BenchmarkDotNet/issues/1422) - [#2127](https://github.com/dotnet/BenchmarkDotNet/pull/2127) - [#2107](https://github.com/dotnet/BenchmarkDotNet/pull/2107) - [#2123](https://github.com/dotnet/BenchmarkDotNet/pull/2123) - [#2070](https://github.com/dotnet/BenchmarkDotNet/issues/2070) - [#2118](https://github.com/dotnet/BenchmarkDotNet/pull/2118) - [#2119](https://github.com/dotnet/BenchmarkDotNet/pull/2119) - [#2234](https://github.com/dotnet/BenchmarkDotNet/pull/2234) - [#2222](https://github.com/dotnet/BenchmarkDotNet/pull/2222) - [#2212](https://github.com/dotnet/BenchmarkDotNet/pull/2212) - [9ee1/Capstone.NET#37](https://github.com/9ee1/Capstone.NET/pull/37) - * Initial .NET 8 support - [#2192](https://github.com/dotnet/BenchmarkDotNet/pull/2192) - * .NET 6/7 MonoVM support - [#2064](https://github.com/dotnet/BenchmarkDotNet/issues/2064) - [#2142](https://github.com/dotnet/BenchmarkDotNet/pull/2142) - [#2227](https://github.com/dotnet/BenchmarkDotNet/pull/2227) - [#2230](https://github.com/dotnet/BenchmarkDotNet/pull/2230) - * Armv6 and Ppc64le architectures support - [#2216](https://github.com/dotnet/BenchmarkDotNet/issues/2216) - [#2219](https://github.com/dotnet/BenchmarkDotNet/pull/2219) -* Improved support - * Improved WASM support - [#2201](https://github.com/dotnet/BenchmarkDotNet/pull/2201) - [#2099](https://github.com/dotnet/BenchmarkDotNet/issues/2099) - [#2154](https://github.com/dotnet/BenchmarkDotNet/pull/2154) - [#2112](https://github.com/dotnet/BenchmarkDotNet/pull/2112) - * Improved NativeAOT support - [#2095](https://github.com/dotnet/BenchmarkDotNet/pull/2095) - [#2221](https://github.com/dotnet/BenchmarkDotNet/pull/2221) - * Improved Android support - [#2231](https://github.com/dotnet/BenchmarkDotNet/pull/2231) - * 32-bit benchmarks can now handle addresses larger than 2GB with the help of `LargeAddressAware` - [#1469](https://github.com/dotnet/BenchmarkDotNet/issues/1469) - [#2145](https://github.com/dotnet/BenchmarkDotNet/pull/2145) - * Support 64bit affinity masks - [#2211](https://github.com/dotnet/BenchmarkDotNet/issues/2211) - [#2228](https://github.com/dotnet/BenchmarkDotNet/pull/2228) -* New features - * Add `ExceptionDiagnoser` - [#1736](https://github.com/dotnet/BenchmarkDotNet/issues/1736) - [#2169](https://github.com/dotnet/BenchmarkDotNet/pull/2169) - [#2182](https://github.com/dotnet/BenchmarkDotNet/pull/2182) - * Add `PerfCollectProfiler` - [#2117](https://github.com/dotnet/BenchmarkDotNet/pull/2117) - * Incremental benchmark execution with the help of `--resume` - [#1799](https://github.com/dotnet/BenchmarkDotNet/issues/1799) - [#2164](https://github.com/dotnet/BenchmarkDotNet/pull/2164) - * Taskbar progress - [#2102](https://github.com/dotnet/BenchmarkDotNet/issues/2102) - [#2158](https://github.com/dotnet/BenchmarkDotNet/pull/2158) - [#2140](https://github.com/dotnet/BenchmarkDotNet/pull/2140) - * Support `--noForcedGCs` to avoid forced GC between benchmark runs - [#2101](https://github.com/dotnet/BenchmarkDotNet/pull/2101) - * Added apples to apples comparison mode - [#2116](https://github.com/dotnet/BenchmarkDotNet/pull/2116) - [#2193](https://github.com/dotnet/BenchmarkDotNet/pull/2193) - * Communication between the host process and the benchmark process is now using pipes instead of output parsing - [#2092](https://github.com/dotnet/BenchmarkDotNet/pull/2092) - [#1933](https://github.com/dotnet/BenchmarkDotNet/issues/1933) - [#2189](https://github.com/dotnet/BenchmarkDotNet/issues/2189) - [#2207](https://github.com/dotnet/BenchmarkDotNet/pull/2207) - [#2200](https://github.com/dotnet/BenchmarkDotNet/pull/2200) -* Dozens of bugfixes - -## Special Thanks - -We would like to highlight some important contributors who helped us with this release: - -1. [OpenHack'22 (devexperts.com)](https://code.devexperts.com/event/openhack22) hackathon sponsored by the DevExperts company. - As part of this hackathon we have received following PRs: - * [#2132](https://github.com/dotnet/BenchmarkDotNet/pull/2132) fix: include argument and param names in --filter (by [@blouflashdb](https://github.com/blouflashdb)) - * [#2140](https://github.com/dotnet/BenchmarkDotNet/pull/2140) Update console title with benchmark information (by [@franciscomoloureiro](https://github.com/franciscomoloureiro)) - * [#2142](https://github.com/dotnet/BenchmarkDotNet/pull/2142) Issue 2064: Mono70 moniker (by [@Serg046](https://github.com/Serg046)) - * [#2148](https://github.com/dotnet/BenchmarkDotNet/pull/2148) adding validation errors when the benchmarks are unsupported (by [@emanuel-v-r](https://github.com/emanuel-v-r)) - * [#2160](https://github.com/dotnet/BenchmarkDotNet/pull/2160) Corrected logic to restore foreground color in ConsoleLogger.cs (by [@farQtech](https://github.com/farQtech)) - * [#2164](https://github.com/dotnet/BenchmarkDotNet/pull/2164) 1799 adding resume arg (by [@melias](https://github.com/melias)) - * [#2169](https://github.com/dotnet/BenchmarkDotNet/pull/2169) Issue #1736: Add ExceptionDiagnoser (by [@Serg046](https://github.com/Serg046)) - * [#2161](https://github.com/dotnet/BenchmarkDotNet/pull/2161) add quiet logger (by [@franciscomoloureiro](https://github.com/franciscomoloureiro)) **(not merged yet)** - * [#2171](https://github.com/dotnet/BenchmarkDotNet/pull/2171) Issue #1024: Calculate baseline by the fastest benchmark (by [@Serg046](https://github.com/Serg046)) **(not merged yet)** - -2. Jan Vorlicek helped to implement arm64 disassembler during an internal Microsoft Hackathon: - * [#2107](https://github.com/dotnet/BenchmarkDotNet/pull/2107) Implement TryGetReferencedAddress for relative branches (by [@janvorli](https://github.com/janvorli)) - * [#2123](https://github.com/dotnet/BenchmarkDotNet/pull/2123) Added other arm64 constant form extraction plus other changes (by [@janvorli](https://github.com/janvorli)) - -3. Ahmed Garhy (maintainer of Capstone.NET) helped to improve Capstone.NET, which was need to implement arm64 disassembler: - * [9ee1/Capstone.NET#37](https://github.com/9ee1/Capstone.NET/pull/37) Sign Assembly with a Strong Name (by [@9ee1](https://github.com/9ee1)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.4.md b/docs/_changelog/header/v0.13.4.md deleted file mode 100644 index 3ae8407ee4..0000000000 --- a/docs/_changelog/header/v0.13.4.md +++ /dev/null @@ -1,26 +0,0 @@ -## Highlights - -* Fixed LINQPad support - [#2237](https://github.com/dotnet/BenchmarkDotNet/issues/2237) - [#2238](https://github.com/dotnet/BenchmarkDotNet/pull/2238) -* New `JitStatsDiagnoser` - [#2243](https://github.com/dotnet/BenchmarkDotNet/pull/2243) -* Minor documentation improvements - [#2206](https://github.com/dotnet/BenchmarkDotNet/pull/2206) - [#2218](https://github.com/dotnet/BenchmarkDotNet/pull/2218) - -## JitStatsDiagnoser - -This new diagnoser introduced in ([#2243](https://github.com/dotnet/BenchmarkDotNet/pull/2243)) allows getting advanced JIT statistics. - -Sample usage: - -```cmd -dotnet run -c Release -f net7.0 --filter *IntroBasic.Sleep --profiler jit -``` - -Result: - -| Method | Mean | Error | StdDev | Methods JITted | Methods Tiered | JIT allocated memory | -|------- |---------:|---------:|---------:|---------------:|---------------:|---------------------:| -| Sleep | 15.53 ms | 0.034 ms | 0.032 ms | 1,102 | 15 | 221,736 B | \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.5.md b/docs/_changelog/header/v0.13.5.md deleted file mode 100644 index c3915a4f6d..0000000000 --- a/docs/_changelog/header/v0.13.5.md +++ /dev/null @@ -1,34 +0,0 @@ -## Highlights - -* Improved `JitStatsDiagnoser`. - This diagnoser was added in v0.13.4, it shows various stats from the JIT compiler that were collected during entire benchmark run - (amount of JITted methods, - amount of [tiered methods](https://learn.microsoft.com/en-us/dotnet/core/whats-new/dotnet-core-3-0#tiered-compilation), - how much memory JIT allocated during the benchmark). - In this release, we improved metric collection - ([#2246](https://github.com/dotnet/BenchmarkDotNet/pull/2246), - [e715d5](https://github.com/dotnet/BenchmarkDotNet/commit/e715d5bb63984fca65120d9a497f7d16395f9e5b)) - and added the `[JitStatsDiagnoser]` attribute - ([#2250](https://github.com/dotnet/BenchmarkDotNet/pull/2250) - [512413](https://github.com/dotnet/BenchmarkDotNet/commit/512413ceb24077154bdf6d6306138accffe64c7a)). -* Enable strong-named assemblies in the released NuGet packages - [#2258](https://github.com/dotnet/BenchmarkDotNet/issues/2258) - [#2263](https://github.com/dotnet/BenchmarkDotNet/pull/2263) - [5cd288](https://github.com/dotnet/BenchmarkDotNet/commit/5cd288996ca13292fcf638be299c097a600aea7b) -* Avoid keeping referenced values returned from a benchmark in memory - [#1942](https://github.com/dotnet/BenchmarkDotNet/issues/1942) - [#2191](https://github.com/dotnet/BenchmarkDotNet/pull/2191) - [ff5dbe](https://github.com/dotnet/BenchmarkDotNet/commit/ff5dbe662478f547e4be8d734eaeb6a106f40875) -* Keep generated files when MSBuild bin log is requested - [#2252](https://github.com/dotnet/BenchmarkDotNet/issues/2252) - [#2254](https://github.com/dotnet/BenchmarkDotNet/pull/2254) - [d3fbc0](https://github.com/dotnet/BenchmarkDotNet/commit/d3fbc03d6dabeb52f23c6b7e50287150e66957cc) -* Add `Id` for `UnresolvedDiagnoser` (an exception fix) - [#2251](https://github.com/dotnet/BenchmarkDotNet/pull/2251) - [a992b5](https://github.com/dotnet/BenchmarkDotNet/commit/a992b57490e844acf587bc2e01b08a7040dbc8e2) -* Add brand names for Windows 22H2 and macOS 13 - [86f212](https://github.com/dotnet/BenchmarkDotNet/commit/86f212b79e297d87d3942e4c50130fe6e214f3c8) - [0c2699](https://github.com/dotnet/BenchmarkDotNet/commit/0c26996ea685a99068aca71e7ae547b0851d3c64) -* Remove deprecated `InProcessToolchain` - [#2248](https://github.com/dotnet/BenchmarkDotNet/pull/2248) - [615384](https://github.com/dotnet/BenchmarkDotNet/commit/615384d2553434d7f35c03ab3174d761f82c6c2d) diff --git a/docs/_changelog/header/v0.13.6.md b/docs/_changelog/header/v0.13.6.md deleted file mode 100644 index 7667d404ce..0000000000 --- a/docs/_changelog/header/v0.13.6.md +++ /dev/null @@ -1,23 +0,0 @@ -## Highlights - -* New [BenchmarkDotNet.Diagnostics.dotTrace](https://www.nuget.org/packages/BenchmarkDotNet.Diagnostics.dotTrace) NuGet package. - Once this package is installed, you can annotate your benchmarks with the `[DotTraceDiagnoser]` and get a [dotTrace](https://www.jetbrains.com/profiler/) performance snapshot at the end of the benchmark run. - [#2328](https://github.com/dotnet/BenchmarkDotNet/pull/2328) -* Updated documentation website. - We migrated to [docfx](https://dotnet.github.io/docfx/) 2.67 and got the refreshed modern template based on bootstrap 5 with dark/light theme switcher. -* Updated [BenchmarkDotNet.Templates](https://www.nuget.org/packages/BenchmarkDotNet.Templates). - Multiple issues were resolved, now you can create new benchmark projects from terminal or your favorite IDE. - [#1658](https://github.com/dotnet/BenchmarkDotNet/issues/1658) - [#1881](https://github.com/dotnet/BenchmarkDotNet/issues/1881) - [#2149](https://github.com/dotnet/BenchmarkDotNet/issues/2149) - [#2338](https://github.com/dotnet/BenchmarkDotNet/pull/2338) -* Response file support. - Now it's possible to pass additional arguments to BenchmarkDotNet using `@filename` syntax. - [#2320](https://github.com/dotnet/BenchmarkDotNet/pull/2320) - [#2348](https://github.com/dotnet/BenchmarkDotNet/pull/2348) -* Custom runtime support. - [#2285](https://github.com/dotnet/BenchmarkDotNet/pull/2285) -* Introduce CategoryDiscoverer, see [`IntroCategoryDiscoverer`](xref:BenchmarkDotNet.Samples.IntroCategoryDiscoverer). - [#2306](https://github.com/dotnet/BenchmarkDotNet/issues/2306) - [#2307](https://github.com/dotnet/BenchmarkDotNet/pull/2307) -* Multiple bug fixes. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.7.md b/docs/_changelog/header/v0.13.7.md deleted file mode 100644 index 5a8487b639..0000000000 --- a/docs/_changelog/header/v0.13.7.md +++ /dev/null @@ -1,3 +0,0 @@ -## Highlights - -This release contains important bug fixes listed below. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.8.md b/docs/_changelog/header/v0.13.8.md deleted file mode 100644 index b9222853e2..0000000000 --- a/docs/_changelog/header/v0.13.8.md +++ /dev/null @@ -1,3 +0,0 @@ -## Highlights - -This release contains important bug fixes. \ No newline at end of file diff --git a/docs/_changelog/header/v0.13.9.md b/docs/_changelog/header/v0.13.9.md deleted file mode 100644 index 07e4813c9f..0000000000 --- a/docs/_changelog/header/v0.13.9.md +++ /dev/null @@ -1,3 +0,0 @@ -## Highlights - -This release contains bug fixes. \ No newline at end of file diff --git a/docs/_changelog/header/v0.14.0.md b/docs/_changelog/header/v0.14.0.md deleted file mode 100644 index 1c2f6693ce..0000000000 --- a/docs/_changelog/header/v0.14.0.md +++ /dev/null @@ -1,15 +0,0 @@ -## Highlights - -* Introduce `BenchmarkDotNet.Diagnostics.dotMemory` [#2549](https://github.com/dotnet/BenchmarkDotNet/pull/2549): memory allocation profile of your benchmarks using [dotMemory](https://www.jetbrains.com/dotmemory/), see @BenchmarkDotNet.Samples.IntroDotMemoryDiagnoser -* Introduce `BenchmarkDotNet.Exporters.Plotting` [#2560](https://github.com/dotnet/BenchmarkDotNet/pull/2560): plotting via [ScottPlot](https://scottplot.net/) (initial version) -* Multiple bugfixes -* The default build toolchains have been updated to pass `IntermediateOutputPath`, `OutputPath`, and `OutDir` properties to the `dotnet build` command. This change forces all build outputs to be placed in a new directory generated by BenchmarkDotNet, and fixes many issues that have been reported with builds. You can also access these paths in your own `.csproj` and `.props` from those properties if you need to copy custom files to the output. - -## Bug fixes - -* Fixed multiple build-related bugs including passing MsBuildArguments and .Net 8's `UseArtifactsOutput`. - -## Breaking Changes - -* `DotNetCliBuilder` removed `retryFailedBuildWithNoDeps` constructor option. -* `DotNetCliCommand` removed `RetryFailedBuildWithNoDeps` property and `BuildNoRestoreNoDependencies()` and `PublishNoBuildAndNoRestore()` methods (replaced with `PublishNoRestore()`). \ No newline at end of file diff --git a/docs/_changelog/header/v0.15.0.md b/docs/_changelog/header/v0.15.0.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/docs/_changelog/header/v0.8.2.md b/docs/_changelog/header/v0.8.2.md deleted file mode 100644 index b06d5ec1c2..0000000000 --- a/docs/_changelog/header/v0.8.2.md +++ /dev/null @@ -1,19 +0,0 @@ -* Changes in the Summary table - * Summary table now supports ResultExtenders that can add new column to the table - * Now we use [StandardError](https://en.wikipedia.org/wiki/Standard_error) (aka `Error`) as the main accuracy metric - * Columns `op/s`, `StdDev` are disabled by default (you can add it via ResultExtenders) -* Statistic improvements, now you have detailed statistic in the console log like follows: -``` -Mean = 118.5298 us, StdError = 1.2863 us (N = 30, StdDev = 7.0454 us) -Min = 109.1602 us, Median = 117.1794 us, Max = 132.5764 us -IQR = 10.1244 us, LowerFence = 98.0834 us, UpperFence = 138.5810 us -ConfidenceInterval = [116.0086 us; 121.0510 us] (CI 95%) -``` -* Added the `Baseline` feature, see [#64](https://github.com/PerfDotNet/BenchmarkDotNet/issues/64) -* Export improvements, now you have files `-report-github.md`, `-report-stackoverflow.md` for easy publishing results on GitHub and StackOverflow. -* Basic plotting. Added new `BenchmarkRPlotExporter` that creates `BuildPlots.R` in the bin directory. It is an R script that generates boxplot and barplot for your benchmarks (you should have installed R with defined `R_HOME` environment variable) -* Updated environment info - * Added Stopwatch `Frequency` and `Resolution` - * Split common benchmark properties (like `Mode`, `Platform`, `Runtime`) in several lines (3 properties per line) -* Log improvements: add total time, statistics, list of exported files -* Bug fixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.0.md b/docs/_changelog/header/v0.9.0.md deleted file mode 100644 index 3d14c4b30c..0000000000 --- a/docs/_changelog/header/v0.9.0.md +++ /dev/null @@ -1,2 +0,0 @@ -* New API -* Autodetermination of amount iteration for warmup/target idle/main iterations, duration of iteration, amount of CLR launches. \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.2.md b/docs/_changelog/header/v0.9.2.md deleted file mode 100644 index 44d8bb5fe0..0000000000 --- a/docs/_changelog/header/v0.9.2.md +++ /dev/null @@ -1 +0,0 @@ -* Dnx451 support (Closed [#51](https://github.com/PerfDotNet/BenchmarkDotNet/issues/51), Merged [#87](https://github.com/PerfDotNet/BenchmarkDotNet/issues/87)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.3.md b/docs/_changelog/header/v0.9.3.md deleted file mode 100644 index f7614fbe00..0000000000 --- a/docs/_changelog/header/v0.9.3.md +++ /dev/null @@ -1 +0,0 @@ -* CoreCLR support (Closed [#52](https://github.com/PerfDotNet/BenchmarkDotNet/issues/52), Merged [#113](https://github.com/PerfDotNet/BenchmarkDotNet/issues/113)) \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.4.md b/docs/_changelog/header/v0.9.4.md deleted file mode 100644 index 63c47b49d0..0000000000 --- a/docs/_changelog/header/v0.9.4.md +++ /dev/null @@ -1,9 +0,0 @@ -* Improved messages about error in benchmarks, see [#104](https://github.com/PerfDotNet/BenchmarkDotNet/issues/104) -* Natural sort order, see [#92](https://github.com/PerfDotNet/BenchmarkDotNet/issues/92), [#95](https://github.com/PerfDotNet/BenchmarkDotNet/issues/95), [#97](https://github.com/PerfDotNet/BenchmarkDotNet/issues/97) -* Improved `double`/`float`/`decimal`/`enum` support for Params, see [#96](https://github.com/PerfDotNet/BenchmarkDotNet/issues/96), [#105](https://github.com/PerfDotNet/BenchmarkDotNet/issues/105), [#116](https://github.com/PerfDotNet/BenchmarkDotNet/issues/116) -* Now environment info includes information about `HardwareTimerKind` and `JitModules` -* Added `DryConfig` -* Improved export performance, closed [#119](https://github.com/PerfDotNet/BenchmarkDotNet/issues/119), merged [#124](https://github.com/PerfDotNet/BenchmarkDotNet/pull/124) -* Better cmd-line discoverability (see [#78](https://github.com/PerfDotNet/BenchmarkDotNet/issues/78)), e.g. run `Benchmark.exe --help` and some useful information will be printed -* Supporting all kinds of references for generated project (exact version, custom paths, GAC, sub-folders, dependent assemblies not copied), closed [#41](https://github.com/PerfDotNet/BenchmarkDotNet/issues/41), [#49](https://github.com/PerfDotNet/BenchmarkDotNet/issues/49), [#72](https://github.com/PerfDotNet/BenchmarkDotNet/issues/72), [#123](https://github.com/PerfDotNet/BenchmarkDotNet/issues/123), merged [#125](https://github.com/PerfDotNet/BenchmarkDotNet/pull/125) -* Friendliness to LinqPad restored, closed [#66](https://github.com/PerfDotNet/BenchmarkDotNet/issues/66), merged [#125](https://github.com/PerfDotNet/BenchmarkDotNet/pull/125) \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.5.md b/docs/_changelog/header/v0.9.5.md deleted file mode 100644 index 1d429dcc35..0000000000 --- a/docs/_changelog/header/v0.9.5.md +++ /dev/null @@ -1,11 +0,0 @@ -* Added validators, JitOptimizationsValidator detects all non-optimzied dlls that were referenced [#134](https://github.com/PerfDotNet/BenchmarkDotNet/issues/134) -* **Strong naming** [#101](https://github.com/PerfDotNet/BenchmarkDotNet/issues/101) -* Add `IOrderProvider` [#107](https://github.com/PerfDotNet/BenchmarkDotNet/issues/107) -* **Putting all the generated artifacts in a separate folder: ./BenchmarkDotNet.Artifacts/results** and ./BenchmarkDotNet.Artifacts/bin [#94](https://github.com/PerfDotNet/BenchmarkDotNet/issues/94) -* Printing dotnet cli version for .NET Core and Dnx451, informing user when not installed. Closed [#128](https://github.com/PerfDotNet/BenchmarkDotNet/issues/128) -* Supporting assembly redirects [#67](https://github.com/PerfDotNet/BenchmarkDotNet/issues/67) -* Changed used msbuild version: 12 for .NET 4.5 (VS 2013), 14 for .NET 4.6 (VS 2015). Closed [#132](https://github.com/PerfDotNet/BenchmarkDotNet/issues/132) and [#137](https://github.com/PerfDotNet/BenchmarkDotNet/issues/137) -* Switched to new ‘dotnet’ target framework monikers (dotnet5.4 instead of dnxcore50), [why](https://github.com/aspnet/Announcements/issues/98) -* dnx452, dnx46, net462 support added -* Executing single Benchmark for multiple Runtimes also with Diagnoser attached (see [#117](https://github.com/PerfDotNet/BenchmarkDotNet/pull/117)) -* Misc minor changes \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.6.md b/docs/_changelog/header/v0.9.6.md deleted file mode 100644 index b69b6591bd..0000000000 --- a/docs/_changelog/header/v0.9.6.md +++ /dev/null @@ -1,3 +0,0 @@ -* Added Percentiles (see [#164](https://github.com/PerfDotNet/BenchmarkDotNet/pull/164)) -* Added support for Json export (see [#84](https://github.com/PerfDotNet/BenchmarkDotNet/issues/84)) -* Bugfixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.7.md b/docs/_changelog/header/v0.9.7.md deleted file mode 100644 index 560d956baf..0000000000 --- a/docs/_changelog/header/v0.9.7.md +++ /dev/null @@ -1,2 +0,0 @@ -* .NET Core RC2 support (see [#187](https://github.com/PerfDotNet/BenchmarkDotNet/pull/187)) -* Bugfixes \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.8.md b/docs/_changelog/header/v0.9.8.md deleted file mode 100644 index 055ca19bf6..0000000000 --- a/docs/_changelog/header/v0.9.8.md +++ /dev/null @@ -1,8 +0,0 @@ -* CoreCLR RTM support (see [#216](https://github.com/PerfDotNet/BenchmarkDotNet/issues/216)). **Breaking change:** we have dropped dnx451 and dnxcore50 support. -* Migration from MSBuild to Roslyn, which supports Mono on Linux and MacOS (see [#149](https://github.com/PerfDotNet/BenchmarkDotNet/issues/149)). **Breaking change:** we have dropped .NET 4.0 support. -* Ability to manage GC mode: turn on/off the Server/Concurrent GC modes, extend to CPU groups, set gcAllowVeryLargeObjects and avoid BenchmarkDotNet from forcing GC.Collect (see [#188](https://github.com/PerfDotNet/BenchmarkDotNet/issues/188), [#76](https://github.com/PerfDotNet/BenchmarkDotNet/issues/76) and [#211](https://github.com/PerfDotNet/BenchmarkDotNet/issues/211)) -* Support CopyToOutput (see [#212](https://github.com/PerfDotNet/BenchmarkDotNet/issues/212)). Now you can use native dependencies as well as custom files. -* Copying custom settings from app.config (see [#108](https://github.com/PerfDotNet/BenchmarkDotNet/issues/108)). It means we support assembly binding redirects as well as custom connection strings etc. -* AsciiDocExporter (see [#169](https://github.com/PerfDotNet/BenchmarkDotNet/pull/169)) -* Framework setting in Job has been removed (see [#194](https://github.com/PerfDotNet/BenchmarkDotNet/issues/194)) -* Minor bugfixes and improvements \ No newline at end of file diff --git a/docs/_changelog/header/v0.9.9.md b/docs/_changelog/header/v0.9.9.md deleted file mode 100644 index a3352ca136..0000000000 --- a/docs/_changelog/header/v0.9.9.md +++ /dev/null @@ -1,9 +0,0 @@ -* Attribute config style (see [#166](https://github.com/PerfDotNet/BenchmarkDotNet/issues/166)) -* [Online documentation](https://perfdotnet.github.io/BenchmarkDotNet/index.htm) (see [#219](https://github.com/PerfDotNet/BenchmarkDotNet/issues/219)) -* Mono LLVM support (see [#226](https://github.com/PerfDotNet/BenchmarkDotNet/issues/226)) -* Async method support (see [#236](https://github.com/PerfDotNet/BenchmarkDotNet/issues/236)) -* NuGet packages and repo layout restructuring (see [#225](https://github.com/PerfDotNet/BenchmarkDotNet/issues/225), [#228](https://github.com/PerfDotNet/BenchmarkDotNet/issues/228)) -* `[Cleanup]` attribute (see [#215](https://github.com/PerfDotNet/BenchmarkDotNet/issues/215)) -* New statistics columns: `Skewness`, `Kurtosis`, `WelchTTestPValue`, Improved math for the `Scaled` column -* Now current default branch is `master` -* Minor improvements and bug fixes \ No newline at end of file From c9548fa72649c7286555ec9402fbf4893f38cd8c Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Wed, 11 Jun 2025 04:43:53 +0900 Subject: [PATCH 04/15] chore: fix benchmark testadapter related issues (#2766) --- .../BenchmarkDotNet.Samples.csproj | 5 ++-- samples/BenchmarkDotNet.Samples/IntroNuGet.cs | 4 ++-- .../BenchmarkEnumerator.cs | 24 ++++++++++++++++++- ...ts.ManualRunning.MultipleFrameworks.csproj | 1 - .../BenchmarkDotNet.IntegrationTests.csproj | 4 ++-- .../BenchmarkDotNet.Tests.csproj | 1 - 6 files changed, 29 insertions(+), 10 deletions(-) diff --git a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj index e5180664ef..729895ed5b 100644 --- a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj +++ b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj @@ -16,11 +16,10 @@ - - - + + diff --git a/samples/BenchmarkDotNet.Samples/IntroNuGet.cs b/samples/BenchmarkDotNet.Samples/IntroNuGet.cs index 6034699612..a368b62b56 100644 --- a/samples/BenchmarkDotNet.Samples/IntroNuGet.cs +++ b/samples/BenchmarkDotNet.Samples/IntroNuGet.cs @@ -27,15 +27,15 @@ public Config() var baseJob = Job.MediumRun; string[] targetVersions = [ + "9.0.0", "9.0.3", - "9.0.4", "9.0.5", ]; foreach (var version in targetVersions) { AddJob(baseJob.WithNuGet("System.Collections.Immutable", version) - .WithId("v"+version)); + .WithId($"v{version}")); } } } diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs b/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs index daf3e2222e..d49c3d24e1 100644 --- a/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs @@ -3,7 +3,7 @@ using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains; using System; -using System.Collections.Generic; +using System.IO; using System.Linq; using System.Reflection; @@ -21,6 +21,28 @@ internal static class BenchmarkEnumerator /// The benchmarks inside the assembly. public static BenchmarkRunInfo[] GetBenchmarksFromAssemblyPath(string assemblyPath) { +#if NET462 + // Temporary workaround for BenchmarkDotNet assembly loading issue that occurred under the following conditions: + // 1. Run BenchmarkDotNet.Samples project with following command. + // > dotnet test -c Release --list-tests --framework net462 -tl:off + // 2. When using `BenchmarkDotNet.TestAdapter` package and targeting .NET Framework. + AppDomain.CurrentDomain.AssemblyResolve += (sender, eventArgs) => + { + if (eventArgs.Name.StartsWith("BenchmarkDotNet, Version=")) + { + var baseDir = Path.GetDirectoryName(assemblyPath); + var path = Path.Combine(baseDir, "BenchmarkDotNet.dll"); + if (File.Exists(path)) + { + return Assembly.LoadFrom(path); + } + } + + // Fallback to default assembly resolver + return null; + }; +#endif + var assembly = Assembly.LoadFrom(assemblyPath); var isDebugAssembly = assembly.IsJitOptimizationDisabled() ?? false; diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj index 4b4325f319..37dfe950f7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks.csproj @@ -30,7 +30,6 @@ - diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj index d063998304..8c0cb40557 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj @@ -35,8 +35,8 @@ - - + + all diff --git a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj index 1d653e20a2..b37e22deda 100755 --- a/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj +++ b/tests/BenchmarkDotNet.Tests/BenchmarkDotNet.Tests.csproj @@ -26,7 +26,6 @@ - From c84de9a0d4af419309743c5f86e765de8d490a90 Mon Sep 17 00:00:00 2001 From: Tim Cassell Date: Tue, 10 Jun 2025 15:49:52 -0400 Subject: [PATCH 05/15] Improve memory diagnoser accuracy (#2562) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added `AggressiveOptimization` to methods involved in measuring allocations. Warm up allocation measurement before taking actual measurement. Isolated allocation measurement. Changed some `RuntimeInformation` properties to static readonly fields. Removed enable monitoring in Engine (GcStats handles it). Removed `GC.Collect()` from allocation measurement. Sleep thread to account for tiered jit in Core runtimes 3.0 to 6.0. Updated MemoryDiagnoserTests. Block finalizer thread during memory tests. Disabled EventSource for integration tests. * Use Monitor.Wait instead of ManualResetEventSlim. * Swap IsWasm and IsNativeAOT declaration order. --- src/BenchmarkDotNet/Engines/Engine.cs | 125 +++++++++++++--- src/BenchmarkDotNet/Engines/GcStats.cs | 22 ++- .../Portability/RuntimeInformation.cs | 59 ++++---- .../BenchmarkDotNet.IntegrationTests.csproj | 2 + .../MemoryDiagnoserTests.cs | 134 ++++++++++++------ 5 files changed, 244 insertions(+), 98 deletions(-) diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index f18f29b9be..44077fcc38 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -4,6 +4,7 @@ using System.Globalization; using System.Linq; using System.Runtime.CompilerServices; +using System.Threading; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; @@ -220,31 +221,56 @@ private ClockSpan Measure(Action action, long invokeCount) private (GcStats, ThreadingStats, double) GetExtraStats(IterationData data) { - // we enable monitoring after main target run, for this single iteration which is executed at the end - // so even if we enable AppDomain monitoring in separate process - // it does not matter, because we have already obtained the results! - EnableMonitoring(); + // Warm up the measurement functions before starting the actual measurement. + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(GcStats.ReadInitial()); + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(GcStats.ReadFinal()); IterationSetupAction(); // we run iteration setup first, so even if it allocates, it is not included in the results var initialThreadingStats = ThreadingStats.ReadInitial(); // this method might allocate var exceptionsStats = new ExceptionsStats(); // allocates exceptionsStats.StartListening(); // this method might allocate - var initialGcStats = GcStats.ReadInitial(); - WorkloadAction(data.InvokeCount / data.UnrollFactor); +#if !NET7_0_OR_GREATER + if (RuntimeInformation.IsNetCore && Environment.Version.Major is >= 3 and <= 6 && RuntimeInformation.IsTieredJitEnabled) + { + // #1542 + // We put the current thread to sleep so tiered jit can kick in, compile its stuff, + // and NOT allocate anything on the background thread when we are measuring allocations. + // This is only an issue on netcoreapp3.0 to net6.0. Tiered jit allocations were "fixed" in net7.0 + // (maybe not completely eliminated forever, but at least reduced to a point where measurements are much more stable), + // and netcoreapp2.X uses only GetAllocatedBytesForCurrentThread which doesn't capture the tiered jit allocations. + Thread.Sleep(TimeSpan.FromMilliseconds(500)); + } +#endif - exceptionsStats.Stop(); - var finalGcStats = GcStats.ReadFinal(); + // GC collect before measuring allocations. + ForceGcCollect(); + GcStats gcStats; + using (FinalizerBlocker.MaybeStart()) + { + gcStats = MeasureWithGc(data.InvokeCount / data.UnrollFactor); + } + + exceptionsStats.Stop(); // this method might (de)allocate var finalThreadingStats = ThreadingStats.ReadFinal(); IterationCleanupAction(); // we run iteration cleanup after collecting GC stats var totalOperationsCount = data.InvokeCount * OperationsPerInvoke; - GcStats gcStats = (finalGcStats - initialGcStats).WithTotalOperations(totalOperationsCount); - ThreadingStats threadingStats = (finalThreadingStats - initialThreadingStats).WithTotalOperations(data.InvokeCount * OperationsPerInvoke); + return (gcStats.WithTotalOperations(totalOperationsCount), + (finalThreadingStats - initialThreadingStats).WithTotalOperations(totalOperationsCount), + exceptionsStats.ExceptionsCount / (double)totalOperationsCount); + } - return (gcStats, threadingStats, exceptionsStats.ExceptionsCount / (double)totalOperationsCount); + // Isolate the allocation measurement and skip tier0 jit to make sure we don't get any unexpected allocations. + [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)] + private GcStats MeasureWithGc(long invokeCount) + { + var initialGcStats = GcStats.ReadInitial(); + WorkloadAction(invokeCount); + var finalGcStats = GcStats.ReadFinal(); + return finalGcStats - initialGcStats; } private void RandomizeManagedHeapMemory() @@ -273,7 +299,7 @@ private void GcCollect() ForceGcCollect(); } - private static void ForceGcCollect() + internal static void ForceGcCollect() { GC.Collect(); GC.WaitForPendingFinalizers(); @@ -284,15 +310,6 @@ private static void ForceGcCollect() public void WriteLine() => Host.WriteLine(); - private static void EnableMonitoring() - { - if (RuntimeInformation.IsOldMono) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes-in-mono - return; - - if (RuntimeInformation.IsFullFramework) - AppDomain.MonitoringIsEnabled = true; - } - [UsedImplicitly] public static class Signals { @@ -315,5 +332,71 @@ private static readonly Dictionary MessagesToSignals public static bool TryGetSignal(string message, out HostSignal signal) => MessagesToSignals.TryGetValue(message, out signal); } + + // Very long key and value so this shouldn't be used outside of unit tests. + internal const string UnitTestBlockFinalizerEnvKey = "BENCHMARKDOTNET_UNITTEST_BLOCK_FINALIZER_FOR_MEMORYDIAGNOSER"; + internal const string UnitTestBlockFinalizerEnvValue = UnitTestBlockFinalizerEnvKey + "_ACTIVE"; + + // To prevent finalizers interfering with allocation measurements for unit tests, + // we block the finalizer thread until we've completed the measurement. + // https://github.com/dotnet/runtime/issues/101536#issuecomment-2077647417 + private readonly struct FinalizerBlocker : IDisposable + { + private readonly object hangLock; + + private FinalizerBlocker(object hangLock) => this.hangLock = hangLock; + + private sealed class Impl + { + // ManualResetEvent(Slim) allocates when it is waited and yields the thread, + // so we use Monitor.Wait instead which does not allocate managed memory. + // This behavior is not documented, but was observed with the VS Profiler. + private readonly object hangLock = new (); + private readonly ManualResetEventSlim enteredFinalizerEvent = new (false); + + ~Impl() + { + lock (hangLock) + { + enteredFinalizerEvent.Set(); + Monitor.Wait(hangLock); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + internal static (object hangLock, ManualResetEventSlim enteredFinalizerEvent) CreateWeakly() + { + var impl = new Impl(); + return (impl.hangLock, impl.enteredFinalizerEvent); + } + } + + internal static FinalizerBlocker MaybeStart() + { + if (Environment.GetEnvironmentVariable(UnitTestBlockFinalizerEnvKey) != UnitTestBlockFinalizerEnvValue) + { + return default; + } + var (hangLock, enteredFinalizerEvent) = Impl.CreateWeakly(); + do + { + GC.Collect(); + // Do NOT call GC.WaitForPendingFinalizers. + } + while (!enteredFinalizerEvent.IsSet); + return new FinalizerBlocker(hangLock); + } + + public void Dispose() + { + if (hangLock is not null) + { + lock (hangLock) + { + Monitor.Pulse(hangLock); + } + } + } + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/GcStats.cs b/src/BenchmarkDotNet/Engines/GcStats.cs index ca60b0deea..3416d14ea4 100644 --- a/src/BenchmarkDotNet/Engines/GcStats.cs +++ b/src/BenchmarkDotNet/Engines/GcStats.cs @@ -1,5 +1,6 @@ using System; using System.Reflection; +using System.Runtime.CompilerServices; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; @@ -106,9 +107,10 @@ public int GetCollectionsCount(int generation) return AllocatedBytes <= AllocationQuantum ? 0L : AllocatedBytes; } + // Skip tier0 jit to make sure we don't get any unexpected allocations in this method. + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] public static GcStats ReadInitial() { - // this will force GC.Collect, so we want to do this before collecting collections counts long? allocatedBytes = GetAllocatedBytes(); return new GcStats( @@ -119,15 +121,14 @@ public static GcStats ReadInitial() 0); } + // Skip tier0 jit to make sure we don't get any unexpected allocations in this method. + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] public static GcStats ReadFinal() { return new GcStats( GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2), - - // this will force GC.Collect, so we want to do this after collecting collections counts - // to exclude this single full forced collection from results GetAllocatedBytes(), 0); } @@ -136,17 +137,16 @@ public static GcStats ReadFinal() public static GcStats FromForced(int forcedFullGarbageCollections) => new GcStats(forcedFullGarbageCollections, forcedFullGarbageCollections, forcedFullGarbageCollections, 0, 0); + // Skip tier0 jit to make sure we don't get any unexpected allocations in this method. + [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)] private static long? GetAllocatedBytes() { // we have no tests for WASM and don't want to risk introducing a new bug (https://github.com/dotnet/BenchmarkDotNet/issues/2226) if (RuntimeInformation.IsWasm) return null; - // "This instance Int64 property returns the number of bytes that have been allocated by a specific - // AppDomain. The number is accurate as of the last garbage collection." - CLR via C# - // so we enforce GC.Collect here just to make sure we get accurate results - GC.Collect(); - + // Do NOT call GC.Collect() here, as it causes finalizers to run and possibly allocate. https://github.com/dotnet/runtime/issues/101536#issuecomment-2077533242 + // Instead, we call it before we start the measurement in the Engine. #if NET6_0_OR_GREATER return GC.GetTotalAllocatedBytes(precise: true); #else @@ -218,9 +218,7 @@ private static long CalculateAllocationQuantumSize() break; } - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); + Engine.ForceGcCollect(); result = GC.GetTotalMemory(false); var tmp = new object(); diff --git a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs index 157a99c910..fb5150db2d 100644 --- a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs +++ b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs @@ -27,47 +27,47 @@ internal static class RuntimeInformation internal const string ReleaseConfigurationName = "RELEASE"; internal const string Unknown = "?"; + // Many of these checks allocate and/or are expensive to compute. We store the results in static readonly fields to keep Engine non-allocating. + // Static readonly fields are used instead of properties to avoid an extra getter method call that might not be tier1 jitted. + // This class is internal, so we don't need to expose these as properties. + /// /// returns true for both the old (implementation of .NET Framework) and new Mono (.NET 6+ flavour) /// - public static bool IsMono { get; } = - Type.GetType("Mono.RuntimeStructs") != null; // it allocates a lot of memory, we need to check it once in order to keep Engine non-allocating! + public static readonly bool IsMono = Type.GetType("Mono.RuntimeStructs") != null; - public static bool IsOldMono { get; } = Type.GetType("Mono.Runtime") != null; + public static readonly bool IsOldMono = Type.GetType("Mono.Runtime") != null; - public static bool IsNewMono { get; } = IsMono && !IsOldMono; + public static readonly bool IsNewMono = IsMono && !IsOldMono; - public static bool IsFullFramework => + public static readonly bool IsFullFramework = #if NET6_0_OR_GREATER + // This could be const, but we want to avoid unreachable code warnings. false; #else FrameworkDescription.StartsWith(".NET Framework", StringComparison.OrdinalIgnoreCase); #endif - [PublicAPI] - public static bool IsNetNative => FrameworkDescription.StartsWith(".NET Native", StringComparison.OrdinalIgnoreCase); - - public static bool IsNetCore - => ((Environment.Version.Major >= 5) || FrameworkDescription.StartsWith(".NET Core", StringComparison.OrdinalIgnoreCase)) - && !string.IsNullOrEmpty(typeof(object).Assembly.Location); + public static readonly bool IsNetNative = FrameworkDescription.StartsWith(".NET Native", StringComparison.OrdinalIgnoreCase); - public static bool IsNativeAOT - => Environment.Version.Major >= 5 - && string.IsNullOrEmpty(typeof(object).Assembly.Location) // it's merged to a single .exe and .Location returns null - && !IsWasm; // Wasm also returns "" for assembly locations + public static readonly bool IsNetCore = + ((Environment.Version.Major >= 5) || FrameworkDescription.StartsWith(".NET Core", StringComparison.OrdinalIgnoreCase)) + && !string.IsNullOrEmpty(typeof(object).Assembly.Location); #if NET6_0_OR_GREATER [System.Runtime.Versioning.SupportedOSPlatformGuard("browser")] -#endif - public static bool IsWasm => -#if NET6_0_OR_GREATER - OperatingSystem.IsBrowser(); + public static readonly bool IsWasm = OperatingSystem.IsBrowser(); #else - IsOSPlatform(OSPlatform.Create("BROWSER")); + public static readonly bool IsWasm = IsOSPlatform(OSPlatform.Create("BROWSER")); #endif + public static readonly bool IsNativeAOT = + Environment.Version.Major >= 5 + && string.IsNullOrEmpty(typeof(object).Assembly.Location) // it's merged to a single .exe and .Location returns null + && !IsWasm; // Wasm also returns "" for assembly locations + #if NETSTANDARD2_0 - public static bool IsAot { get; } = IsAotMethod(); // This allocates, so we only want to call it once statically. + public static readonly bool IsAot = IsAotMethod(); private static bool IsAotMethod() { @@ -85,11 +85,22 @@ private static bool IsAotMethod() return false; } #else - public static bool IsAot => !System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeCompiled; + public static readonly bool IsAot = !System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeCompiled; #endif - public static bool IsRunningInContainer => string.Equals(Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"), "true"); - + public static readonly bool IsTieredJitEnabled = + IsNetCore + && (Environment.Version.Major < 3 + // Disabled by default in netcoreapp2.X, check if it's enabled. + ? Environment.GetEnvironmentVariable("COMPlus_TieredCompilation") == "1" + || Environment.GetEnvironmentVariable("DOTNET_TieredCompilation") == "1" + || (AppContext.TryGetSwitch("System.Runtime.TieredCompilation", out bool isEnabled) && isEnabled) + // Enabled by default in netcoreapp3.0+, check if it's disabled. + : Environment.GetEnvironmentVariable("COMPlus_TieredCompilation") != "0" + && Environment.GetEnvironmentVariable("DOTNET_TieredCompilation") != "0" + && (!AppContext.TryGetSwitch("System.Runtime.TieredCompilation", out isEnabled) || isEnabled)); + + public static readonly bool IsRunningInContainer = string.Equals(Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"), "true"); internal static string GetArchitecture() => GetCurrentPlatform().ToString(); diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj index 8c0cb40557..46e9b0edac 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj @@ -18,6 +18,8 @@ Always + + diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 603926c2a6..7d2e6d60df 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -36,7 +36,11 @@ public class MemoryDiagnoserTests public static IEnumerable GetToolchains() { yield return new object[] { Job.Default.GetToolchain() }; - yield return new object[] { InProcessEmitToolchain.Instance }; + // InProcessEmit reports flaky allocations in current .Net 8. + if (!RuntimeInformation.IsNetCore) + { + yield return new object[] { InProcessEmitToolchain.Instance }; + } } public class AccurateAllocations @@ -67,7 +71,7 @@ public void MemoryDiagnoserIsAccurate(IToolchain toolchain) }); } - [FactEnvSpecific("We don't want to test NativeAOT twice (for .NET Framework 4.6.2 and .NET 7.0)", EnvRequirement.DotNetCoreOnly)] + [FactEnvSpecific("We don't want to test NativeAOT twice (for .NET Framework 4.6.2 and .NET 8.0)", EnvRequirement.DotNetCoreOnly)] public void MemoryDiagnoserSupportsNativeAOT() { if (OsDetector.IsMacOS()) @@ -105,7 +109,7 @@ private void AllocateUntilGcWakesUp() } } - [Theory(Skip = "#1542 Tiered JIT Thread allocates memory in the background"), MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains))] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup(IToolchain toolchain) { @@ -118,23 +122,41 @@ public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup(IToolcha public class NoAllocationsAtAll { [Benchmark] public void EmptyMethod() { } + + [Benchmark] + public ulong TimeConsuming() + { + var r = 1ul; + for (var i = 0; i < 50_000_000; i++) + { + r /= 1; + } + return r; + } } [Theory, MemberData(nameof(GetToolchains))] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void EngineShouldNotInterfereAllocationResults(IToolchain toolchain) { - if (RuntimeInformation.IsFullFramework && toolchain.IsInProcess) - { - return; // this test is flaky on Full Framework - } - AssertAllocations(toolchain, typeof(NoAllocationsAtAll), new Dictionary { { nameof(NoAllocationsAtAll.EmptyMethod), 0 } }); } + // #1542 + [Theory, MemberData(nameof(GetToolchains))] + [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] + public void TieredJitShouldNotInterfereAllocationResults(IToolchain toolchain) + { + AssertAllocations(toolchain, typeof(NoAllocationsAtAll), new Dictionary + { + { nameof(NoAllocationsAtAll.TimeConsuming), 0 } + }, + disableTieredJit: false, iterationCount: 10); // 1 iteration is not enough to repro the problem + } + public class NoBoxing { [Benchmark] public ValueTuple ReturnsValueType() => new ValueTuple(0); @@ -165,11 +187,6 @@ public class NonAllocatingAsynchronousBenchmarks [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AwaitingTasksShouldNotInterfereAllocationResults(IToolchain toolchain) { - if (toolchain.IsInProcess) - { - return; // it's flaky: https://github.com/dotnet/BenchmarkDotNet/issues/1925 - } - AssertAllocations(toolchain, typeof(NonAllocatingAsynchronousBenchmarks), new Dictionary { { nameof(NonAllocatingAsynchronousBenchmarks.CompletedTask), 0 }, @@ -217,8 +234,8 @@ public byte[] SixtyFourBytesArray() } } - [Theory(Skip = "#1542 Tiered JIT Thread allocates memory in the background"), MemberData(nameof(GetToolchains))] - //[TheoryNetCoreOnly("Only .NET Core 2.0+ API is bug free for this case"), MemberData(nameof(GetToolchains))] + [TheoryEnvSpecific("Full Framework cannot measure precisely enough for low invocation counts.", EnvRequirement.DotNetCoreOnly)] + [MemberData(nameof(GetToolchains))] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) { @@ -233,31 +250,57 @@ public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) public class MultiThreadedAllocation { - public const int Size = 1_000_000; + public const int Size = 1024; public const int ThreadsCount = 10; + // We cache the threads in GlobalSetup and reuse them for each benchmark invocation + // to avoid measuring the cost of thread start and join, which varies across different runtimes. private Thread[] threads; + private volatile bool keepRunning = true; + private readonly Barrier barrier = new (ThreadsCount + 1); + private readonly CountdownEvent countdownEvent = new (ThreadsCount); - [IterationSetup] - public void SetupIteration() + [GlobalSetup] + public void Setup() { threads = Enumerable.Range(0, ThreadsCount) - .Select(_ => new Thread(() => GC.KeepAlive(new byte[Size]))) + .Select(_ => new Thread(() => + { + while (keepRunning) + { + barrier.SignalAndWait(); + GC.KeepAlive(new byte[Size]); + countdownEvent.Signal(); + } + })) .ToArray(); + foreach (var thread in threads) + { + thread.Start(); + } } - [Benchmark] - public void Allocate() + [GlobalCleanup] + public void Cleanup() { + keepRunning = false; + barrier.SignalAndWait(); foreach (var thread in threads) { - thread.Start(); thread.Join(); } } + + [Benchmark] + public void Allocate() + { + countdownEvent.Reset(ThreadsCount); + barrier.SignalAndWait(); + countdownEvent.Wait(); + } } - [Theory(Skip = "Test is flaky even in latest .Net")] + [TheoryEnvSpecific("Full Framework cannot measure precisely enough", EnvRequirement.DotNetCoreOnly)] [MemberData(nameof(GetToolchains))] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolchain) @@ -265,18 +308,16 @@ public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolc long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length long memoryAllocatedPerArray = (MultiThreadedAllocation.Size + objectAllocationOverhead + arraySizeOverhead); - long threadStartAndJoinOverhead = 112; // this is more or less a magic number taken from memory profiler - long allocatedMemoryPerThread = memoryAllocatedPerArray + threadStartAndJoinOverhead; AssertAllocations(toolchain, typeof(MultiThreadedAllocation), new Dictionary { - { nameof(MultiThreadedAllocation.Allocate), allocatedMemoryPerThread * MultiThreadedAllocation.ThreadsCount } + { nameof(MultiThreadedAllocation.Allocate), memoryAllocatedPerArray * MultiThreadedAllocation.ThreadsCount } }); } - private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators) + private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators, bool disableTieredJit = true, int iterationCount = 1) { - var config = CreateConfig(toolchain); + var config = CreateConfig(toolchain, disableTieredJit, iterationCount); var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType, config); var summary = BenchmarkRunner.Run(benchmarks); @@ -312,24 +353,35 @@ private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Diction } } - private IConfig CreateConfig(IToolchain toolchain) - => ManualConfig.CreateEmpty() - .AddJob(Job.ShortRun - .WithEvaluateOverhead(false) // no need to run idle for this test - .WithWarmupCount(0) // don't run warmup to save some time for our CI runs - .WithIterationCount(1) // single iteration is enough for us - .WithGcForce(false) - .WithGcServer(false) - .WithGcConcurrent(false) - .WithEnvironmentVariables([ - // Tiered JIT can allocate some memory on a background thread, let's disable it to make our tests less flaky (#1542) + private IConfig CreateConfig(IToolchain toolchain, + // Tiered JIT can allocate some memory on a background thread, let's disable it by default to make our tests less flaky (#1542). + // This was mostly fixed in net7.0, but tiered jit thread is not guaranteed to not allocate, so we disable it just in case. + bool disableTieredJit = true, + // Single iteration is enough for most of the tests. + int iterationCount = 1) + { + var job = Job.ShortRun + .WithEvaluateOverhead(false) // no need to run idle for this test + .WithWarmupCount(0) // don't run warmup to save some time for our CI runs + .WithIterationCount(iterationCount) + .WithGcForce(false) + .WithGcServer(false) + .WithGcConcurrent(false) + // To prevent finalizers allocating out of our control, we hang the finalizer thread. + // https://github.com/dotnet/runtime/issues/101536#issuecomment-2077647417 + .WithEnvironmentVariable(Engines.Engine.UnitTestBlockFinalizerEnvKey, Engines.Engine.UnitTestBlockFinalizerEnvValue) + .WithToolchain(toolchain); + return ManualConfig.CreateEmpty() + .AddJob(disableTieredJit + ? job.WithEnvironmentVariables( new EnvironmentVariable("DOTNET_TieredCompilation", "0"), new EnvironmentVariable("COMPlus_TieredCompilation", "0") - ]) - .WithToolchain(toolchain)) + ) + : job) .AddColumnProvider(DefaultColumnProviders.Instance) .AddDiagnoser(MemoryDiagnoser.Default) .AddLogger(toolchain.IsInProcess ? ConsoleLogger.Default : new OutputLogger(output)); // we can't use OutputLogger for the InProcess toolchains because it allocates memory on the same thread + } // note: don't copy, never use in production systems (it should work but I am not 100% sure) private int CalculateRequiredSpace() From fe3b31f2ea254d1fd45a1a85d17aad5e856abdb6 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Fri, 13 Jun 2025 08:10:29 +0900 Subject: [PATCH 06/15] chore: add setting to skip test reports when original workflow is cancelled (#2772) --- .github/workflows/report-test-results.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/report-test-results.yaml b/.github/workflows/report-test-results.yaml index 77aafa2fe7..f1c627295e 100644 --- a/.github/workflows/report-test-results.yaml +++ b/.github/workflows/report-test-results.yaml @@ -11,6 +11,7 @@ jobs: report: runs-on: ubuntu-latest permissions: write-all + if: ${{ github.event.workflow_run.conclusion != 'cancelled' }} steps: # Cleanup Old Files - name: Cleanup Old Files From 6730bf1570bc68e403d17eeaf9749c5f9b0bee24 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Fri, 13 Jun 2025 10:03:13 +0900 Subject: [PATCH 07/15] feat: add JobOrderPolicy option to sort jobs numeric order (#2770) --- .../Attributes/OrdererAttribute.cs | 5 +- src/BenchmarkDotNet/Jobs/JobComparer.cs | 93 ++++++++++- src/BenchmarkDotNet/Order/DefaultOrderer.cs | 8 +- src/BenchmarkDotNet/Order/JobOrderPolicy.cs | 14 ++ .../Order/JobOrderTests.cs | 146 ++++++++++++++++++ 5 files changed, 258 insertions(+), 8 deletions(-) create mode 100644 src/BenchmarkDotNet/Order/JobOrderPolicy.cs create mode 100644 tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs diff --git a/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs b/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs index 83b5051895..b20a9108ca 100644 --- a/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs @@ -9,9 +9,10 @@ public class OrdererAttribute : Attribute, IConfigSource { public OrdererAttribute( SummaryOrderPolicy summaryOrderPolicy = SummaryOrderPolicy.Default, - MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared) + MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared, + JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default) { - Config = ManualConfig.CreateEmpty().WithOrderer(new DefaultOrderer(summaryOrderPolicy, methodOrderPolicy)); + Config = ManualConfig.CreateEmpty().WithOrderer(new DefaultOrderer(summaryOrderPolicy, methodOrderPolicy, jobOrderPolicy)); } public IConfig Config { get; } diff --git a/src/BenchmarkDotNet/Jobs/JobComparer.cs b/src/BenchmarkDotNet/Jobs/JobComparer.cs index 0a0c287523..ac1539ce7c 100644 --- a/src/BenchmarkDotNet/Jobs/JobComparer.cs +++ b/src/BenchmarkDotNet/Jobs/JobComparer.cs @@ -1,12 +1,23 @@ -using System; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Order; +using System; using System.Collections.Generic; -using BenchmarkDotNet.Characteristics; namespace BenchmarkDotNet.Jobs { internal class JobComparer : IComparer, IEqualityComparer { - public static readonly JobComparer Instance = new JobComparer(); + private readonly IComparer Comparer; + + public static readonly JobComparer Instance = new JobComparer(JobOrderPolicy.Default); + public static readonly JobComparer Numeric = new JobComparer(JobOrderPolicy.Numeric); + + public JobComparer(JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default) + { + Comparer = jobOrderPolicy == JobOrderPolicy.Default + ? StringComparer.Ordinal + : new NumericStringComparer(); // TODO: Use `StringComparer.Create(CultureInfo.InvariantCulture, CompareOptions.NumericOrdering)` for .NET10 or greater. + } public int Compare(Job x, Job y) { @@ -39,7 +50,7 @@ public int Compare(Job x, Job y) continue; } - int compare = string.CompareOrdinal( + int compare = Comparer.Compare( presenter.ToPresentation(x, characteristic), presenter.ToPresentation(y, characteristic)); if (compare != 0) @@ -52,5 +63,79 @@ public int Compare(Job x, Job y) public bool Equals(Job x, Job y) => Compare(x, y) == 0; public int GetHashCode(Job obj) => obj.Id.GetHashCode(); + + internal class NumericStringComparer : IComparer + { + public int Compare(string? x, string? y) + { + if (ReferenceEquals(x, y)) return 0; + if (x == null) return -1; + if (y == null) return 1; + + ReadOnlySpan spanX = x.AsSpan(); + ReadOnlySpan spanY = y.AsSpan(); + + int i = 0, j = 0; + + while (i < spanX.Length && j < spanY.Length) + { + char cx = spanX[i]; + char cy = spanY[j]; + + if (!char.IsDigit(cx) || !char.IsDigit(cy)) + { + int cmp = cx.CompareTo(cy); + if (cmp != 0) + return cmp; + + i++; + j++; + continue; + } + + int ixStart = i; + int iyStart = j; + + // Skip leading zeros + while (ixStart < spanX.Length && spanX[ixStart] == '0') ixStart++; + while (iyStart < spanY.Length && spanY[iyStart] == '0') iyStart++; + + int ix = ixStart; + int iy = iyStart; + + // Skip digits + while (ix < spanX.Length && char.IsDigit(spanX[ix])) ix++; + while (iy < spanY.Length && char.IsDigit(spanY[iy])) iy++; + + int lenX = ix - ixStart; + int lenY = iy - iyStart; + + // Compare by digits length + if (lenX != lenY) + return lenX.CompareTo(lenY); + + // Compare digits + for (int k = 0; k < lenX; k++) + { + int cmp = spanX[ixStart + k].CompareTo(spanY[iyStart + k]); + if (cmp != 0) + return cmp; + } + + // Compare by leading zeros + int leadingZerosX = ixStart - i; + int leadingZerosY = iyStart - j; + if (leadingZerosX != leadingZerosY) + return 0; // Leading zero differences are ignored (`CompareOptions.NumericOrdering` behavior of .NET) + + // Move to the next character after the digits + i = ix; + j = iy; + } + + // Compare remaining chars + return (spanX.Length - i).CompareTo(spanY.Length - j); + } + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Order/DefaultOrderer.cs b/src/BenchmarkDotNet/Order/DefaultOrderer.cs index 656de7ce98..d0dc2d475b 100644 --- a/src/BenchmarkDotNet/Order/DefaultOrderer.cs +++ b/src/BenchmarkDotNet/Order/DefaultOrderer.cs @@ -19,7 +19,7 @@ public class DefaultOrderer : IOrderer private readonly IComparer categoryComparer = CategoryComparer.Instance; private readonly IComparer paramsComparer = ParameterComparer.Instance; - private readonly IComparer jobComparer = JobComparer.Instance; + private readonly IComparer jobComparer; private readonly IComparer targetComparer; public SummaryOrderPolicy SummaryOrderPolicy { get; } @@ -27,10 +27,14 @@ public class DefaultOrderer : IOrderer public DefaultOrderer( SummaryOrderPolicy summaryOrderPolicy = SummaryOrderPolicy.Default, - MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared) + MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared, + JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default) { SummaryOrderPolicy = summaryOrderPolicy; MethodOrderPolicy = methodOrderPolicy; + jobComparer = jobOrderPolicy == JobOrderPolicy.Default + ? JobComparer.Instance + : JobComparer.Numeric; targetComparer = new DescriptorComparer(methodOrderPolicy); } diff --git a/src/BenchmarkDotNet/Order/JobOrderPolicy.cs b/src/BenchmarkDotNet/Order/JobOrderPolicy.cs new file mode 100644 index 0000000000..be1d66186d --- /dev/null +++ b/src/BenchmarkDotNet/Order/JobOrderPolicy.cs @@ -0,0 +1,14 @@ +namespace BenchmarkDotNet.Order; + +public enum JobOrderPolicy +{ + /// + /// Compare job characteristics in ordinal order. + /// + Default, + + /// + /// Compare job characteristics in numeric order. + /// + Numeric, +} diff --git a/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs b/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs new file mode 100644 index 0000000000..c0e645ea22 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs @@ -0,0 +1,146 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.CsProj; +using System.Linq; +using Xunit; + +namespace BenchmarkDotNet.Tests.Order; + +public class JobOrderTests +{ + [Fact] + public void TestJobOrders_ByJobId() + { + // Arrange + Job[] jobs = + [ + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80) + .WithRuntime(CoreRuntime.Core80) + .WithId("v1.4.1"), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90) + .WithId("v1.4.10"), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0) + .WithRuntime(CoreRuntime.Core10_0) + .WithId("v1.4.2"), + ]; + + // Verify jobs are sorted by JobId's ordinal order. + { + // Act + var comparer = JobComparer.Instance; + var results = jobs.OrderBy(x => x, comparer) + .Select(x => x.Job.Id) + .ToArray(); + + // Assert + Assert.Equal(["v1.4.1", "v1.4.10", "v1.4.2"], results); + } + + // Verify jobs are sorted by JobId's numeric order. + { + // Act + var comparer = JobComparer.Numeric; + var results = jobs.OrderBy(d => d, comparer) + .Select(x => x.Job.Id) + .ToArray(); + // Assert + Assert.Equal(["v1.4.1", "v1.4.2", "v1.4.10"], results); + } + } + + [Fact] + public void TestJobOrders_ByRuntime() + { + // Arrange + Job[] jobs = + [ + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0) + .WithRuntime(CoreRuntime.Core80), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80) + .WithRuntime(CoreRuntime.Core10_0), + ]; + + // Act + // Verify jobs are sorted by Runtime's numeric order. + var results = jobs.OrderBy(d => d, JobComparer.Numeric) + .Select(x => x.Job.Environment.GetRuntime().Name) + .ToArray(); + + // Assert + var expected = new[] + { + CoreRuntime.Core80.Name, + CoreRuntime.Core90.Name, + CoreRuntime.Core10_0.Name + }; + Assert.Equal(expected, results); + } + + [Fact] + public void TestJobOrders_ByToolchain() + { + // Arrange + Job[] jobs = + [ + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80), + ]; + + // Act + // Verify jobs are sorted by Toolchain's numeric order. + var results = jobs.OrderBy(d => d, JobComparer.Numeric) + .Select(x => x.Job.GetToolchain().Name) + .ToArray(); + + // Assert + var expected = new[] + { + CsProjCoreToolchain.NetCoreApp80.Name, + CsProjCoreToolchain.NetCoreApp90.Name, + CsProjCoreToolchain.NetCoreApp10_0.Name, + }; + Assert.Equal(expected, results); + } + + [Theory] + [InlineData("item1", "item1", 0)] + [InlineData("item123", "item123", 0)] + // Compare different values + [InlineData("item1", "item2", -1)] + [InlineData("item2", "item1", 1)] + [InlineData("item2", "item10", -1)] + [InlineData("item10", "item2", 1)] + [InlineData("item1a", "item1b", -1)] + [InlineData("item1b", "item1a", 1)] + [InlineData("item", "item1", -1)] + [InlineData("item10", "item", 1)] + [InlineData(".NET 8", ".NET 10", -1)] + [InlineData(".NET 10", ".NET 8", 1)] + [InlineData("v1.4.1", "v1.4.10", -1)] + [InlineData("v1.4.10", "v1.4.2", 1)] + // Compare zero paddeed numeric string. + [InlineData("item01", "item1", 0)] + [InlineData("item001", "item1", 0)] + [InlineData("item1", "item001", 0)] + [InlineData("item1", "item01", 0)] + [InlineData("item9", "item09", 0)] + [InlineData(".NET 08", ".NET 10", -1)] + [InlineData(".NET 10", ".NET 08", 1)] + // Arguments that contains null + [InlineData(null, "a", -1)] + [InlineData("a", null, 1)] + [InlineData(null, null, 0)] + public void TestNumericComparer(string? a, string? b, int expectedSign) + { + int result = new JobComparer.NumericStringComparer().Compare(a, b); + Assert.Equal(expectedSign, NormalizeSign(result)); + + static int NormalizeSign(int value) + => value == 0 ? 0 : value < 0 ? -1 : 1; + } +} From a08f01c1a682e3aa7dab92c44d18b5b52a38b062 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sat, 14 Jun 2025 16:06:46 +0900 Subject: [PATCH 08/15] chore: Modify macos runner image for CI (#2775) * chore: modify runner image for macos * chore: fix workflow name and artifact name * Update .github/workflows/run-tests.yaml --------- Co-authored-by: Tim Cassell --- .github/workflows/run-tests.yaml | 16 +++++++++++++--- .../Runners/UnitTestRunner.cs | 4 +++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index d06efb331e..9d3f25a9e8 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -91,7 +91,17 @@ jobs: path: "**/*.trx" test-macos: - runs-on: macos-15 + name: test-macos (${{ matrix.os.arch }}) + runs-on: ${{ matrix.os.runs-on }} + strategy: + matrix: + os: + - runs-on: 'macos-latest' + jsvu-os: 'mac64arm' + arch: 'arm64' + - runs-on: 'macos-13' + jsvu-os: 'mac64' + arch: 'x64' steps: - uses: actions/checkout@v4 - name: Set up node @@ -99,7 +109,7 @@ jobs: with: node-version: "22" - name: Set up v8 - run: npm install jsvu -g && jsvu --os=mac64 --engines=v8 && echo "$HOME/.jsvu/bin" >> $GITHUB_PATH + run: npm install jsvu -g && jsvu --os=${{ matrix.os.jsvu-os }} --engines=v8 && echo "$HOME/.jsvu/bin" >> $GITHUB_PATH - name: Install wasm-tools workload run: ./build.cmd install-wasm-tools # Build and Test @@ -114,7 +124,7 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: test-macos-trx-${{ github.run_id }} + name: test-macos(${{ matrix.os.arch }})-trx-${{ github.run_id }} path: "**/*.trx" test-pack: diff --git a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs index 9b10d54ea4..89b777d09f 100644 --- a/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs +++ b/build/BenchmarkDotNet.Build/Runners/UnitTestRunner.cs @@ -4,6 +4,7 @@ using Cake.Common.Tools.DotNet; using Cake.Common.Tools.DotNet.Test; using Cake.Core.IO; +using System.Runtime.InteropServices; namespace BenchmarkDotNet.Build.Runners; @@ -47,7 +48,8 @@ private DotNetTestSettings GetTestSettingsParameters(FilePath logFile, string tf private void RunTests(FilePath projectFile, string alias, string tfm) { var os = Utils.GetOs(); - var trxFileName = $"{os}-{alias}-{tfm}.trx"; + var arch = RuntimeInformation.OSArchitecture.ToString().ToLower(); + var trxFileName = $"{os}({arch})-{alias}-{tfm}.trx"; var trxFile = TestOutputDirectory.CombineWithFilePath(trxFileName); var settings = GetTestSettingsParameters(trxFile, tfm); From de8ba0796cdeab18fd8abc4988864a134f6dae8f Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sat, 14 Jun 2025 16:38:47 +0900 Subject: [PATCH 09/15] chore: suppress xunit non-serializable data warnings (#2769) --- .../ArgumentsTests.cs | 58 +++++++++---------- .../DisassemblyDiagnoserTests.cs | 8 +-- .../MemoryDiagnoserTests.cs | 18 +++--- .../ParamSourceTests.cs | 8 +-- .../TailCallDiagnoserTests.cs | 4 +- .../ThreadingDiagnoserTests.cs | 4 +- .../ValuesReturnedByBenchmarkTest.cs | 2 +- 7 files changed, 51 insertions(+), 51 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs index bcdaa9cdcf..77d8ecb931 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs @@ -24,7 +24,7 @@ public static IEnumerable GetToolchains() public ArgumentsTests(ITestOutputHelper output) : base(output) { } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArgumentsArePassedToBenchmarks(IToolchain toolchain) => CanExecute(toolchain); public class WithArguments @@ -61,7 +61,7 @@ public ValueTask SimpleValueTaskAsync(bool boolean, int number) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArgumentsFromSourceArePassedToBenchmarks(IToolchain toolchain) => CanExecute(toolchain); public class WithArgumentsSource @@ -81,7 +81,7 @@ public IEnumerable ArgumentsProvider() } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArgumentsFromSourceInAnotherClassArePassedToBenchmarks(IToolchain toolchain) => CanExecute(toolchain); public class WithArgumentsSourceInAnotherClass @@ -103,7 +103,7 @@ public static IEnumerable ArgumentsProvider() } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArgumentsCanBePassedByReferenceToBenchmark(IToolchain toolchain) => CanExecute(toolchain); public class WithRefArguments @@ -118,7 +118,7 @@ public void Simple(ref bool boolean, ref int number) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArgumentsCanBePassedByReadonlyReferenceToBenchmark(IToolchain toolchain) => CanExecute(toolchain); public class WithInArguments @@ -133,7 +133,7 @@ public void Simple(in bool boolean, in int number) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void NonCompileTimeConstantsCanBeReturnedFromSource(IToolchain toolchain) => CanExecute(toolchain); public class WithComplexTypesReturnedFromSources @@ -190,7 +190,7 @@ public struct SomeStruct } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ArrayCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithArray @@ -208,7 +208,7 @@ public void AcceptingArray(int[] array) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void IEnumerableCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithIEnumerable @@ -233,7 +233,7 @@ public IEnumerable Sources() public void Any(string name, IEnumerable source) => source.Any(); } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void JaggedArrayCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithJaggedArray @@ -269,7 +269,7 @@ public IEnumerable CreateMatrix() } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void GenericTypeCanBePassedByRefAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithGenericByRef @@ -305,7 +305,7 @@ public IEnumerable GetInputData() } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void AnArrayOfTypeWithNoParameterlessCtorCanBePassedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithArrayOfStringAsArgument @@ -322,7 +322,7 @@ public void TypeReflectionArrayGetType(object anArray) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void AnArrayCanBePassedToBenchmarkAsSpan(IToolchain toolchain) => CanExecute(toolchain); public class WithArrayToSpan @@ -342,7 +342,7 @@ public void AcceptsSpan(Span span) [TheoryEnvSpecific("The implicit cast operator is available only in .NET Core 2.1+ (See https://github.com/dotnet/corefx/issues/30121 for more)", EnvRequirement.DotNetCoreOnly)] - [MemberData(nameof(GetToolchains))] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void StringCanBePassedToBenchmarkAsReadOnlySpan(IToolchain toolchain) => CanExecute(toolchain); public class WithStringToReadOnlySpan @@ -360,7 +360,7 @@ public void AcceptsReadOnlySpan(ReadOnlySpan notString) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void AnArrayOfStringsCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); @@ -380,7 +380,7 @@ public void TypeReflectionArrayGetType(string[] array) } } - [Theory, MemberData(nameof(GetToolchains))] // make sure BDN mimics xunit's MemberData behaviour + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] // make sure BDN mimics xunit's MemberData behaviour public void AnIEnumerableOfArrayOfObjectsCanBeUsedAsArgumentForBenchmarkAcceptingSingleArgument(IToolchain toolchain) => CanExecute(toolchain); @@ -400,7 +400,7 @@ public void SingleArgument(bool boolean) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void BenchmarkCanAcceptFewArrays(IToolchain toolchain) => CanExecute(toolchain); public class FewArrays @@ -429,7 +429,7 @@ public void AcceptsArrays(int[] even, int[] notEven) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void VeryBigIntegersAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithVeryBigInteger @@ -450,7 +450,7 @@ public void Method(BigInteger passed) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void SpecialDoubleValuesAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithSpecialDoubleValues @@ -495,7 +495,7 @@ public void Method(double passed, string name) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void SpecialFloatValuesAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithSpecialFloatValues @@ -540,7 +540,7 @@ public void Method(float passed, string name) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void SpecialDecimalValuesAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithSpecialDecimalValues @@ -569,7 +569,7 @@ public void Method(decimal passed, string name) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void DateTimeCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class WithDateTime @@ -590,7 +590,7 @@ public void Test(DateTime passed) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void CustomTypeThatAlsoExistsInTheSystemNamespaceAsArgument(IToolchain toolchain) => CanExecute(toolchain); public class CustomTypeThatAlsoExistsInTheSystemNamespace @@ -611,7 +611,7 @@ public void Test(Action passed) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void EnumFlagsAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithEnumFlags @@ -648,7 +648,7 @@ public void Test(LongFlagEnum passedLongFlagEnum, ByteFlagEnum passedByteFlagEnu } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void UndefinedEnumValuesAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithUndefinedEnumValue @@ -674,7 +674,7 @@ public void Test(SomeEnum defined, SomeEnum undefined, SomeEnum undefinedNegativ } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void StaticMethodsAndPropertiesCanBeUsedAsSources_EnumerableOfObjects(IToolchain toolchain) => CanExecute(toolchain); @@ -722,7 +722,7 @@ public void TestProperty(int argument) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void StaticMethodsAndPropertiesCanBeUsedAsSources_EnumerableOfArrayOfObjects(IToolchain toolchain) => CanExecute(toolchain); @@ -769,7 +769,7 @@ public void TestProperty(int argument) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void MethodsAndPropertiesFromAnotherClassCanBeUsedAsSources(IToolchain toolchain) => CanExecute(toolchain); @@ -805,7 +805,7 @@ public static IEnumerable Property } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void VeryLongStringsAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class WithVeryLongString @@ -829,7 +829,7 @@ public void Test(string first, string second) } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void ComplexStringPattersAreSupported(IToolchain toolchain) => CanExecute(toolchain); public class Perf_Regex_Industry_RustLang_Sherlock diff --git a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs index 26c17eed04..dbd220f007 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs @@ -88,7 +88,7 @@ public void Recursive() } [Theory] - [MemberData(nameof(GetAllJits))] + [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, Runtime runtime) { @@ -107,7 +107,7 @@ public void CanDisassembleAllMethodCalls(Jit jit, Platform platform, Runtime run } [Theory] - [MemberData(nameof(GetAllJits))] + [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, Runtime runtime) { @@ -132,7 +132,7 @@ public void CanDisassembleAllMethodCallsUsingFilters(Jit jit, Platform platform, } [Theory] - [MemberData(nameof(GetAllJits))] + [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleGenericTypes(Jit jit, Platform platform, Runtime runtime) { @@ -154,7 +154,7 @@ [Benchmark] public void JustReturn() { } } [Theory] - [MemberData(nameof(GetAllJits))] + [MemberData(nameof(GetAllJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void CanDisassembleInlinableBenchmarks(Jit jit, Platform platform, Runtime runtime) { diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 7d2e6d60df..169d18f454 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -51,7 +51,7 @@ public class AccurateAllocations [Benchmark] public Task AllocateTask() => Task.FromResult(-12345); } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurate(IToolchain toolchain) { @@ -109,7 +109,7 @@ private void AllocateUntilGcWakesUp() } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup(IToolchain toolchain) { @@ -135,7 +135,7 @@ public ulong TimeConsuming() } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void EngineShouldNotInterfereAllocationResults(IToolchain toolchain) { @@ -146,7 +146,7 @@ public void EngineShouldNotInterfereAllocationResults(IToolchain toolchain) } // #1542 - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void TieredJitShouldNotInterfereAllocationResults(IToolchain toolchain) { @@ -162,7 +162,7 @@ public class NoBoxing [Benchmark] public ValueTuple ReturnsValueType() => new ValueTuple(0); } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void EngineShouldNotIntroduceBoxing(IToolchain toolchain) { @@ -183,7 +183,7 @@ public class NonAllocatingAsynchronousBenchmarks [Benchmark] public ValueTask CompletedValueTaskOfT() => new ValueTask(default(int)); } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AwaitingTasksShouldNotInterfereAllocationResults(IToolchain toolchain) { @@ -208,7 +208,7 @@ public void WithOperationsPerInvoke() private void DoNotInline(object left, object right) { } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocatedMemoryShouldBeScaledForOperationsPerInvoke(IToolchain toolchain) { @@ -235,7 +235,7 @@ public byte[] SixtyFourBytesArray() } [TheoryEnvSpecific("Full Framework cannot measure precisely enough for low invocation counts.", EnvRequirement.DotNetCoreOnly)] - [MemberData(nameof(GetToolchains))] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) { @@ -301,7 +301,7 @@ public void Allocate() } [TheoryEnvSpecific("Full Framework cannot measure precisely enough", EnvRequirement.DotNetCoreOnly)] - [MemberData(nameof(GetToolchains))] + [MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolchain) { diff --git a/tests/BenchmarkDotNet.IntegrationTests/ParamSourceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ParamSourceTests.cs index ed105f8290..e58f6a67d7 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ParamSourceTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ParamSourceTests.cs @@ -96,7 +96,7 @@ public static IEnumerable GetSource() public int Benchmark() => ParamsTarget?.Data ?? 0; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void PrivateClassWithPublicInterface_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(PrivateClassWithPublicInterface), toolchain); public class PrivateClassWithPublicInterface_Array @@ -116,7 +116,7 @@ public IEnumerable GetSource() public int Benchmark() => ParamsTarget?.Sum(p => p?.Data ?? 0) ?? 0; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void PrivateClassWithPublicInterface_Array_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(PrivateClassWithPublicInterface_Array), toolchain); public class PrivateClassWithPublicInterface_Enumerable @@ -137,7 +137,7 @@ public IEnumerable> GetSource() public int Benchmark() => ParamsTarget?.Sum(p => p?.Data ?? 0) ?? 0; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void PrivateClassWithPublicInterface_Enumerable_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(PrivateClassWithPublicInterface_Enumerable), toolchain); public class PrivateClassWithPublicInterface_AsObject @@ -156,7 +156,7 @@ public static IEnumerable GetSource() public int Benchmark() => ParamsTarget?.Data ?? 0; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void PrivateClassWithPublicInterface_AsObject_Succeeds(IToolchain toolchain) => CanExecuteWithExtraInfo(typeof(PrivateClassWithPublicInterface_AsObject), toolchain); public class PublicSource diff --git a/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs index ab9fbbb645..71988ba7c9 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/TailCallDiagnoserTests.cs @@ -53,7 +53,7 @@ public class NonTailCallBenchmarks } [TheoryEnvSpecific(WindowsOnly, EnvRequirement.WindowsOnly)] - [MemberData(nameof(GetJits))] + [MemberData(nameof(GetJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void TailCallDiagnoserCatchesTailCallEvents(Jit jit, Platform platform, Runtime runtime) { @@ -63,7 +63,7 @@ public void TailCallDiagnoserCatchesTailCallEvents(Jit jit, Platform platform, R } [TheoryEnvSpecific(WindowsOnly, EnvRequirement.WindowsOnly)] - [MemberData(nameof(GetJits))] + [MemberData(nameof(GetJits), DisableDiscoveryEnumeration = true)] [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void TailCallDiagnoserNotCatchesTailCallEvents(Jit jit, Platform platform, Runtime runtime) { diff --git a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs index 724a882e22..c1a45d4e01 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs @@ -41,7 +41,7 @@ public static IEnumerable GetToolchains() // yield return new object[] { InProcessEmitToolchain.Instance }; } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void CompletedWorkItemCountIsAccurate(IToolchain toolchain) { var config = CreateConfig(toolchain); @@ -82,7 +82,7 @@ public void CompleteOneWorkItem() public void DoNothing() { } } - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void LockContentionCountIsAccurate(IToolchain toolchain) { var config = CreateConfig(toolchain); diff --git a/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs b/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs index f6a5d10428..e4eb071029 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs @@ -22,7 +22,7 @@ public static IEnumerable GetToolchains() => new[] new object[] { InProcessEmitToolchain.Instance }, }; - [Theory, MemberData(nameof(GetToolchains))] + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void AnyValueCanBeReturned(IToolchain toolchain) => CanExecute(ManualConfig.CreateEmpty().AddJob(Job.Dry.WithToolchain(toolchain))); public class ValuesReturnedByBenchmark From ed5316b8309c373204a471c57a65e363d399139c Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sun, 15 Jun 2025 15:15:19 +0900 Subject: [PATCH 10/15] feat: Add validator for benchmarks that contains null runtime (#2771) * feat: add validator for job that contains null runtime * chore: update validator logics * Update src/BenchmarkDotNet/Validators/RuntimeValidator.cs Co-authored-by: Tim Cassell * chore: fix test validation messages --------- Co-authored-by: Tim Cassell --- src/BenchmarkDotNet/Configs/DefaultConfig.cs | 1 + .../Validators/RuntimeValidator.cs | 43 ++++++ .../Validators/RuntimeValidatorTests.cs | 125 ++++++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 src/BenchmarkDotNet/Validators/RuntimeValidator.cs create mode 100644 tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index 8585a77797..d188b67415 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -72,6 +72,7 @@ public IEnumerable GetValidators() yield return DeferredExecutionValidator.FailOnError; yield return ParamsAllValuesValidator.FailOnError; yield return ParamsValidator.FailOnError; + yield return RuntimeValidator.DontFailOnError; } public IOrderer Orderer => null; diff --git a/src/BenchmarkDotNet/Validators/RuntimeValidator.cs b/src/BenchmarkDotNet/Validators/RuntimeValidator.cs new file mode 100644 index 0000000000..7be547890c --- /dev/null +++ b/src/BenchmarkDotNet/Validators/RuntimeValidator.cs @@ -0,0 +1,43 @@ +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Characteristics; + +namespace BenchmarkDotNet.Validators; + +/// +/// Validator for runtime characteristic. +/// +/// +public class RuntimeValidator : IValidator +{ + public static readonly IValidator DontFailOnError = new RuntimeValidator(); + + private RuntimeValidator() { } + + public bool TreatsWarningsAsErrors => false; + + public IEnumerable Validate(ValidationParameters input) + { + var allBenchmarks = input.Benchmarks.ToArray(); + var nullRuntimeBenchmarks = allBenchmarks.Where(x => x.Job.Environment.Runtime == null).ToArray(); + + // There is no validation error if all the runtimes are set or if all the runtimes are null. + if (allBenchmarks.Length == nullRuntimeBenchmarks.Length) + { + return []; + } + + var errors = new List(); + foreach (var benchmark in nullRuntimeBenchmarks) + { + var job = benchmark.Job; + var jobText = job.HasValue(CharacteristicObject.IdCharacteristic) + ? job.Id + : CharacteristicSetPresenter.Display.ToPresentation(job); // Use job text representation instead for auto generated JobId. + + var message = $"Job({jobText}) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; + errors.Add(new ValidationError(false, message)); + } + return errors; + } +} diff --git a/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs new file mode 100644 index 0000000000..9eac2dd6f2 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs @@ -0,0 +1,125 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Validators; +using System.Linq; +using Xunit; + +namespace BenchmarkDotNet.Tests.Validators; + +public class RuntimeValidatorTests +{ + [Fact] + public void SameRuntime_Should_Success() + { + // Arrange + var config = new TestConfig1().CreateImmutableConfig(); + var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); + var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); + + // Act + var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray(); + + // Assert + Assert.Empty(errors); + } + + [Fact] + public void NullRuntimeMixed_Should_Failed() + { + // Arrange + var config = new TestConfig2().CreateImmutableConfig(); + var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); + var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); + + // Act + var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray(); + + // Assert + { + var expectedMessage = "Job(Dry) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; + Assert.Contains(expectedMessage, errors); + } + { + var expectedMessage = "Job(Toolchain=.NET 10.0) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; + Assert.Contains(expectedMessage, errors); + } + } + + [Fact] + public void NotNullRuntimeOnly_Should_Success() + { + // Arrange + var config = new TestConfig3().CreateImmutableConfig(); + var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); + var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); + + // Act + var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray(); + + // Assert + Assert.Empty(errors); + } + + public class DummyBenchmark + { + [Benchmark] + public void Benchmark() + { + } + } + + // TestConfig that expicitly specify runtime. + private class TestConfig1 : ManualConfig + { + public TestConfig1() + { + var baseJob = Job.Dry; + + WithOption(ConfigOptions.DisableOptimizationsValidator, true); + AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); + + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80)); + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90)); + } + } + + // TestConfig that contains job that don't specify runtime. + private class TestConfig2 : ManualConfig + { + public TestConfig2() + { + var baseJob = Job.Dry; + + WithOption(ConfigOptions.DisableOptimizationsValidator, true); + AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); + + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80)); + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90)); + + // Validate error message for auto generated jobid. + AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0)); + } + } + + // TestConfig that expicitly specify runtime. + private class TestConfig3 : ManualConfig + { + public TestConfig3() + { + var baseJob = Job.Dry; + + WithOption(ConfigOptions.DisableOptimizationsValidator, true); + AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); + + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80) + .WithRuntime(CoreRuntime.Core80)); ; + AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90)); + } + } +} From 5db72844abbfd0ea317397ac3da6f5f4ee24b602 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Sat, 14 Jun 2025 21:01:36 +0900 Subject: [PATCH 11/15] fix: auto-generate jobid between benchmark runs --- src/BenchmarkDotNet/Jobs/JobIdGenerator.cs | 15 +++++++++-- .../Configs/ImmutableConfigTests.cs | 2 +- .../Jobs/JobIdGeneratorTests.cs | 27 +++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 tests/BenchmarkDotNet.Tests/Jobs/JobIdGeneratorTests.cs diff --git a/src/BenchmarkDotNet/Jobs/JobIdGenerator.cs b/src/BenchmarkDotNet/Jobs/JobIdGenerator.cs index fa8796d112..a208b3c9b7 100644 --- a/src/BenchmarkDotNet/Jobs/JobIdGenerator.cs +++ b/src/BenchmarkDotNet/Jobs/JobIdGenerator.cs @@ -10,12 +10,23 @@ public static string GenerateRandomId(Job job) string presentation = CharacteristicSetPresenter.Display.ToPresentation(job); if (presentation == "") return "DefaultJob"; - int seed = presentation.GetHashCode(); + int seed = GetStableHashCode(presentation); var random = new Random(seed); string id = ""; for (int i = 0; i < 6; i++) - id += (char) ('A' + random.Next(26)); + id += (char)('A' + random.Next(26)); return "Job-" + id; } + + // Compute string hash value with DJB2 algorithm. + private static int GetStableHashCode(string value) + { + uint hash = 5381; + foreach (char c in value) + { + hash = ((hash << 5) + hash) + c; // hash * 32 + hash + c + } + return unchecked((int)hash); + } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs index d77c5202ed..c68134b7cd 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs @@ -312,7 +312,7 @@ public void WhenTwoConfigsAreAddedTheMutatorJobsAreAppliedToDefaultJobIfCustomDe Assert.Equal(warmupCount, mergedJob.Run.WarmupCount); Assert.False(mergedJob.Meta.IsDefault); // after the merge the "child" job becomes a standard job Assert.False(mergedJob.Meta.IsMutator); // after the merge the "child" job becomes a standard job - Assert.Single(mergedJob.GetCharacteristicsWithValues(), changedCharacteristic => ReferenceEquals(changedCharacteristic, Jobs.RunMode.WarmupCountCharacteristic)); + Assert.Single(mergedJob.GetCharacteristicsWithValues(), changedCharacteristic => ReferenceEquals(changedCharacteristic, BenchmarkDotNet.Jobs.RunMode.WarmupCountCharacteristic)); } } diff --git a/tests/BenchmarkDotNet.Tests/Jobs/JobIdGeneratorTests.cs b/tests/BenchmarkDotNet.Tests/Jobs/JobIdGeneratorTests.cs new file mode 100644 index 0000000000..c0e845bb7a --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Jobs/JobIdGeneratorTests.cs @@ -0,0 +1,27 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.CsProj; +using Xunit; + +namespace BenchmarkDotNet.Tests.Jobs; + +public class JobIdGeneratorTests +{ + [Theory] + [MemberData(nameof(GetTheoryData), DisableDiscoveryEnumeration = true)] + public void AutoGenerateJobId(string expectedId, Job job) + { + // Act + var result = job.ResolvedId; + + // Assert + Assert.Equal(expectedId, result); + } + + public static TheoryData GetTheoryData() => new TheoryData() + { + {"Job-OOTPKI", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp80) }, + {"Job-QAODSR", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp90) }, + {"Job-KHMDUZ", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp80).WithRuntime(CoreRuntime.Core80) }, + }; +} From 27d864a41b3d97b46f45cfed747f477b82d718ab Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Mon, 16 Jun 2025 08:56:12 +0900 Subject: [PATCH 12/15] chore: fix flaky memory allocation test (#2782) Update MemoryDiagnoserTests.cs t --- .../MemoryDiagnoserTests.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index 169d18f454..9d92b38755 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -245,7 +245,7 @@ public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) AssertAllocations(toolchain, typeof(TimeConsuming), new Dictionary { { nameof(TimeConsuming.SixtyFourBytesArray), 64 + objectAllocationOverhead + arraySizeOverhead } - }); + }, warmupCount: 2); } public class MultiThreadedAllocation @@ -315,9 +315,9 @@ public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolc }); } - private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators, bool disableTieredJit = true, int iterationCount = 1) + private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators, bool disableTieredJit = true, int iterationCount = 1, int warmupCount = 0) { - var config = CreateConfig(toolchain, disableTieredJit, iterationCount); + var config = CreateConfig(toolchain, disableTieredJit, iterationCount, warmupCount); var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType, config); var summary = BenchmarkRunner.Run(benchmarks); @@ -358,11 +358,13 @@ private IConfig CreateConfig(IToolchain toolchain, // This was mostly fixed in net7.0, but tiered jit thread is not guaranteed to not allocate, so we disable it just in case. bool disableTieredJit = true, // Single iteration is enough for most of the tests. - int iterationCount = 1) + int iterationCount = 1, + // Don't run warmup by default to save some time for our CI runs + int warmupCount = 0) { var job = Job.ShortRun .WithEvaluateOverhead(false) // no need to run idle for this test - .WithWarmupCount(0) // don't run warmup to save some time for our CI runs + .WithWarmupCount(warmupCount) .WithIterationCount(iterationCount) .WithGcForce(false) .WithGcServer(false) From 5f1dcd8edba23ff9dfc6cb84f7363148aeb190fb Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Mon, 16 Jun 2025 08:57:12 +0900 Subject: [PATCH 13/15] chore: skip null runtime validation for inprocess toolchain (#2780) --- src/BenchmarkDotNet/Validators/RuntimeValidator.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Validators/RuntimeValidator.cs b/src/BenchmarkDotNet/Validators/RuntimeValidator.cs index 7be547890c..debadf880e 100644 --- a/src/BenchmarkDotNet/Validators/RuntimeValidator.cs +++ b/src/BenchmarkDotNet/Validators/RuntimeValidator.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Linq; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Toolchains; namespace BenchmarkDotNet.Validators; @@ -28,7 +29,7 @@ public IEnumerable Validate(ValidationParameters input) } var errors = new List(); - foreach (var benchmark in nullRuntimeBenchmarks) + foreach (var benchmark in nullRuntimeBenchmarks.Where(x=> !x.GetToolchain().IsInProcess)) { var job = benchmark.Job; var jobText = job.HasValue(CharacteristicObject.IdCharacteristic) From 973f6b9a0eb1be8cda87a73b0472d9d334fb6a98 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Mon, 16 Jun 2025 12:09:25 +0200 Subject: [PATCH 14/15] [build] Allow workflow_dispatch for publish-nightly workflow --- .github/workflows/publish-nightly.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-nightly.yaml b/.github/workflows/publish-nightly.yaml index f3e025a6de..01f439f41e 100644 --- a/.github/workflows/publish-nightly.yaml +++ b/.github/workflows/publish-nightly.yaml @@ -5,6 +5,7 @@ on: push: branches: - master + workflow_dispatch: jobs: publish: From 2306babcc2430ee50f1c3ff6ba40359ae0799ee5 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Mon, 16 Jun 2025 13:15:13 +0200 Subject: [PATCH 15/15] [build] Enable --force-clone for docs-fetch in generate-gh-pages.yaml --- .github/workflows/generate-gh-pages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate-gh-pages.yaml b/.github/workflows/generate-gh-pages.yaml index 170ace69b1..8983ed711f 100644 --- a/.github/workflows/generate-gh-pages.yaml +++ b/.github/workflows/generate-gh-pages.yaml @@ -23,7 +23,7 @@ jobs: run: ./build.cmd build - name: Fetch changelog - run: ./build.cmd docs-fetch --depth 1 --preview + run: ./build.cmd docs-fetch --depth 1 --preview --force-clone env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}