From fb4bb1f9f5388d09b94dec8dd0cbaa617cc2dbe5 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 13:38:47 +0200 Subject: [PATCH 01/17] [main] Update dependencies from dotnet/sourcelink (#13786) * Update dependencies from https://github.com/dotnet/sourcelink build 20230609.2 Microsoft.SourceLink.AzureRepos.Git , Microsoft.SourceLink.GitHub From Version 8.0.0-beta.23252.2 -> To Version 8.0.0-beta.23309.2 * Update dependencies from https://github.com/dotnet/sourcelink build 20230609.3 Microsoft.SourceLink.AzureRepos.Git , Microsoft.SourceLink.GitHub From Version 8.0.0-beta.23252.2 -> To Version 8.0.0-beta.23309.3 --------- Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2be0162a5cc..f065cc3286a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -60,14 +60,14 @@ https://github.com/dotnet/linker 3efd231da430baa0fd670e278f6b5c3e62834bde - + https://github.com/dotnet/sourcelink - 54eb3b811c57f5e94617d31a102fc9cb664ccdd5 + 23bda65700e70b6697390dcc4e0f87e2dfbce63a - + https://github.com/dotnet/sourcelink - 54eb3b811c57f5e94617d31a102fc9cb664ccdd5 + 23bda65700e70b6697390dcc4e0f87e2dfbce63a https://github.com/dotnet/symreader-converter diff --git a/eng/Versions.props b/eng/Versions.props index f80fc2364df..7d6365f1764 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -63,8 +63,8 @@ 8.0.100-preview.3.23178.3 - 8.0.0-beta.23252.2 - 8.0.0-beta.23252.2 + 8.0.0-beta.23309.3 + 8.0.0-beta.23309.3 1.1.0-beta2-19575-01 From d78246c21c4fae0e4b10f4b9e63acad68dc4da3f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 13:39:02 +0200 Subject: [PATCH 02/17] Update dependencies from https://github.com/dotnet/arcade build 20230609.8 (#13789) Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Feed , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.SignTool , Microsoft.DotNet.SwaggerGenerator.MSBuild From Version 8.0.0-beta.23308.2 -> To Version 8.0.0-beta.23309.8 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 20 ++++++++++---------- eng/Versions.props | 6 +++--- global.json | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f065cc3286a..104305f14f9 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -19,26 +19,26 @@ https://github.com/dotnet/templating 9982bb9be5c0d2ceca1776b8eb12392f7d3bf613 - + https://github.com/dotnet/arcade - 1205248d5cc014b68c2fb8cfbf091a77aa433453 + 9994c5ed43856c35591d94ead68ef501a020eb2e - + https://github.com/dotnet/arcade - 1205248d5cc014b68c2fb8cfbf091a77aa433453 + 9994c5ed43856c35591d94ead68ef501a020eb2e - + https://github.com/dotnet/arcade - 1205248d5cc014b68c2fb8cfbf091a77aa433453 + 9994c5ed43856c35591d94ead68ef501a020eb2e - + https://github.com/dotnet/arcade - 1205248d5cc014b68c2fb8cfbf091a77aa433453 + 9994c5ed43856c35591d94ead68ef501a020eb2e - + https://github.com/dotnet/arcade - 1205248d5cc014b68c2fb8cfbf091a77aa433453 + 9994c5ed43856c35591d94ead68ef501a020eb2e https://github.com/dotnet/arcade-services diff --git a/eng/Versions.props b/eng/Versions.props index 7d6365f1764..4a7b2a5514f 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -8,9 +8,9 @@ - 8.0.0-beta.23308.2 - 8.0.0-beta.23308.2 - 8.0.0-beta.23308.2 + 8.0.0-beta.23309.8 + 8.0.0-beta.23309.8 + 8.0.0-beta.23309.8 1.1.0-beta.21553.1 1.1.0-beta.22076.4 diff --git a/global.json b/global.json index 6a41f5a086d..f8acbd162c8 100644 --- a/global.json +++ b/global.json @@ -3,8 +3,8 @@ "dotnet": "8.0.100-preview.4.23260.5" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23308.2", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23308.2", + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23309.8", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23309.8", "Microsoft.Build.NoTargets": "3.7.0" } } From 84e2aa322c4fc102110c40f8b99066e5aed3a2af Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Mon, 12 Jun 2023 13:39:29 +0200 Subject: [PATCH 03/17] update minVsReqd in tools.ps1 (#13782) --- eng/common/tools.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index ffe0b4e2dfc..c9eced9f7df 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -379,7 +379,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } # Minimum VS version to require. - $vsMinVersionReqdStr = '16.8' + $vsMinVersionReqdStr = '17.6' $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) # If the version of msbuild is going to be xcopied, From f3a26a53283a3740c6004f7a47aa628d3042adbd Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 13:02:25 +0000 Subject: [PATCH 04/17] Update dependencies from https://github.com/dotnet/xharness build 20230607.1 (#13794) [main] Update dependencies from dotnet/xharness --- eng/Version.Details.xml | 4 ++-- eng/Versions.props | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 104305f14f9..6ea701e0d2c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -48,9 +48,9 @@ https://github.com/dotnet/arcade-services a5f3ed9d5f560555ff6d26b286acdcfbb7ce3b14 - + https://github.com/dotnet/xharness - 230501dc45d8c96e995ceed1189852eb0ee08693 + 98cbe592ee7494c5813d0e5173bc7a160fbcce70 https://github.com/dotnet/roslyn diff --git a/eng/Versions.props b/eng/Versions.props index 4a7b2a5514f..9a1842addbd 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -72,7 +72,7 @@ 17.5.0 - 8.0.0-prerelease.23279.1 + 8.0.0-prerelease.23307.1 1.0.0-beta.23305.1 From ce628a21d7e30b06fe9bc64748d039a95e5e97da Mon Sep 17 00:00:00 2001 From: dotnet bot Date: Mon, 12 Jun 2023 10:19:47 -0700 Subject: [PATCH 05/17] Add scorecards for 2023-05-03 (#13434) Co-authored-by: Missy Messa <47990216+missymessa@users.noreply.github.com> --- .../Scorecard_2023-05-03.md | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 Documentation/TeamProcess/Rollout-Scorecards/Scorecard_2023-05-03.md diff --git a/Documentation/TeamProcess/Rollout-Scorecards/Scorecard_2023-05-03.md b/Documentation/TeamProcess/Rollout-Scorecards/Scorecard_2023-05-03.md new file mode 100644 index 00000000000..7eaed69ac1f --- /dev/null +++ b/Documentation/TeamProcess/Rollout-Scorecards/Scorecard_2023-05-03.md @@ -0,0 +1,75 @@ +# 03 May 2023 Rollout Summaries + +## arcade-services + +| Metric | Value | Target | Score | +|:--------------------------------:|:--------:|:--------:|:---------:| +| Time to Rollout | 02:03:56 | 06:00:00 | 0 | +| Critical/blocking issues created | 0 | 0 | 0 | +| Hotfixes | 0 | 0 | 0 | +| Rollbacks | 0 | 0 | 0 | +| Service downtime | 00:00:00 | 00:00:00 | 0 | +| Failed to rollout | FALSE | FALSE | 0 | +| Total | | | **0** | + + +## dotnet-helix-service + +| Metric | Value | Target | Score | +|:--------------------------------:|:--------:|:--------:|:---------:| +| Time to Rollout | 03:59:25 | 06:00:00 | 0 | +| Critical/blocking issues created | 0 | 0 | 0 | +| Hotfixes | 0 | 0 | 0 | +| Rollbacks | 1 | 0 | 10 | +| Service downtime | 00:00:00 | 00:00:00 | 0 | +| Failed to rollout | TRUE | FALSE | 50 | +| Total | | | **60** | + + +## dotnet-helix-machines + +| Metric | Value | Target | Score | +|:--------------------------------:|:--------:|:--------:|:---------:| +| Time to Rollout | 04:21:50 | 06:00:00 | 0 | +| Critical/blocking issues created | 0 | 0 | 0 | +| Hotfixes | 0 | 0 | 0 | +| Rollbacks | 0 | 0 | 0 | +| Service downtime | 00:00:00 | 00:00:00 | 0 | +| Failed to rollout | FALSE | FALSE | 0 | +| Total | | | **0** | + + +# Itemized Scorecard + +## arcade-services + +| Metric | [20230503.3](https://dev.azure.com/dnceng/7ea9116e-9fac-403d-b258-b31fcf1bb293/_build/results?buildId=2172754) | +|:-----:|:-----:| +| Time to Rollout | 02:03:56 | +| Critical/blocking issues created | 0 | +| Hotfixes | 0 | +| Rollbacks | 0 | +| Service downtime | 00:00:00 | + + +## dotnet-helix-service + +| Metric | [2023050301](https://dev.azure.com/dnceng/7ea9116e-9fac-403d-b258-b31fcf1bb293/_build/results?buildId=2172779) | [2023050302](https://dev.azure.com/dnceng/7ea9116e-9fac-403d-b258-b31fcf1bb293/_build/results?buildId=2173059) | +|:-----:|:-----:|:-----:| +| Time to Rollout | 02:03:52 | 01:55:33 | +| Critical/blocking issues created | 0 | 0 | +| Hotfixes | 0 | 0 | +| Rollbacks | 0 | 1 | +| Service downtime | 00:00:00 | 00:00:00 | + + +## dotnet-helix-machines + +| Metric | [20230503.02](https://dev.azure.com/dnceng/7ea9116e-9fac-403d-b258-b31fcf1bb293/_build/results?buildId=2172777) | +|:-----:|:-----:| +| Time to Rollout | 04:21:50 | +| Critical/blocking issues created | 0 | +| Hotfixes | 0 | +| Rollbacks | 0 | +| Service downtime | 00:00:00 | + From c5d48f954888151c1daabe206956aea894749c6a Mon Sep 17 00:00:00 2001 From: dotnet bot Date: Mon, 12 Jun 2023 10:37:50 -0700 Subject: [PATCH 06/17] Add scorecards for 2023-05-10, 2023-05-17 (#13624) Co-authored-by: Missy Messa <47990216+missymessa@users.noreply.github.com> --- .../Scorecard_2023-05-10.md | 20 ++++---- .../Scorecard_2023-05-17.md | 51 +++++++++++++++++++ 2 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 Documentation/TeamProcess/Rollout-Scorecards/Scorecard_2023-05-17.md diff --git a/Documentation/TeamProcess/Rollout-Scorecards/Scorecard_2023-05-10.md b/Documentation/TeamProcess/Rollout-Scorecards/Scorecard_2023-05-10.md index 69f98f7d5f5..dda74b6b9ba 100644 --- a/Documentation/TeamProcess/Rollout-Scorecards/Scorecard_2023-05-10.md +++ b/Documentation/TeamProcess/Rollout-Scorecards/Scorecard_2023-05-10.md @@ -4,24 +4,24 @@ | Metric | Value | Target | Score | |:--------------------------------:|:--------:|:--------:|:---------:| -| Time to Rollout | 23:27:15 | 06:00:00 | 50 | +| Time to Rollout | 1.01:34:17 | 06:00:00 | 50 | | Critical/blocking issues created | 0 | 0 | 0 | | Hotfixes | 0 | 0 | 0 | | Rollbacks | 0 | 0 | 0 | | Service downtime | 00:00:00 | 00:00:00 | 0 | -| Failed to rollout | TRUE | FALSE | 50 | -| Total | | | **100** | +| Failed to rollout | FALSE | FALSE | 0 | +| Total | | | **50** | # Itemized Scorecard ## arcade-services -| Metric | [20230510.1](https://dev.azure.com/dnceng/7ea9116e-9fac-403d-b258-b31fcf1bb293/_build/results?buildId=2177776) | [20230510.2](https://dev.azure.com/dnceng/7ea9116e-9fac-403d-b258-b31fcf1bb293/_build/results?buildId=2177932) | [20230510.4](https://dev.azure.com/dnceng/7ea9116e-9fac-403d-b258-b31fcf1bb293/_build/results?buildId=2178116) | -|:-----:|:-----:|:-----:|:-----:| -| Time to Rollout | 01:44:04 | 03:50:21 | 17:52:50 | -| Critical/blocking issues created | 0 | 0 | 0 | -| Hotfixes | 0 | 0 | 0 | -| Rollbacks | 0 | 0 | 0 | -| Service downtime | 00:00:00 | 00:00:00 | 00:00:00 | +| Metric | [20230510.1](https://dev.azure.com/dnceng/7ea9116e-9fac-403d-b258-b31fcf1bb293/_build/results?buildId=2177776) | [20230510.2](https://dev.azure.com/dnceng/7ea9116e-9fac-403d-b258-b31fcf1bb293/_build/results?buildId=2177932) | [20230510.4](https://dev.azure.com/dnceng/7ea9116e-9fac-403d-b258-b31fcf1bb293/_build/results?buildId=2178116) | [20230517.1](https://dev.azure.com/dnceng/7ea9116e-9fac-403d-b258-b31fcf1bb293/_build/results?buildId=2182427) | +|:-----:|:-----:|:-----:|:-----:|:-----:| +| Time to Rollout | 01:44:04 | 03:50:21 | 17:52:50 | 02:07:02 | +| Critical/blocking issues created | 0 | 0 | 0 | 0 | +| Hotfixes | 0 | 0 | 0 | 0 | +| Rollbacks | 0 | 0 | 0 | 0 | +| Service downtime | 00:00:00 | 00:00:00 | 00:00:00 | 00:00:00 | diff --git a/Documentation/TeamProcess/Rollout-Scorecards/Scorecard_2023-05-17.md b/Documentation/TeamProcess/Rollout-Scorecards/Scorecard_2023-05-17.md new file mode 100644 index 00000000000..891ed529f7d --- /dev/null +++ b/Documentation/TeamProcess/Rollout-Scorecards/Scorecard_2023-05-17.md @@ -0,0 +1,51 @@ +# 17 May 2023 Rollout Summaries + +## dotnet-helix-service + +| Metric | Value | Target | Score | +|:--------------------------------:|:--------:|:--------:|:---------:| +| Time to Rollout | 01:55:14 | 06:00:00 | 0 | +| Critical/blocking issues created | 0 | 0 | 0 | +| Hotfixes | 0 | 0 | 0 | +| Rollbacks | 0 | 0 | 0 | +| Service downtime | 00:00:00 | 00:00:00 | 0 | +| Failed to rollout | FALSE | FALSE | 0 | +| Total | | | **0** | + + +## dotnet-helix-machines + +| Metric | Value | Target | Score | +|:--------------------------------:|:--------:|:--------:|:---------:| +| Time to Rollout | 04:12:56 | 06:00:00 | 0 | +| Critical/blocking issues created | 0 | 0 | 0 | +| Hotfixes | 0 | 0 | 0 | +| Rollbacks | 0 | 0 | 0 | +| Service downtime | 00:00:00 | 00:00:00 | 0 | +| Failed to rollout | FALSE | FALSE | 0 | +| Total | | | **0** | + + +# Itemized Scorecard + +## dotnet-helix-service + +| Metric | [2023051701](https://dev.azure.com/dnceng/7ea9116e-9fac-403d-b258-b31fcf1bb293/_build/results?buildId=2182422) | +|:-----:|:-----:| +| Time to Rollout | 01:55:14 | +| Critical/blocking issues created | 0 | +| Hotfixes | 0 | +| Rollbacks | 0 | +| Service downtime | 00:00:00 | + + +## dotnet-helix-machines + +| Metric | [20230517.01](https://dev.azure.com/dnceng/7ea9116e-9fac-403d-b258-b31fcf1bb293/_build/results?buildId=2182188) | +|:-----:|:-----:| +| Time to Rollout | 04:12:56 | +| Critical/blocking issues created | 0 | +| Hotfixes | 0 | +| Rollbacks | 0 | +| Service downtime | 00:00:00 | + From f3952775e6d00a5b6f43b0615a8a766e095185eb Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 12 Jun 2023 11:25:07 -0700 Subject: [PATCH 07/17] Make sure to emit the override header directive before the command and put spaces before each argument (#13796) --- .../deb-package-tool/scripts/config_template_generator.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.DotNet.Build.Tasks.Installers/build/deb-package-tool/scripts/config_template_generator.py b/src/Microsoft.DotNet.Build.Tasks.Installers/build/deb-package-tool/scripts/config_template_generator.py index 5fc9343f504..6b9843ecd38 100644 --- a/src/Microsoft.DotNet.Build.Tasks.Installers/build/deb-package-tool/scripts/config_template_generator.py +++ b/src/Microsoft.DotNet.Build.Tasks.Installers/build/deb-package-tool/scripts/config_template_generator.py @@ -69,8 +69,10 @@ def generate_rules(config_data, template_dir): ignored_libraries = config_data.get("debian_ignored_libraries", None) override_text = "" - if ignored_dependency_packages != None: + if ignored_dependency_packages != None or ignored_libraries != None: override_text = "override_dh_shlibdeps:\n\tdh_shlibdeps" + + if ignored_dependency_packages != None: override_text += " --dpkg-shlibdeps-params=\"" for package in ignored_dependency_packages: @@ -80,7 +82,7 @@ def generate_rules(config_data, template_dir): if ignored_libraries != None: for library in ignored_libraries: - override_text += "-X{0} ".format(library) + override_text += " -X{0} ".format(library) return template.format(overrides=override_text) From b6f36913f4a3767cb399989b6c0062cd65fac554 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 13 Jun 2023 17:59:19 +0000 Subject: [PATCH 08/17] [main] Update dependencies from dotnet/templating dotnet/xliff-tasks dotnet/source-build-reference-packages dotnet/source-build-externals (#13795) [main] Update dependencies from dotnet/templating dotnet/xliff-tasks dotnet/source-build-reference-packages dotnet/source-build-externals --- eng/Version.Details.xml | 16 ++++++++-------- eng/Versions.props | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6ea701e0d2c..484e674e95c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -15,9 +15,9 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-symuploader 62ceb439e80bf0814d0ffa17f022d4624ea4aa6c - + https://github.com/dotnet/templating - 9982bb9be5c0d2ceca1776b8eb12392f7d3bf613 + 812fc96b2953dfbb5cf46ee60cf47f4e287c8179 https://github.com/dotnet/arcade @@ -77,9 +77,9 @@ https://github.com/dotnet/symreader-converter c5ba7c88f92e2dde156c324a8c8edc04d9fa4fe0 - + https://github.com/dotnet/xliff-tasks - d033e96327903618482de22f76c971c7fe91e8f4 + 4234ffac87e305da80597cb46dc0d4a87fc4f7c2 @@ -94,14 +94,14 @@ 547c506abe05e510bd43330fc8f6d4c5961e9223 - + https://github.com/dotnet/source-build-reference-packages - b0f656e394f8e98f48f11612ac997750bb85ff6c + ceb1133c46abe15ee48ea799950e7a6a74417b74 - + https://github.com/dotnet/source-build-externals - 7f9ae67f86a5adc1d9bf2f22f4bf3ec05b6d7b68 + 50600f7fcd6e56a496233a859ed7d4d90c173b0d diff --git a/eng/Versions.props b/eng/Versions.props index 9a1842addbd..551fdeee47a 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -68,13 +68,13 @@ 1.1.0-beta2-19575-01 - 8.0.100-preview.6.23305.5 + 8.0.100-preview.6.23311.1 17.5.0 8.0.0-prerelease.23307.1 - 1.0.0-beta.23305.1 + 1.0.0-beta.23312.2 1.31.0 12.16.0 From 0e4d820a3b23fcf8aa3220e9d0e33ba95744d1ac Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Tue, 13 Jun 2023 11:28:14 -0700 Subject: [PATCH 09/17] Update to Preview 5 (#13805) --- global.json | 2 +- src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.props | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/global.json b/global.json index f8acbd162c8..11bb5fdf46f 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "8.0.100-preview.4.23260.5" + "dotnet": "8.0.100-preview.5.23303.2" }, "msbuild-sdks": { "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23309.8", diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.props b/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.props index 6ce028b4b67..d51677d46db 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.props +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/dotnet-cli/DotNetCli.props @@ -2,7 +2,7 @@ false - 8.0.0-preview.4.23260.4 + 8.0.0-preview.5.23302.2 runtime $(BundledNETCoreAppPackageVersion) From 74bd8c5340936e8125d1ed89d38817bc06646450 Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Tue, 13 Jun 2023 15:50:58 -0700 Subject: [PATCH 10/17] Reduce likelyhood that large containers cause OOMs when repacking (#13800) * Reduce likelyhood that large containers cause OOMs when repacking - 'Using' archive stream in ZipData.cs - When repacking, make an initial pass over the container set to separate out large/small containers. Then repack small ones in parallel, and large ones in serial. Large and small are determined by a 2GB limit/max parallelism. In the code, we could have used two loops instead of 3. Instead, I prioritized printing the correct number of containers being packed in parallel vs. serial. --- .../src/BatchSignUtil.cs | 68 +++++++++++++++---- src/Microsoft.DotNet.SignTool/src/ZipData.cs | 3 +- 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/src/Microsoft.DotNet.SignTool/src/BatchSignUtil.cs b/src/Microsoft.DotNet.SignTool/src/BatchSignUtil.cs index d1c78ba6689..8b00b9fe55e 100644 --- a/src/Microsoft.DotNet.SignTool/src/BatchSignUtil.cs +++ b/src/Microsoft.DotNet.SignTool/src/BatchSignUtil.cs @@ -214,32 +214,76 @@ void repackGroup(IEnumerable files, out int repackCount) var repackList = files.Where(w => toRepackSet.Contains(w.FullPath)).ToList(); repackCount = repackList.Count(); - if(repackCount == 0) + + if (repackCount == 0) { return; } _log.LogMessage(MessageImportance.High, $"Repacking {repackCount} containers."); + const int repackParallelism = 16; ParallelOptions parallelOptions = new ParallelOptions(); - parallelOptions.MaxDegreeOfParallelism = 16; - Parallel.ForEach(repackList, parallelOptions, file => + parallelOptions.MaxDegreeOfParallelism = repackParallelism; + + // It's possible that there are large containers within this set that, if + // repacked in parallel, could cause OOMs. To avoid this, we set a limit on the size of containers + // that we will repack in parallel based on the parallelism degree and a 2GB limit. + // Repack these in serial later. + var largeRepackList = new List(); + var smallRepackList = new List(); + const long parallelRepackLimitInBytes = (2 * 1024 / repackParallelism) * 1024 * 1024; + + foreach (var file in repackList) { - if (file.IsZipContainer()) + FileInfo fileInfo = new FileInfo(file.FullPath); + if (fileInfo.Length > parallelRepackLimitInBytes) { - _log.LogMessage($"Repacking container: '{file.FileName}'"); - _batchData.ZipDataMap[file.FileContentKey].Repack(_log); - } - else if (file.IsWixContainer()) - { - _log.LogMessage($"Packing wix container: '{file.FileName}'"); - _batchData.ZipDataMap[file.FileContentKey].Repack(_log, _signTool.TempDir, _signTool.WixToolsPath); + largeRepackList.Add(file); } else { - _log.LogError($"Don't know how to repack file '{file.FullPath}'"); + smallRepackList.Add(file); } + } + + _log.LogMessage(MessageImportance.High, $"Repacking {smallRepackList.Count} containers in parallel."); + + Parallel.ForEach(smallRepackList, parallelOptions, file => + { + repackContainer(file); toRepackSet.Remove(file.FullPath); }); + + if (largeRepackList.Count == 0) + { + return; + } + + _log.LogMessage(MessageImportance.High, $"Repacking {largeRepackList.Count} large containers in serial."); + + foreach (var file in largeRepackList) + { + repackContainer(file); + toRepackSet.Remove(file.FullPath); + } + } + + void repackContainer(FileSignInfo file) + { + if (file.IsZipContainer()) + { + _log.LogMessage($"Repacking container: '{file.FileName}'"); + _batchData.ZipDataMap[file.FileContentKey].Repack(_log); + } + else if (file.IsWixContainer()) + { + _log.LogMessage($"Packing wix container: '{file.FileName}'"); + _batchData.ZipDataMap[file.FileContentKey].Repack(_log, _signTool.TempDir, _signTool.WixToolsPath); + } + else + { + _log.LogError($"Don't know how to repack file '{file.FullPath}'"); + } } // Is this file ready to be signed or repackaged? That is are all of the items that it depends on already diff --git a/src/Microsoft.DotNet.SignTool/src/ZipData.cs b/src/Microsoft.DotNet.SignTool/src/ZipData.cs index a798d97076e..20bfa532d5e 100644 --- a/src/Microsoft.DotNet.SignTool/src/ZipData.cs +++ b/src/Microsoft.DotNet.SignTool/src/ZipData.cs @@ -113,7 +113,8 @@ string getPartRelativeFileName(PackagePart part) /// private void RepackRawZip(TaskLoggingHelper log) { - using (var archive = new ZipArchive(File.Open(FileSignInfo.FullPath, FileMode.Open), ZipArchiveMode.Update)) + using (var zipStream = File.Open(FileSignInfo.FullPath, FileMode.Open)) + using (var archive = new ZipArchive(zipStream, ZipArchiveMode.Update)) { foreach (ZipArchiveEntry entry in archive.Entries) { From c27937680063bcf06804c9fd0466d26a0f9ea4e6 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 14 Jun 2023 09:40:42 -0700 Subject: [PATCH 11/17] [main] Update dependencies from dotnet/arcade dotnet/source-build-reference-packages (#13808) * Update dependencies from https://github.com/dotnet/arcade build 20230613.6 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Feed , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.SignTool , Microsoft.DotNet.SwaggerGenerator.MSBuild From Version 8.0.0-beta.23309.8 -> To Version 8.0.0-beta.23313.6 * Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20230614.1 Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 8.0.0-alpha.1.23312.2 -> To Version 8.0.0-alpha.1.23314.1 --------- Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 24 ++++++++++++------------ eng/Versions.props | 6 +++--- global.json | 4 ++-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 484e674e95c..e8f8d5ce4ca 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -19,26 +19,26 @@ https://github.com/dotnet/templating 812fc96b2953dfbb5cf46ee60cf47f4e287c8179 - + https://github.com/dotnet/arcade - 9994c5ed43856c35591d94ead68ef501a020eb2e + 74bd8c5340936e8125d1ed89d38817bc06646450 - + https://github.com/dotnet/arcade - 9994c5ed43856c35591d94ead68ef501a020eb2e + 74bd8c5340936e8125d1ed89d38817bc06646450 - + https://github.com/dotnet/arcade - 9994c5ed43856c35591d94ead68ef501a020eb2e + 74bd8c5340936e8125d1ed89d38817bc06646450 - + https://github.com/dotnet/arcade - 9994c5ed43856c35591d94ead68ef501a020eb2e + 74bd8c5340936e8125d1ed89d38817bc06646450 - + https://github.com/dotnet/arcade - 9994c5ed43856c35591d94ead68ef501a020eb2e + 74bd8c5340936e8125d1ed89d38817bc06646450 https://github.com/dotnet/arcade-services @@ -94,9 +94,9 @@ 547c506abe05e510bd43330fc8f6d4c5961e9223 - + https://github.com/dotnet/source-build-reference-packages - ceb1133c46abe15ee48ea799950e7a6a74417b74 + f8ebadcc83f7fc8cfd5147078c87d6e583cb32f1 diff --git a/eng/Versions.props b/eng/Versions.props index 551fdeee47a..0366045b9d9 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -8,9 +8,9 @@ - 8.0.0-beta.23309.8 - 8.0.0-beta.23309.8 - 8.0.0-beta.23309.8 + 8.0.0-beta.23313.6 + 8.0.0-beta.23313.6 + 8.0.0-beta.23313.6 1.1.0-beta.21553.1 1.1.0-beta.22076.4 diff --git a/global.json b/global.json index 11bb5fdf46f..d2a36b98c0f 100644 --- a/global.json +++ b/global.json @@ -3,8 +3,8 @@ "dotnet": "8.0.100-preview.5.23303.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23309.8", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23309.8", + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23313.6", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23313.6", "Microsoft.Build.NoTargets": "3.7.0" } } From 728a7d3314ca72bdace354d16ebb400c2e87a59d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Wed, 14 Jun 2023 10:37:40 -0700 Subject: [PATCH 12/17] Remove Source Link package references (#13260) * Remove Source Link package references * Update Versions.props * Update Microsoft.DotNet.Arcade.Sdk.csproj * Update Versions.props --------- Co-authored-by: Matt Mitchell Co-authored-by: Viktor Hofer --- .../Microsoft.DotNet.Arcade.Sdk.csproj | 2 -- .../tools/RepositoryInfo.targets | 8 -------- 2 files changed, 10 deletions(-) diff --git a/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj b/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj index 15d80c32bd4..9267acf92a9 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj +++ b/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj @@ -67,8 +67,6 @@ $(PackageVersion) $(MicrosoftNetCompilersToolsetVersion) $(MicrosoftNetILLinkTasksVersion) - $(MicrosoftSourceLinkGitHubVersion) - $(MicrosoftSourceLinkAzureReposGitVersion) $(MicrosoftDiaSymReaderPdb2PdbVersion) $(MicrosoftDotNetXliffTasksVersion) $(MicrosoftDotNetMaestroTasksVersion) diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/RepositoryInfo.targets b/src/Microsoft.DotNet.Arcade.Sdk/tools/RepositoryInfo.targets index e1a566d2809..faf87d5106f 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/RepositoryInfo.targets +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/RepositoryInfo.targets @@ -1,13 +1,5 @@ - - - - - - false From bbb7c9d3d47bf180be830671d1dad59e3e186090 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 15 Jun 2023 14:30:25 +0000 Subject: [PATCH 13/17] Update dependencies from https://github.com/dotnet/arcade build 20230614.1 (#13816) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 20 ++++++++++---------- eng/Versions.props | 6 +++--- global.json | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e8f8d5ce4ca..73dbcaeac5c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -19,26 +19,26 @@ https://github.com/dotnet/templating 812fc96b2953dfbb5cf46ee60cf47f4e287c8179 - + https://github.com/dotnet/arcade - 74bd8c5340936e8125d1ed89d38817bc06646450 + c27937680063bcf06804c9fd0466d26a0f9ea4e6 - + https://github.com/dotnet/arcade - 74bd8c5340936e8125d1ed89d38817bc06646450 + c27937680063bcf06804c9fd0466d26a0f9ea4e6 - + https://github.com/dotnet/arcade - 74bd8c5340936e8125d1ed89d38817bc06646450 + c27937680063bcf06804c9fd0466d26a0f9ea4e6 - + https://github.com/dotnet/arcade - 74bd8c5340936e8125d1ed89d38817bc06646450 + c27937680063bcf06804c9fd0466d26a0f9ea4e6 - + https://github.com/dotnet/arcade - 74bd8c5340936e8125d1ed89d38817bc06646450 + c27937680063bcf06804c9fd0466d26a0f9ea4e6 https://github.com/dotnet/arcade-services diff --git a/eng/Versions.props b/eng/Versions.props index 0366045b9d9..30367dec645 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -8,9 +8,9 @@ - 8.0.0-beta.23313.6 - 8.0.0-beta.23313.6 - 8.0.0-beta.23313.6 + 8.0.0-beta.23314.1 + 8.0.0-beta.23314.1 + 8.0.0-beta.23314.1 1.1.0-beta.21553.1 1.1.0-beta.22076.4 diff --git a/global.json b/global.json index d2a36b98c0f..a0d8aab4de5 100644 --- a/global.json +++ b/global.json @@ -3,8 +3,8 @@ "dotnet": "8.0.100-preview.5.23303.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23313.6", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23313.6", + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23314.1", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23314.1", "Microsoft.Build.NoTargets": "3.7.0" } } From 7c3a2046a3f28a8bb3f38a2833db1ecc1bb87452 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 16 Jun 2023 09:14:38 +0200 Subject: [PATCH 14/17] [main] Update dependencies from dotnet/sourcelink (#13809) * Update dependencies from https://github.com/dotnet/sourcelink build 20230613.2 Microsoft.SourceLink.AzureRepos.Git , Microsoft.SourceLink.GitHub From Version 8.0.0-beta.23309.3 -> To Version 8.0.0-beta.23313.2 * Update dependencies from https://github.com/dotnet/sourcelink build 20230614.2 Microsoft.SourceLink.AzureRepos.Git , Microsoft.SourceLink.GitHub From Version 8.0.0-beta.23309.3 -> To Version 8.0.0-beta.23314.2 --------- Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 73dbcaeac5c..5fa2f74d263 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -60,14 +60,14 @@ https://github.com/dotnet/linker 3efd231da430baa0fd670e278f6b5c3e62834bde - + https://github.com/dotnet/sourcelink - 23bda65700e70b6697390dcc4e0f87e2dfbce63a + 4d2c8bf58e8cb7900ec2d9077c155572e2d3cd88 - + https://github.com/dotnet/sourcelink - 23bda65700e70b6697390dcc4e0f87e2dfbce63a + 4d2c8bf58e8cb7900ec2d9077c155572e2d3cd88 https://github.com/dotnet/symreader-converter diff --git a/eng/Versions.props b/eng/Versions.props index 30367dec645..d876e448490 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -63,8 +63,8 @@ 8.0.100-preview.3.23178.3 - 8.0.0-beta.23309.3 - 8.0.0-beta.23309.3 + 8.0.0-beta.23314.2 + 8.0.0-beta.23314.2 1.1.0-beta2-19575-01 From 4bbdd777f424056c823dabbe0212ed2ba0a6dddd Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Fri, 16 Jun 2023 16:17:14 +0200 Subject: [PATCH 15/17] Revert "Remove Source Link package references (#13260)" (#13825) This reverts commit 728a7d3314ca72bdace354d16ebb400c2e87a59d. --- .../Microsoft.DotNet.Arcade.Sdk.csproj | 2 ++ .../tools/RepositoryInfo.targets | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj b/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj index 9267acf92a9..15d80c32bd4 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj +++ b/src/Microsoft.DotNet.Arcade.Sdk/Microsoft.DotNet.Arcade.Sdk.csproj @@ -67,6 +67,8 @@ $(PackageVersion) $(MicrosoftNetCompilersToolsetVersion) $(MicrosoftNetILLinkTasksVersion) + $(MicrosoftSourceLinkGitHubVersion) + $(MicrosoftSourceLinkAzureReposGitVersion) $(MicrosoftDiaSymReaderPdb2PdbVersion) $(MicrosoftDotNetXliffTasksVersion) $(MicrosoftDotNetMaestroTasksVersion) diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/RepositoryInfo.targets b/src/Microsoft.DotNet.Arcade.Sdk/tools/RepositoryInfo.targets index faf87d5106f..e1a566d2809 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/RepositoryInfo.targets +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/RepositoryInfo.targets @@ -1,5 +1,13 @@ + + + + + + false From 6dd7a52b6d315a46c0aed6786f58348ffa4fc718 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 16 Jun 2023 15:15:01 +0000 Subject: [PATCH 16/17] [main] Update dependencies from dotnet/source-build-reference-packages dotnet/source-build-externals (#13828) [main] Update dependencies from dotnet/source-build-reference-packages dotnet/source-build-externals --- eng/Version.Details.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5fa2f74d263..6423f316a19 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -94,14 +94,14 @@ 547c506abe05e510bd43330fc8f6d4c5961e9223 - + https://github.com/dotnet/source-build-reference-packages - f8ebadcc83f7fc8cfd5147078c87d6e583cb32f1 + b34c6434a7b422a79e104ef7f24cee0d6ddba3c5 - + https://github.com/dotnet/source-build-externals - 50600f7fcd6e56a496233a859ed7d4d90c173b0d + 06913fc4c3fcb0065ee390d135fb766870d2c38a From e2334b2be36919347923d0ec872a46acddb1e385 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Fri, 16 Jun 2023 18:46:50 +0300 Subject: [PATCH 17/17] Use arm architecture instead of arm64 on 32-bit operating systems (#13392) Fixes dotnet/arcade#13386 --- eng/common/dotnet-install.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index abd045a3247..7e69e3a9e24 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -54,6 +54,10 @@ cpuname=$(uname -m) case $cpuname in arm64|aarch64) buildarch=arm64 + if [ "$(getconf LONG_BIT)" -lt 64 ]; then + # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS) + buildarch=arm + fi ;; loongarch64) buildarch=loongarch64