diff --git a/.spelling b/.spelling index 22137b0c151..e3150d2c138 100644 --- a/.spelling +++ b/.spelling @@ -1403,6 +1403,8 @@ pwsh.exe 6.0.100-rc.2.21505.57 ThirdPartyNotices.txt rtm.21527.11 +SKUs +vmImage - CHANGELOG/7.0.md codesign release-BuildJson diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index e4ddeaf27f8..8efcb91db03 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -30,7 +30,7 @@ pr: - .vsts-ci/misc-analysis.yml - .vsts-ci/windows.yml - .vsts-ci/windows/* - - cgmanifest.json + - tools/cgmanifest.json - LICENSE.txt - test/common/markdown/* - test/perf/* diff --git a/.vsts-ci/mac.yml b/.vsts-ci/mac.yml index 84d3d1ef02a..9392995ab15 100644 --- a/.vsts-ci/mac.yml +++ b/.vsts-ci/mac.yml @@ -31,7 +31,7 @@ pr: - .vsts-ci/misc-analysis.yml - .vsts-ci/windows.yml - .vsts-ci/windows/* - - cgmanifest.json + - tools/cgmanifest.json - LICENSE.txt - test/common/markdown/* - test/perf/* diff --git a/.vsts-ci/misc-analysis/mdSpell.yml b/.vsts-ci/misc-analysis/mdSpell.yml new file mode 100644 index 00000000000..e9d046e5e96 --- /dev/null +++ b/.vsts-ci/misc-analysis/mdSpell.yml @@ -0,0 +1,56 @@ +jobs: +- job: markdown + displayName: Markdown Spelling + + pool: + vmImage: ubuntu-20.04 + + variables: + - name: repoPath + value: $(Agent.BuildDirectory)/$(repoFolder) + + steps: + - checkout: self + clean: true + path: $(repoFolder) + + - checkout: ComplianceRepo + + - powershell: | + Get-ChildItem -Path env: + displayName: Capture Environment + condition: succeededOrFailed() + + - bash: | + curl -o- --progress-bar -L https://yarnpkg.com/install.sh | bash + displayName: Bootstrap Yarn + condition: succeededOrFailed() + + - bash: | + sudo yarn global add markdown-spellcheck@0.11.0 + displayName: Install mdspell + condition: succeededOrFailed() + + - bash: | + mdspell '**/*.md' '!**/Pester/**/*.md' '!**/dotnet-tools/**/*.md' --ignore-numbers --ignore-acronyms --report --en-us; + displayName: Test Spelling in Markdown + condition: succeededOrFailed() + workingDirectory: '$(repoPath)' + + - ${{ if not(contains(variables['SYSTEM.COLLECTIONURI'],'mscodehub')) }}: + - pwsh: | + Import-module ./build.psm1 + $path = Join-Path -Path $pwd -ChildPath './commonTestResults.xml' + $results = invoke-pester -Script ./test/common/markdown-lint -OutputFile $path -OutputFormat NUnitXml -PassThru + Write-Host "##vso[results.publish type=NUnit;mergeResults=true;runTitle=Markdown Lint;publishRunAttachments=true;resultFiles=$path;]" + if($results.TotalCount -eq 0 -or $results.FailedCount -gt 0) + { + throw "Markdown tests failed" + } + displayName: Run Markdown Lint Tests + condition: succeededOrFailed() + workingDirectory: '$(repoPath)' + + - template: dailyBuildCompliance.yml@ComplianceRepo + parameters: + sourceScanPath: '$(repoPath)/test/common' diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index 70775761fba..4386c9efed0 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -28,7 +28,7 @@ pr: - .dependabot/config.yml - .github/ISSUE_TEMPLATE/* - .vsts-ci/misc-analysis.yml - - cgmanifest.json + - tools/cgmanifest.json - LICENSE.txt - test/common/markdown/* - test/perf/* diff --git a/.vsts-ci/windows/templates/windows-packaging.yml b/.vsts-ci/windows/templates/windows-packaging.yml index 353d722bb92..88a557ca032 100644 --- a/.vsts-ci/windows/templates/windows-packaging.yml +++ b/.vsts-ci/windows/templates/windows-packaging.yml @@ -69,6 +69,7 @@ jobs: BuildDropPath: '$(System.ArtifactsDirectory)/mainBuild' Build_Repository_Uri: $(build.repository.uri) displayName: SBOM + sourceScanPath: '$(repoPath)\tools' - pwsh: | Import-Module .\tools\ci.psm1 diff --git a/Analyzers.props b/Analyzers.props index dacf53c4625..d2527a40b7a 100644 --- a/Analyzers.props +++ b/Analyzers.props @@ -2,6 +2,6 @@ - + diff --git a/CHANGELOG/7.2.md b/CHANGELOG/7.2.md index 8eb70874ef5..718f02f8a87 100644 --- a/CHANGELOG/7.2.md +++ b/CHANGELOG/7.2.md @@ -1,5 +1,37 @@ # 7.2 Changelog +## [7.2.2] - 2022-03-16 + +### Build and Packaging Improvements + +
+ + + +

Bump .NET SDK to 6.0.201

+ + + + + +
+ +[7.2.2]: https://github.com/PowerShell/PowerShell/compare/v7.2.1...v7.2.2 + ## [7.2.1] - 2021-12-14 ### General Cmdlet Updates and Fixes diff --git a/DotnetRuntimeMetadata.json b/DotnetRuntimeMetadata.json index 1b95abccd68..225b908f1e2 100644 --- a/DotnetRuntimeMetadata.json +++ b/DotnetRuntimeMetadata.json @@ -1,15 +1,15 @@ { "sdk": { - "channel": "6.0.1xx", - "quality": "signed", + "channel": "6.0.2xx", + "quality": "ga", "qualityFallback": "daily", "packageVersionPattern": "6.0.0", - "sdkImageVersion": "6.0.100", - "nextChannel": "6.0.1xx", - "azureFeed": "https://dotnetstage.blob.core.windows.net/6-0-100-rtm-21527-11-public", - "sdkImageOverride": "6.0.100-rtm.21527.11" + "sdkImageVersion": "6.0.200", + "nextChannel": "6.0.2xx", + "azureFeed": "", + "sdkImageOverride": "" }, "internalfeed" : { - "url": "https://pkgs.dev.azure.com/dnceng/public/_packaging/6.0.100-rtm.21527.11-shipping/nuget/v2" + "url": "" } } diff --git a/assets/wix/files.wxs b/assets/wix/files.wxs index 78b21b8d643..6e1f8ff3a30 100644 --- a/assets/wix/files.wxs +++ b/assets/wix/files.wxs @@ -3154,8 +3154,8 @@ - - + + @@ -4164,7 +4164,6 @@ - @@ -4174,6 +4173,7 @@ + diff --git a/build.psm1 b/build.psm1 index 33a372d13d7..47b8d6267b9 100644 --- a/build.psm1 +++ b/build.psm1 @@ -619,40 +619,33 @@ Fix steps: # publish powershell.config.json $config = @{} - if ($environment.IsWindows) { + + if ($Options.Runtime -like "*win*") { + # Execution Policy is only supported on Windows $config = @{ "Microsoft.PowerShell:ExecutionPolicy" = "RemoteSigned"; - "WindowsPowerShellCompatibilityModuleDenyList" = @("PSScheduledJob","BestPractices","UpdateServices") } + "WindowsPowerShellCompatibilityModuleDenyList" = @("PSScheduledJob", "BestPractices", "UpdateServices") + } } - # When building preview, we want the configuration to enable all experiemental features by default - # ARM is cross compiled, so we can't run pwsh to enumerate Experimental Features if (-not $SkipExperimentalFeatureGeneration -and (Test-IsPreview $psVersion) -and - -not (Test-IsReleaseCandidate $psVersion) -and - -not $Runtime.Contains("arm") -and - -not ($Runtime -like 'fxdependent*')) { - - $json = & $publishPath\pwsh -noprofile -command { - # Special case for DSC code in PS; - # this experimental feature requires new DSC module that is not inbox, - # so we don't want default DSC use case be broken - [System.Collections.ArrayList] $expFeatures = Get-ExperimentalFeature | Where-Object Name -NE PS7DscSupport | ForEach-Object -MemberName Name - - $expFeatures | Out-String | Write-Verbose -Verbose - - # Make sure ExperimentalFeatures from modules in PSHome are added - # https://github.com/PowerShell/PowerShell/issues/10550 - $ExperimentalFeaturesFromGalleryModulesInPSHome = @() - $ExperimentalFeaturesFromGalleryModulesInPSHome | ForEach-Object { - if (!$expFeatures.Contains($_)) { - $null = $expFeatures.Add($_) - } - } + -not (Test-IsReleaseCandidate $psVersion) + ) { + + $ExperimentalFeatureJsonFilePath = if ($Options.Runtime -like "*win*") { + "$PSScriptRoot/experimental-feature-windows.json" + } else { + "$PSScriptRoot/experimental-feature-linux.json" + } - ConvertTo-Json $expFeatures + if (-not (Test-Path $ExperimentalFeatureJsonFilePath)) { + throw "ExperimentalFeatureJsonFilePath: $ExperimentalFeatureJsonFilePath does not exist" } + $json = Get-Content -Raw $ExperimentalFeatureJsonFilePath $config += @{ ExperimentalFeatures = ([string[]] ($json | ConvertFrom-Json)) } + } else { + Write-Warning -Message "Experimental features are not enabled in powershell.config.json file" } if ($config.Count -gt 0) { @@ -1894,6 +1887,43 @@ function Get-RedHatPackageManager { } } +function Install-GlobalGem { + param( + [Parameter()] + [string] + $Sudo = "", + + [Parameter(Mandatory)] + [string] + $GemName, + + [Parameter(Mandatory)] + [string] + $GemVersion + ) + try { + # We cannot guess if the user wants to run gem install as root on linux and windows, + # but macOs usually requires sudo + $gemsudo = '' + if($environment.IsMacOS -or $env:TF_BUILD) { + $gemsudo = $sudo + } + + Start-NativeExecution ([ScriptBlock]::Create("$gemsudo gem install $GemName -v $GemVersion --no-document")) + + } catch { + Write-Warning "Installation of gem $GemName $GemVersion failed! Must resolve manually." + $logs = Get-ChildItem "/var/lib/gems/*/extensions/x86_64-linux/*/$GemName-*/gem_make.out" | Select-Object -ExpandProperty FullName + foreach ($log in $logs) { + Write-Verbose "Contents of: $log" -Verbose + Get-Content -Raw -Path $log -ErrorAction Ignore | ForEach-Object { Write-Verbose $_ -Verbose } + Write-Verbose "END Contents of: $log" -Verbose + } + + throw + } +} + function Start-PSBootstrap { [CmdletBinding()] param( @@ -1930,11 +1960,7 @@ function Start-PSBootstrap { $Deps = @() if ($environment.IsLinux -and $environment.IsUbuntu) { # Build tools - $Deps += "curl", "g++", "cmake", "make" - - if ($BuildLinuxArm) { - $Deps += "gcc-arm-linux-gnueabihf", "g++-arm-linux-gnueabihf" - } + $Deps += "curl", "wget" # .NET Core required runtime libraries $Deps += "libunwind8" @@ -1942,7 +1968,7 @@ function Start-PSBootstrap { elseif ($environment.IsUbuntu18) { $Deps += "libicu60"} # Packaging tools - if ($Package) { $Deps += "ruby-dev", "groff", "libffi-dev" } + if ($Package) { $Deps += "ruby-dev", "groff", "libffi-dev", "rpm", "g++", "make" } # Install dependencies # change the fontend from apt-get to noninteractive @@ -1960,13 +1986,13 @@ function Start-PSBootstrap { } } elseif ($environment.IsLinux -and $environment.IsRedHatFamily) { # Build tools - $Deps += "which", "curl", "gcc-c++", "cmake", "make" + $Deps += "which", "curl", "wget" # .NET Core required runtime libraries $Deps += "libicu", "libunwind" # Packaging tools - if ($Package) { $Deps += "ruby-devel", "rpm-build", "groff", 'libffi-devel' } + if ($Package) { $Deps += "ruby-devel", "rpm-build", "groff", 'libffi-devel', "gcc-c++" } $PackageManager = Get-RedHatPackageManager @@ -1984,10 +2010,10 @@ function Start-PSBootstrap { } } elseif ($environment.IsLinux -and $environment.IsSUSEFamily) { # Build tools - $Deps += "gcc", "cmake", "make" + $Deps += "wget" # Packaging tools - if ($Package) { $Deps += "ruby-devel", "rpmbuild", "groff", 'libffi-devel' } + if ($Package) { $Deps += "ruby-devel", "rpmbuild", "groff", 'libffi-devel', "gcc" } $PackageManager = "zypper --non-interactive install" $baseCommand = "$sudo $PackageManager" @@ -2009,8 +2035,8 @@ function Start-PSBootstrap { $PackageManager = "$sudo port" } - # Build tools - $Deps += "cmake" + # wget for downloading dotnet + $Deps += "wget" # .NET Core required runtime libraries $Deps += "openssl" @@ -2019,7 +2045,7 @@ function Start-PSBootstrap { # ignore exitcode, because they may be already installed Start-NativeExecution ([ScriptBlock]::Create("$PackageManager install $Deps")) -IgnoreExitcode } elseif ($environment.IsLinux -and $environment.IsAlpine) { - $Deps += 'libunwind', 'libcurl', 'bash', 'cmake', 'clang', 'build-base', 'git', 'curl' + $Deps += 'libunwind', 'libcurl', 'bash', 'build-base', 'git', 'curl', 'wget' Start-NativeExecution { Invoke-Expression "apk add $Deps" @@ -2028,19 +2054,9 @@ function Start-PSBootstrap { # Install [fpm](https://github.com/jordansissel/fpm) and [ronn](https://github.com/rtomayko/ronn) if ($Package) { - try { - # We cannot guess if the user wants to run gem install as root on linux and windows, - # but macOs usually requires sudo - $gemsudo = '' - if($environment.IsMacOS -or $env:TF_BUILD) { - $gemsudo = $sudo - } - Start-NativeExecution ([ScriptBlock]::Create("$gemsudo gem install ffi -v 1.12.0 --no-document")) - Start-NativeExecution ([ScriptBlock]::Create("$gemsudo gem install fpm -v 1.11.0 --no-document")) - Start-NativeExecution ([ScriptBlock]::Create("$gemsudo gem install ronn -v 0.7.3 --no-document")) - } catch { - Write-Warning "Installation of fpm and ronn gems failed! Must resolve manually." - } + Install-GlobalGem -Sudo $sudo -GemName "ffi" -GemVersion "1.12.0" + Install-GlobalGem -Sudo $sudo -GemName "fpm" -GemVersion "1.11.0" + Install-GlobalGem -Sudo $sudo -GemName "ronn" -GemVersion "0.7.3" } } @@ -2255,7 +2271,7 @@ function Find-Dotnet() { $dotnetCLIInstalledVersion = Get-LatestInstalledSDK Pop-Location - Write-Verbose -Verbose "dotnetCLIInstalledVersion = $dotnetCLIInstalledVersion`nchosenDotNetVersion = $chosenDotNetVersion" + Write-Verbose -Message "Find-DotNet: dotnetCLIInstalledVersion = $dotnetCLIInstalledVersion; chosenDotNetVersion = $chosenDotNetVersion" if ($dotnetCLIInstalledVersion -ne $chosenDotNetVersion) { Write-Warning "The 'dotnet' in the current path can't find SDK version ${dotnetCLIRequiredVersion}, prepending $dotnetPath to PATH." @@ -3207,3 +3223,29 @@ function Set-CorrectLocale # Output the locale to log it locale } + +function Install-AzCopy { + $testPath = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + if (Test-Path $testPath) { + Write-Verbose "AzCopy already installed" -Verbose + return + } + + $destination = "$env:TEMP\azcopy81.msi" + Invoke-WebRequest "https://aka.ms/downloadazcopy" -OutFile $destination + Start-Process -FilePath $destination -ArgumentList "/quiet" -Wait +} + +function Find-AzCopy { + $searchPaths = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + + foreach ($filter in $searchPaths) { + $azCopy = Get-ChildItem -Path $filter -Recurse -ErrorAction SilentlyContinue | Select-Object -ExpandProperty FullName -First 1 + if ($azCopy) { + return $azCopy + } + } + + $azCopy = Get-Command -Name azCopy -ErrorAction Stop | Select-Object -First 1 + return $azCopy.Path +} diff --git a/docs/building/linux.md b/docs/building/linux.md index 7b42b04d342..9db44398e38 100644 --- a/docs/building/linux.md +++ b/docs/building/linux.md @@ -18,8 +18,7 @@ refer to the [Working with the PowerShell Repository](../git/README.md), ### Toolchain Setup -We use the [.NET Command-Line Interface][dotnet-cli] (`dotnet`) to build the managed components, -and [CMake][] to build the native components. +We use the [.NET Command-Line Interface][dotnet-cli] (`dotnet`) to build the managed components. Installing the toolchain is as easy as running `Start-PSBootstrap` in PowerShell. Of course, this requires a self-hosted copy of PowerShell on Linux. @@ -55,7 +54,6 @@ The `Start-PSBootstrap` function does the following: If you want to use `dotnet` outside of `Start-PSBuild`, add `~/.dotnet` to your `PATH` environment variable. [dotnet-cli]: https://docs.microsoft.com/dotnet/core/tools/ -[CMake]: https://cmake.org/cmake/help/v2.8.12/cmake.html ## Build using our module diff --git a/docs/building/macos.md b/docs/building/macos.md index e398451e8da..fc11c9d442b 100644 --- a/docs/building/macos.md +++ b/docs/building/macos.md @@ -14,7 +14,7 @@ From `pwsh.exe`, run `Import-Module ./build.psm1` and use `Start-PSBootstrap` to The `Start-PSBootstrap` function does the following: -- Uses `brew` or `port` to install CMake, OpenSSL, and GNU WGet +- Uses `brew` or `port` to install OpenSSL, and GNU WGet - Uninstalls any prior versions of .NET CLI - Downloads and installs .NET Core SDK to `~/.dotnet` diff --git a/experimental-feature-linux.json b/experimental-feature-linux.json new file mode 100644 index 00000000000..12df4c5db5c --- /dev/null +++ b/experimental-feature-linux.json @@ -0,0 +1,14 @@ +[ + "PSAMSIMethodInvocationLogging", + "PSAnsiRenderingFileInfo", + "PSCleanBlock", + "PSCommandNotFoundSuggestion", + "PSExec", + "PSLoadAssemblyFromNativeCode", + "PSNativeCommandArgumentPassing", + "PSNativeCommandErrorActionPreference", + "PSNativePSPathResolution", + "PSRemotingSSHTransportErrorHandling", + "PSStrictModeAssignment", + "PSSubsystemPluginModel" +] diff --git a/experimental-feature-windows.json b/experimental-feature-windows.json new file mode 100644 index 00000000000..12df4c5db5c --- /dev/null +++ b/experimental-feature-windows.json @@ -0,0 +1,14 @@ +[ + "PSAMSIMethodInvocationLogging", + "PSAnsiRenderingFileInfo", + "PSCleanBlock", + "PSCommandNotFoundSuggestion", + "PSExec", + "PSLoadAssemblyFromNativeCode", + "PSNativeCommandArgumentPassing", + "PSNativeCommandErrorActionPreference", + "PSNativePSPathResolution", + "PSRemotingSSHTransportErrorHandling", + "PSStrictModeAssignment", + "PSSubsystemPluginModel" +] diff --git a/global.json b/global.json index 047020849f2..c009e93aec2 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "6.0.100" + "version": "6.0.201" } } diff --git a/prquantifier.yaml b/prquantifier.yaml index 240f660f88d..ea891ba4988 100644 --- a/prquantifier.yaml +++ b/prquantifier.yaml @@ -7,5 +7,5 @@ Excluded: - '*.md' - '*.sln' # autogenerated files -- cgmanifest.json +- tools/cgmanifest.json - assets/wix/files.wxs diff --git a/cgmanifest.json b/tools/cgmanifest.json similarity index 99% rename from cgmanifest.json rename to tools/cgmanifest.json index 5544b21725e..01423263731 100644 --- a/cgmanifest.json +++ b/tools/cgmanifest.json @@ -685,7 +685,7 @@ "Type": "nuget", "Nuget": { "Name": "StyleCop.Analyzers.Unstable", - "Version": "1.2.0.354" + "Version": "1.2.0.406" } }, "DevelopmentDependency": true diff --git a/tools/findMissingNotices.ps1 b/tools/findMissingNotices.ps1 index f66f88f1295..315a83284d7 100644 --- a/tools/findMissingNotices.ps1 +++ b/tools/findMissingNotices.ps1 @@ -13,8 +13,14 @@ Import-Module dotnet.project.assets Import-Module "$PSScriptRoot\..\.github\workflows\GHWorkflowHelper" -Force . "$PSScriptRoot\..\tools\buildCommon\startNativeExecution.ps1" +$packageSourceName = 'findMissingNoticesNugetOrg' +if (!(Get-PackageSource -Name $packageSourceName -ErrorAction SilentlyContinue)) { + $null = Register-PackageSource -Name $packageSourceName -Location https://www.nuget.org/api/v2 -ProviderName NuGet +} + $existingRegistrationTable = @{} -$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable +$cgManifestPath = (Resolve-Path -Path $PSScriptRoot\..\tools\cgmanifest.json).ProviderPath +$existingRegistrationsJson = Get-Content $cgManifestPath | ConvertFrom-Json -AsHashtable $existingRegistrationsJson.Registrations | ForEach-Object { $registration = [Registration]$_ if ($registration.Component) { @@ -108,7 +114,9 @@ function New-NugetComponent { $nugetPublicVersionCache = [System.Collections.Generic.Dictionary[string, string]]::new() function Get-NuGetPublicVersion { param( + [parameter(Mandatory)] [string]$Name, + [parameter(Mandatory)] [string]$Version ) @@ -123,7 +131,7 @@ function Get-NuGetPublicVersion { } $publicVersion = $null - $publicVersion = Find-Package -Name $Name -AllowPrereleaseVersions -source nuget.org -AllVersions -ErrorAction SilentlyContinue | ForEach-Object { + $publicVersion = Find-Package -Name $Name -AllowPrereleaseVersions -source $packageSourceName -AllVersions -ErrorAction SilentlyContinue | ForEach-Object { try { $packageVersion = [System.Management.Automation.SemanticVersion]$_.Version } catch { @@ -269,7 +277,6 @@ $newRegistrations = $registrations.Keys | Sort-Object | ForEach-Object { $regist $count = $newRegistrations.Count $newJson = @{Registrations = $newRegistrations } | ConvertTo-Json -depth 99 if ($Fix -and $registrationChanged) { - $cgManifestPath = (Resolve-Path -Path $PSScriptRoot\..\cgmanifest.json).ProviderPath $newJson | Set-Content $cgManifestPath Set-GWVariable -Name CGMANIFEST_PATH -Value $cgManifestPath } diff --git a/tools/packaging/packaging.psd1 b/tools/packaging/packaging.psd1 index 8de303ef45c..e9dec523acd 100644 --- a/tools/packaging/packaging.psd1 +++ b/tools/packaging/packaging.psd1 @@ -15,6 +15,8 @@ 'New-DotnetSdkContainerFxdPackage' 'New-ExePackage' 'New-GlobalToolNupkg' + 'New-ILNugetPackageSource' + 'New-ILNugetPackageFromSource' 'New-ILNugetPackage' 'New-MSIPatch' 'New-PSBuildZip' diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 2eab521a530..88c4e4b304b 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -982,7 +982,15 @@ function New-UnixPackage { try { if ($PSCmdlet.ShouldProcess("Create $type package")) { Write-Log "Creating package with fpm..." - $Output = Start-NativeExecution { fpm $Arguments } + try { + $Output = Start-NativeExecution { fpm $Arguments } + } + catch { + Write-Verbose -Message "!!!Handling error in FPM!!!" -Verbose -ErrorAction SilentlyContinue + Write-Verbose -Message "$Output" -Verbose -ErrorAction SilentlyContinue + Get-Error -InputObject $_ + throw + } } } finally { if ($Environment.IsMacOS) { @@ -1699,6 +1707,9 @@ function New-PdbZipPackage function CreateNugetPlatformFolder { param( + [Parameter(Mandatory = $true)] + [string] $FileName, + [Parameter(Mandatory = $true)] [string] $Platform, @@ -1710,46 +1721,192 @@ function CreateNugetPlatformFolder ) $destPath = New-Item -ItemType Directory -Path (Join-Path $PackageRuntimesFolder "$Platform/lib/$script:netCoreRuntime") - $fullPath = Join-Path $PlatformBinPath $file + $fullPath = Join-Path $PlatformBinPath $FileName if (-not(Test-Path $fullPath)) { throw "File not found: $fullPath" } Copy-Item -Path $fullPath -Destination $destPath - Write-Log "Copied $file to $Platform" + Write-Log "Copied $FileName to $Platform at path: $destPath" +} + +<# +.SYNOPSIS +Creates a CGManifest file containing package dependencies for specified file. + +.PARAMETER FilePath +File path name of CGManifest file to be created. + +.PARAMETER Dependencies +Array list of dependency tuples: +[tuple[ [tuple[string, string]], [tuple[string, string]] ] []] +["Id", "Microsoft.PowerShell.SecretStore"], ["Version", "1.1.1.0"] +#> +function New-CGManifest +{ + param ( + [parameter(Mandatory = $true)] + [string] $FilePath, + + [parameter(Mandatory = $false)] + [tuple[ [tuple[string, string]], [tuple[string, string]] ] []] $Dependencies + ) + + Write-Verbose -Verbose -Message "Creating CGManifest for SBOM: $Filepath" + + $Registrations = @() + + foreach ($dependency in $Dependencies) { + $component = @{ + Component = @{ + Type = "nuget"; + NuGet = @{ + Name = ($dependency.Item1.Item2); Version = ($dependency.Item2.Item2) + } + }; + DevelopmentDependency = "true" + } + + $Registrations += $component + } + + $manifest = @{ Registrations = $Registrations } + $jsonManifest = $manifest | ConvertTo-Json -Depth 10 + + $jsonManifest | Out-File -FilePath $FilePath +} + +function New-FileDependencies +{ + param ( + [parameter(Mandatory = $true)] + [string] $FileBaseName, + + [parameter(Mandatory = $true)] + [string] $PackageVersion + ) + + # Filed a tracking bug for automating generation of dependecy list: https://github.com/PowerShell/PowerShell/issues/6247 + $deps = [System.Collections.ArrayList]::new() + + switch ($FileBaseName) { + 'Microsoft.Management.Infrastructure.CimCmdlets' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + } + + 'Microsoft.PowerShell.Commands.Diagnostics' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + } + + 'Microsoft.PowerShell.Commands.Management' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Security'), [tuple]::Create('version', $PackageVersion))) > $null + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + } + + 'Microsoft.PowerShell.Commands.Utility' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + } + + 'Microsoft.PowerShell.ConsoleHost' { + $deps.Add([tuple]::Create( [tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + } + + 'Microsoft.PowerShell.CoreCLR.Eventing' { + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + } + + 'Microsoft.PowerShell.SDK' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Commands.Management'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Commands.Utility'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.ConsoleHost'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Security'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.WSMan.Management'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Commands.Diagnostics'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.Management.Infrastructure.CimCmdlets'), [tuple]::Create('version', $PackageVersion))) > $null + } + + 'Microsoft.PowerShell.Security' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + } + + 'Microsoft.WSMan.Management' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.WSMan.Runtime'), [tuple]::Create('version', $PackageVersion))) > $null + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + } + + 'Microsoft.WSMan.Runtime' { + ## No dependencies + } + + 'System.Management.Automation' { + $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.CoreCLR.Eventing'), [tuple]::Create('version', $PackageVersion))) > $null + foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $FileBaseName)) + { + $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null + } + } + } + + Write-Output $deps } <# .SYNOPSIS -Creates NuGet packages containing linux, osx and Windows runtime assemblies. +Creates nuget package sources for a single provided binary file. .DESCRIPTION -Creates a NuGet package of IL assemblies for unix and windows. -The packages for Microsoft.PowerShell.Commands.Diagnostics, Microsoft.PowerShell.Commands.Management, -Microsoft.PowerShell.Commands.Utility, Microsoft.PowerShell.ConsoleHost, Microsoft.PowerShell.CoreCLR.Eventing, -Microsoft.PowerShell.SDK, Microsoft.PowerShell.Security, Microsoft.WSMan.Management, Microsoft.WSMan.Runtime, -System.Management.Automation are created. +Creates IL assemblies, for a single binary file, to be packaged in a NuGet file. +Includes runtime assemblies for linux and Windows runtime assemblies. + +.PARAMETER FileName +File name of binary to create nuget sources for. .PARAMETER PackagePath -Path where the package will be created. +Path where the package source files will be created. .PARAMETER PackageVersion Version of the created package. .PARAMETER WinFxdBinPath -Path to folder containing Windows framework dependent assemblies. +Path to source folder containing Windows framework dependent assemblies. .PARAMETER LinuxFxdBinPath -Path to folder containing Linux framework dependent assemblies. +Path to source folder containing Linux framework dependent assemblies. .PARAMETER GenAPIToolPath Path to the GenAPI.exe tool. #> -function New-ILNugetPackage +function New-ILNugetPackageSource { [CmdletBinding(SupportsShouldProcess = $true)] - param( + param ( + [Parameter(Mandatory = $true)] + [string] $FileName, [Parameter(Mandatory = $true)] [string] $PackagePath, @@ -1764,12 +1921,20 @@ function New-ILNugetPackage [string] $LinuxFxdBinPath, [Parameter(Mandatory = $true)] - [string] $GenAPIToolPath + [string] $GenAPIToolPath, + + [Parameter(Mandatory = $true)] + [string] $CGManifestPath ) - if (-not $Environment.IsWindows) + if (! $Environment.IsWindows) + { + throw "New-ILNugetPackageSource can be only executed on Windows platform." + } + + if (! $PSCmdlet.ShouldProcess("Create nuget packages at: $PackagePath")) { - throw "New-ILNugetPackage can be only executed on Windows platform." + return } $fileList = @( @@ -1792,191 +1957,166 @@ function New-ILNugetPackage "Microsoft.WSMan.Management.dll", "Microsoft.WSMan.Runtime.dll") - if ($PSCmdlet.ShouldProcess("Create nuget packages at: $PackagePath")) - { - $refBinPath = New-TempFolder - $SnkFilePath = "$RepoRoot\src\signing\visualstudiopublic.snk" + $refBinPath = New-TempFolder + $SnkFilePath = "$RepoRoot\src\signing\visualstudiopublic.snk" - New-ReferenceAssembly -linux64BinPath $LinuxFxdBinPath -RefAssemblyDestinationPath $refBinPath -RefAssemblyVersion $PackageVersion -SnkFilePath $SnkFilePath -GenAPIToolPath $GenAPIToolPath + New-ReferenceAssembly -linux64BinPath $LinuxFxdBinPath -RefAssemblyDestinationPath $refBinPath -RefAssemblyVersion $PackageVersion -SnkFilePath $SnkFilePath -GenAPIToolPath $GenAPIToolPath - foreach ($file in $fileList) - { - $tmpPackageRoot = New-TempFolder - # Remove '.dll' at the end - $fileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($file) - $filePackageFolder = New-Item (Join-Path $tmpPackageRoot $fileBaseName) -ItemType Directory -Force - $packageRuntimesFolder = New-Item (Join-Path $filePackageFolder.FullName 'runtimes') -ItemType Directory + if (! (Test-Path $PackagePath)) { + $null = New-Item -Path $PackagePath -ItemType Directory + } - #region ref - $refFolder = New-Item (Join-Path $filePackageFolder.FullName "ref/$script:netCoreRuntime") -ItemType Directory -Force - CopyReferenceAssemblies -assemblyName $fileBaseName -refBinPath $refBinPath -refNugetPath $refFolder -assemblyFileList $fileList - #endregion ref + # Remove '.dll' at the end + $fileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($FileName) + $filePackageFolder = New-Item (Join-Path $PackagePath $fileBaseName) -ItemType Directory -Force + $packageRuntimesFolder = New-Item (Join-Path $filePackageFolder.FullName 'runtimes') -ItemType Directory - $packageRuntimesFolderPath = $packageRuntimesFolder.FullName + Write-Verbose -Verbose "New-ILNugetPackageSource: Creating package source folder for file: $FileName at: $filePackageFolder" - CreateNugetPlatformFolder -Platform 'win' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $WinFxdBinPath + #region ref + $refFolder = New-Item (Join-Path $filePackageFolder.FullName "ref/$script:netCoreRuntime") -ItemType Directory -Force + CopyReferenceAssemblies -assemblyName $fileBaseName -refBinPath $refBinPath -refNugetPath $refFolder -assemblyFileList $fileList + #endregion ref - if ($linuxExceptionList -notcontains $file ) - { - CreateNugetPlatformFolder -Platform 'unix' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $LinuxFxdBinPath - } + $packageRuntimesFolderPath = $packageRuntimesFolder.FullName - if ($file -eq "Microsoft.PowerShell.SDK.dll") - { - # Copy the '$PSHOME\ref' folder to the NuGet package, so 'dotnet publish' can deploy the 'ref' folder to the publish folder. - # This is to make 'Add-Type' work in application that hosts PowerShell. - - $contentFolder = New-Item (Join-Path $filePackageFolder "contentFiles\any\any") -ItemType Directory -Force - $dotnetRefAsmFolder = Join-Path -Path $WinFxdBinPath -ChildPath "ref" - Copy-Item -Path $dotnetRefAsmFolder -Destination $contentFolder -Recurse -Force - Write-Log "Copied the reference assembly folder to contentFiles for the SDK package" - - # Copy the built-in module folders to the NuGet package, so 'dotnet publish' can deploy those modules to the $pshome module path. - # This is for enabling applications that hosts PowerShell to ship the built-in modules. - - $winBuiltInModules = @( - "CimCmdlets", - "Microsoft.PowerShell.Diagnostics", - "Microsoft.PowerShell.Host", - "Microsoft.PowerShell.Management", - "Microsoft.PowerShell.Security", - "Microsoft.PowerShell.Utility", - "Microsoft.WSMan.Management", - "PSDiagnostics" - ) - - $unixBuiltInModules = @( - "Microsoft.PowerShell.Host", - "Microsoft.PowerShell.Management", - "Microsoft.PowerShell.Security", - "Microsoft.PowerShell.Utility" - ) - - $winModuleFolder = New-Item (Join-Path $contentFolder "runtimes\win\lib\$script:netCoreRuntime\Modules") -ItemType Directory -Force - $unixModuleFolder = New-Item (Join-Path $contentFolder "runtimes\unix\lib\$script:netCoreRuntime\Modules") -ItemType Directory -Force - - foreach ($module in $winBuiltInModules) { - $source = Join-Path $WinFxdBinPath "Modules\$module" - Copy-Item -Path $source -Destination $winModuleFolder -Recurse -Force - } + CreateNugetPlatformFolder -FileName $FileName -Platform 'win' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $WinFxdBinPath - foreach ($module in $unixBuiltInModules) { - $source = Join-Path $LinuxFxdBinPath "Modules\$module" - Copy-Item -Path $source -Destination $unixModuleFolder -Recurse -Force - } + if ($linuxExceptionList -notcontains $FileName ) + { + CreateNugetPlatformFolder -FileName $FileName -Platform 'unix' -PackageRuntimesFolder $packageRuntimesFolderPath -PlatformBinPath $LinuxFxdBinPath + } - Write-Log "Copied the built-in modules to contentFiles for the SDK package" - } + if ($FileName -eq "Microsoft.PowerShell.SDK.dll") + { + # Copy the '$PSHOME\ref' folder to the NuGet package, so 'dotnet publish' can deploy the 'ref' folder to the publish folder. + # This is to make 'Add-Type' work in application that hosts PowerShell. + + $contentFolder = New-Item (Join-Path $filePackageFolder "contentFiles\any\any") -ItemType Directory -Force + $dotnetRefAsmFolder = Join-Path -Path $WinFxdBinPath -ChildPath "ref" + Copy-Item -Path $dotnetRefAsmFolder -Destination $contentFolder -Recurse -Force + Write-Log "Copied the reference assembly folder to contentFiles for the SDK package" + + # Copy the built-in module folders to the NuGet package, so 'dotnet publish' can deploy those modules to the $pshome module path. + # This is for enabling applications that hosts PowerShell to ship the built-in modules. + + $winBuiltInModules = @( + "CimCmdlets", + "Microsoft.PowerShell.Diagnostics", + "Microsoft.PowerShell.Host", + "Microsoft.PowerShell.Management", + "Microsoft.PowerShell.Security", + "Microsoft.PowerShell.Utility", + "Microsoft.WSMan.Management", + "PSDiagnostics" + ) - #region nuspec - # filed a tracking bug for automating generation of dependecy list: https://github.com/PowerShell/PowerShell/issues/6247 - $deps = [System.Collections.ArrayList]::new() + $unixBuiltInModules = @( + "Microsoft.PowerShell.Host", + "Microsoft.PowerShell.Management", + "Microsoft.PowerShell.Security", + "Microsoft.PowerShell.Utility" + ) - switch ($fileBaseName) { - 'Microsoft.Management.Infrastructure.CimCmdlets' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null - } + $winModuleFolder = New-Item (Join-Path $contentFolder "runtimes\win\lib\$script:netCoreRuntime\Modules") -ItemType Directory -Force + $unixModuleFolder = New-Item (Join-Path $contentFolder "runtimes\unix\lib\$script:netCoreRuntime\Modules") -ItemType Directory -Force - 'Microsoft.PowerShell.Commands.Diagnostics' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null - } + foreach ($module in $winBuiltInModules) { + $source = Join-Path $WinFxdBinPath "Modules\$module" + Copy-Item -Path $source -Destination $winModuleFolder -Recurse -Force + } - 'Microsoft.PowerShell.Commands.Management' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Security'), [tuple]::Create('version', $PackageVersion))) > $null - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - } + foreach ($module in $unixBuiltInModules) { + $source = Join-Path $LinuxFxdBinPath "Modules\$module" + Copy-Item -Path $source -Destination $unixModuleFolder -Recurse -Force + } - 'Microsoft.PowerShell.Commands.Utility' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null + Write-Log "Copied the built-in modules to contentFiles for the SDK package" + } - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - } + # Create a CGManifest file that lists all dependencies for this package, which is used when creating the SBOM. + if (! (Test-Path -Path $CGManifestPath)) { + $null = New-Item -Path $CGManifestPath -ItemType Directory + } + $deps = New-FileDependencies -FileBaseName $fileBaseName -PackageVersion $PackageVersion + New-CGManifest -FilePath (Join-Path -Path $CGManifestPath -ChildPath "CGManifest.json") -Dependencies $deps - 'Microsoft.PowerShell.ConsoleHost' { - $deps.Add([tuple]::Create( [tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - } + if (Test-Path $refBinPath) { + Remove-Item $refBinPath -Recurse -Force -ErrorAction SilentlyContinue + } +} - 'Microsoft.PowerShell.CoreCLR.Eventing' { - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - } +<# +.SYNOPSIS +Creates a nuget package file from the provided source path. - 'Microsoft.PowerShell.SDK' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Commands.Management'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Commands.Utility'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.ConsoleHost'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Security'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.WSMan.Management'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.Commands.Diagnostics'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.Management.Infrastructure.CimCmdlets'), [tuple]::Create('version', $PackageVersion))) > $null - } +.PARAMETER FileName +File name of binary to create nuget package for. - 'Microsoft.PowerShell.Security' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null - } +.PARAMETER PackagePath +Path for the source files and the created NuGet package file. +#> +function New-ILNugetPackageFromSource +{ + [CmdletBinding(SupportsShouldProcess = $true)] + param ( + [Parameter(Mandatory = $true)] + [string] $FileName, - 'Microsoft.WSMan.Management' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'System.Management.Automation'), [tuple]::Create('version', $PackageVersion))) > $null - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.WSMan.Runtime'), [tuple]::Create('version', $PackageVersion))) > $null - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - } + [Parameter(Mandatory = $true)] + [string] $PackageVersion, - 'Microsoft.WSMan.Runtime' { - ## No dependencies - } + [Parameter(Mandatory = $true)] + [string] $PackagePath + ) - 'System.Management.Automation' { - $deps.Add([tuple]::Create([tuple]::Create('id', 'Microsoft.PowerShell.CoreCLR.Eventing'), [tuple]::Create('version', $PackageVersion))) > $null - foreach($packageInfo in (Get-ProjectPackageInformation -ProjectName $fileBaseName)) - { - $deps.Add([tuple]::Create([tuple]::Create('id', $packageInfo.Name), [tuple]::Create('version', $packageInfo.Version))) > $null - } - } - } + if (! $Environment.IsWindows) + { + throw "New-ILNugetPackageFromSource can be only executed on Windows platform." + } - New-NuSpec -PackageId $fileBaseName -PackageVersion $PackageVersion -Dependency $deps -FilePath (Join-Path $filePackageFolder.FullName "$fileBaseName.nuspec") + if (! $PSCmdlet.ShouldProcess("Create nuget package for file $FileName at: $PackagePath")) + { + return + } - # Copy icon file to package - Copy-Item -Path $iconPath -Destination "$($filePackageFolder.Fullname)/$iconFileName" -Verbose + $fileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($FileName) - New-NugetPackage -NuSpecPath $filePackageFolder.FullName -PackageDestinationPath $PackagePath - } + $deps = New-FileDependencies -FileBaseName $fileBaseName -PackageVersion $PackageVersion - if (Test-Path $refBinPath) - { - Remove-Item $refBinPath -Recurse -Force -ErrorAction SilentlyContinue - } + $srcFilePackagePath = Join-Path $PackagePath $fileBaseName - if (Test-Path $tmpPackageRoot) - { - Remove-Item $tmpPackageRoot -Recurse -Force -ErrorAction SilentlyContinue - } + Write-Verbose -Verbose "New-ILNugetPackageFromSource: Creating nuget package for file: $FileName from source path: $srcFilePackagePath" + + if (! (Test-Path $srcFilePackagePath)) { + $msg = "Expected nuget source path $srcFilePackagePath for file $fileBaseName does not exist." + Write-Verbose -Verbose -Message $msg + throw $msg } + + # Remove the CGManifest file used to create the SBOM. + $cgManifestPath = Join-Path -Path $PackagePath -ChildPath 'CGManifest' + $cgManifestFilePath = Join-Path -Path $cgManifestPath -ChildPath 'CGManifest.json' + if (Test-Path -Path $cgManifestFilePath) + { + Write-Verbose -Verbose "Removing CGManifest file: $cgManifestFilePath" + Remove-Item -Path $cgManifestFilePath -Force -ErrorAction Continue + } + + New-NuSpec -PackageId $fileBaseName -PackageVersion $PackageVersion -Dependency $deps -FilePath (Join-Path $srcFilePackagePath "$fileBaseName.nuspec") + + # Copy icon file to package + Copy-Item -Path $iconPath -Destination "$srcFilePackagePath/$iconFileName" -Verbose + + New-NugetPackage -NuSpecPath $srcFilePackagePath -PackageDestinationPath $PackagePath + + # Remove file nuget package source directory + Remove-Item $srcFilePackagePath -Recurse -Force -ErrorAction SilentlyContinue } <# Copy the generated reference assemblies to the 'ref/net6.0' folder properly. - This is a helper function used by 'New-ILNugetPackage' + This is a helper function used by 'New-ILNugetPackageSource'. #> function CopyReferenceAssemblies { @@ -2081,7 +2221,7 @@ function New-NuSpec { [Parameter(Mandatory = $false)] # An array of tuples of tuples to define the dependencies. # First tuple defines 'id' and value eg: ["id", "System.Data.SqlClient"] - # Second tuple defines 'version' and vale eg: ["version", "4.4.2"] + # Second tuple defines 'version' and value eg: ["version", "4.4.2"] # Both these tuples combined together define one dependency. # An array represents all the dependencies. [tuple[ [tuple[string, string]], [tuple[string, string]] ] []] $Dependency, @@ -2184,6 +2324,9 @@ function New-ReferenceAssembly "Microsoft.PowerShell.ConsoleHost" ) + # Ensure needed dotNet version is available. Find-DotNet does this, and is part of build.psm1 which should already be imported. + Find-DotNet -Verbose + foreach ($assemblyName in $assemblyNames) { Write-Log "Building reference assembly for '$assemblyName'" @@ -2426,7 +2569,7 @@ function GenerateBuildArguments Create a NuGet package from a nuspec. .DESCRIPTION -Creates a NuGet using the nuspec using at the specified folder. +Creates a NuGet using the nuspec at the specified folder. It is expected that the lib / ref / runtime folders are welformed. The genereated NuGet package is copied over to the $PackageDestinationPath @@ -4097,7 +4240,7 @@ function Invoke-AzDevOpsLinuxPackageCreation { } } catch { - Get-Error + Get-Error -InputObject $_ throw } } @@ -4135,8 +4278,6 @@ function Invoke-AzDevOpsLinuxPackageBuild { } 'alpine' { $buildParams.Add("Runtime", 'alpine-x64') - # We are cross compiling, so we can't generate experimental features - $buildParams.Add("SkipExperimentalFeatureGeneration", $true) } } @@ -4178,7 +4319,7 @@ function Invoke-AzDevOpsLinuxPackageBuild { } } catch { - Get-Error + Get-Error -InputObject $_ throw } } diff --git a/tools/releaseBuild/Images/microsoft_powershell_alpine3/Dockerfile b/tools/releaseBuild/Images/microsoft_powershell_alpine3/Dockerfile deleted file mode 100644 index fb1070fcd52..00000000000 --- a/tools/releaseBuild/Images/microsoft_powershell_alpine3/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -# Docker image file that describes an Centos7 image with PowerShell installed from Microsoft YUM Repo - -FROM mcr.microsoft.com/powershell:alpine-3.12 -LABEL maintainer="PowerShell Team " - -# Install dependencies and clean up -RUN apk update \ - && apk add libunwind libcurl cmake clang build-base git bash curl - -COPY PowerShellPackage.ps1 / - -ENV DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - -ENTRYPOINT [ "pwsh" ] diff --git a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 index 40fe755cd68..ae0bc4f2b10 100644 --- a/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 +++ b/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 @@ -77,7 +77,7 @@ try Sync-PSTags -AddRemoteIfMissing Write-Verbose "Bootstrapping powershell build..." -Verbose - Start-PSBootstrap -Force -Package + Start-PSBootstrap -Force -Package -ErrorAction Stop if ($PSCmdlet.ParameterSetName -eq 'packageSigned') { diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index 4616f734b25..59fb0498a57 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -44,6 +44,8 @@ variables: value: https://github.com/powershell/powershell - name: SBOMGenerator_Formats value: spdx:2.2 + - name: BUILDSECMON_OPT_IN + value: true stages: - stage: prep @@ -207,7 +209,7 @@ stages: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals PSMMS2019-Secure steps: - checkout: self clean: true diff --git a/tools/releaseBuild/azureDevOps/releasePipeline.yml b/tools/releaseBuild/azureDevOps/releasePipeline.yml index fe5cb1c2464..c403aff28a1 100644 --- a/tools/releaseBuild/azureDevOps/releasePipeline.yml +++ b/tools/releaseBuild/azureDevOps/releasePipeline.yml @@ -19,6 +19,8 @@ variables: value: none - name: skipComponentGovernanceDetection value: true + - name: BUILDSECMON_OPT_IN + value: true stages: - stage: MSIXBundle @@ -110,6 +112,7 @@ stages: - job: ValidatePkgNames displayName: Validate Package Names pool: + # testing vmImage: windows-latest variables: - group: 'Azure Blob variable group' @@ -118,6 +121,7 @@ stages: - job: ValidatePkgBOM displayName: Validate Package BOM pool: + # testing vmImage: windows-latest variables: - group: 'Azure Blob variable group' @@ -174,8 +178,12 @@ stages: jobs: - deployment: AzureBlobPublic displayName: Make Azure Blob Public + pool: - vmImage: windows-latest + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-Secure + variables: - group: 'Staging_ACR' environment: PSReleaseAzureBlobPublic @@ -187,8 +195,12 @@ stages: - job: GitHubDraft displayName: Create GitHub Draft release + pool: - vmImage: windows-latest + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-Secure + variables: - group: 'Azure Blob variable group' - group: 'AzDevOpsArtifacts' @@ -220,14 +232,22 @@ stages: dependsOn: GitHubManualTasks jobs: - job: PublishNuget + pool: - vmImage: windows-latest + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-Secure + steps: - template: templates/release-ReleaseToNuGet.yml - job: PublishPkgsMsftCom + pool: - vmImage: ubuntu-latest + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMSUbuntu20.04-Secure + variables: - group: 'AzDevOpsArtifacts' - group: 'packages.microsoft.com' @@ -320,8 +340,12 @@ stages: jobs: - deployment: UploadJson displayName: Upload BuildInfoJson + pool: - vmImage: windows-latest + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-Secure + variables: - group: 'Azure Blob variable group' environment: PSReleaseBuildInfoJson diff --git a/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml b/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml index 4376c4a1e58..dd9252a406f 100644 --- a/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml +++ b/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml @@ -16,21 +16,31 @@ steps: - powershell: | $path = "./build.psm1" + if($env:REPOROOT){ + Write-Verbose "reporoot already set to ${env:REPOROOT}" -Verbose + exit 0 + } + if(Test-Path -Path $path) { - $vstsCommandString = "vso[task.setvariable variable=repoRoot]." - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" + Write-Verbose "reporoot detect at: ." -Verbose + $repoRoot = '.' } else{ $path = "./PowerShell/build.psm1" if(Test-Path -Path $path) { - $vstsCommandString = "vso[task.setvariable variable=repoRoot]./PowerShell" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" + Write-Verbose "reporoot detect at: ./PowerShell" -Verbose + $repoRoot = './PowerShell' } } + if($repoRoot) { + $vstsCommandString = "vso[task.setvariable variable=repoRoot]$repoRoot" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + } else { + Write-Verbose -Verbose "repo not found" + } displayName: 'Set repo Root' - powershell: | diff --git a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml index 8019a5de3b9..49082849e9b 100644 --- a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml +++ b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml @@ -10,7 +10,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals PSMMS2019-Secure steps: - checkout: self clean: true diff --git a/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml b/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml index 0a52f887671..352458390f9 100644 --- a/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml +++ b/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml @@ -1,7 +1,10 @@ +parameters: + nativePathRoot: '' + steps: - powershell: | $dirSeparatorChar = [system.io.path]::DirectorySeparatorChar - $nativePath = "${dirSeparatorChar}PowerShell" + $nativePath = "${{parameters.nativePathRoot }}${dirSeparatorChar}PowerShell" Write-Host "##vso[task.setvariable variable=PowerShellRoot]$nativePath" if ((Test-Path "$nativePath")) { diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml index 8b3ec94d0d1..0c4a162bb11 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -18,7 +18,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals PSMMS2019-Secure # APIScan can take a long time timeoutInMinutes: 180 @@ -138,5 +138,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(Build.SourcesDirectory)\tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml index 4d1007957c8..e81bdcf3709 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml @@ -19,7 +19,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals PSMMS2019-Secure # APIScan can take a long time timeoutInMinutes: 180 @@ -86,5 +86,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(Build.SourcesDirectory)\tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml b/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml index 3002c18bdb4..0e63dd9007b 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml @@ -16,7 +16,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals PSMMS2019-Secure timeoutInMinutes: 15 @@ -43,8 +43,7 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' - ignoreDirectories: '$(Build.SourcesDirectory)\test,$(Build.SourcesDirectory)\docs' + sourceScanPath: '$(Build.SourcesDirectory)\tools' - task: msospo.ospo-extension.8d7f9abb-6896-461d-9e25-4f74ed65ddb2.notice@0 displayName: 'NOTICE File Generator' @@ -82,9 +81,3 @@ jobs: targetPath: $(System.ArtifactsDirectory) artifactName: notice displayName: Publish notice artifacts - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(Build.SourcesDirectory)' - snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/json.yml b/tools/releaseBuild/azureDevOps/templates/json.yml index 570a658756a..714978022a5 100644 --- a/tools/releaseBuild/azureDevOps/templates/json.yml +++ b/tools/releaseBuild/azureDevOps/templates/json.yml @@ -15,7 +15,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals PSMMS2019-Secure steps: #- task: @ @@ -51,5 +51,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(Build.SourcesDirectory)\tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index 353f6e2431c..3d9625201a0 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -10,7 +10,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMSUbuntu20.04 + - ImageOverride -equals PSMMSUbuntu20.04-Secure dependsOn: ${{ parameters.parentJob }} variables: - name: runCodesignValidationInjection @@ -76,6 +76,9 @@ jobs: BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuild' Build_Repository_Uri: $(Github_Build_Repository_Uri) displayName: ${{ parameters.buildName }} SBOM + PackageName: PowerShell Linux + PackageVersion: $(Version) + sourceScanPath: '$(PowerShellRoot)/tools' - ${{ if eq(variables.build,'deb') }} : - template: Sbom.yml@ComplianceRepo @@ -83,6 +86,9 @@ jobs: BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildMinSize' Build_Repository_Uri: $(Github_Build_Repository_Uri) displayName: MinSize SBOM + PackageName: PowerShell Linux Minimum Size + PackageVersion: $(Version) + sourceScanPath: '$(PowerShellRoot)/tools' - ${{ if eq(variables.build,'deb') }} : - template: Sbom.yml@ComplianceRepo @@ -90,6 +96,9 @@ jobs: BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildArm32' Build_Repository_Uri: $(Github_Build_Repository_Uri) displayName: Arm32 SBOM + PackageName: PowerShell Linux Arm32 + PackageVersion: $(Version) + sourceScanPath: '$(PowerShellRoot)/tools' - ${{ if eq(variables.build,'deb') }} : - template: Sbom.yml@ComplianceRepo @@ -97,6 +106,9 @@ jobs: BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildArm64' Build_Repository_Uri: $(Github_Build_Repository_Uri) displayName: Arm64 SBOM + PackageName: PowerShell Linux Arm64 + PackageVersion: $(Version) + sourceScanPath: '$(PowerShellRoot)/tools' - powershell: | Import-Module "$env:POWERSHELLROOT/build.psm1" @@ -139,7 +151,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals PSMMS2019-Secure variables: - name: buildName value: ${{ parameters.buildName }} @@ -148,6 +160,8 @@ jobs: value: false - name: NugetSecurityAnalysisWarningLevel value: none + - name: skipComponentGovernanceDetection + value: true steps: - checkout: self @@ -297,9 +311,3 @@ jobs: parameters: artifactPath: '$(Build.StagingDirectory)\signedPackages\release' condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(Build.SourcesDirectory)' - snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml index 1f43d39ab61..a5c5a0811ec 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml @@ -9,20 +9,29 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals PSMMS2019-Secure + variables: - group: ESRP - name: runCodesignValidationInjection value: false - name: NugetSecurityAnalysisWarningLevel value: none + - name: repoFolder + value: PowerShell + - name: repoRoot + value: $(Agent.BuildDirectory)\$(repoFolder) + - name: complianceRepoFolder + value: compliance steps: - checkout: self clean: true + path: $(repoFolder) - checkout: ComplianceRepo clean: true + path: $(complianceRepoFolder) - template: SetVersionVariables.yml parameters: @@ -107,5 +116,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(repoRoot)\tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml index 989cdbda039..ef16761f707 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml @@ -8,7 +8,7 @@ jobs: dependsOn: MacFileSigningJob_${{ parameters.buildArchitecture }} condition: succeeded() pool: - vmImage: internal-macos-10.14 + vmImage: macos-latest variables: # Turn off Homebrew analytics - name: HOMEBREW_NO_ANALYTICS @@ -23,11 +23,11 @@ jobs: - pwsh: | # create folder - sudo mkdir /PowerShell + sudo mkdir "$(Agent.TempDirectory)/PowerShell" # make the current user the owner - sudo chown $env:USER /PowerShell - displayName: 'Create /PowerShell' + sudo chown $env:USER "$(Agent.TempDirectory)/PowerShell" + displayName: 'Create $(Agent.TempDirectory)/PowerShell' - template: SetVersionVariables.yml parameters: @@ -36,6 +36,8 @@ jobs: - template: shouldSign.yml - template: cloneToOfficialPath.yml + parameters: + nativePathRoot: '$(Agent.TempDirectory)' - task: DownloadBuildArtifacts@0 displayName: Download macosBinResults @@ -83,6 +85,9 @@ jobs: parameters: BuildDropPath: '$(System.ArtifactsDirectory)/$(SymbolsFolder)' Build_Repository_Uri: $(Github_Build_Repository_Uri) + PackageName: PowerShell macOS ${{ parameters.buildArchitecture }} + PackageVersion: $(Version) + sourceScanPath: '$(PowerShellRoot)/tools' - pwsh: | Import-Module $(PowerShellRoot)/build.psm1 -Force @@ -108,11 +113,13 @@ jobs: displayName: 'Bootstrap VM' - pwsh: | - $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -location $(PowerShellRoot) -ArtifactName macosPkgResults -BuildZip $(BuildPackagePath) -ExtraPackage "tar" -Runtime 'osx-${{ parameters.buildArchitecture }}' + # Add -SkipReleaseChecks as a mitigation to unblock release. + # macos-10.15 does not allow creating a folder under root. Hence, moving the folder. + $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -location $(PowerShellRoot) -ArtifactName macosPkgResults -BuildZip $(BuildPackagePath) -ExtraPackage "tar" -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks displayName: 'Package' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(PowerShellRoot)/tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index cba0ed6f9c3..e41d0609d4c 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -9,19 +9,28 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals PSMMS2019-Secure variables: - group: ESRP - name: runCodesignValidationInjection value: false - name: NugetSecurityAnalysisWarningLevel value: none + - name: repoFolder + value: PowerShell + - name: repoRoot + value: $(Agent.BuildDirectory)\$(repoFolder) + - name: complianceRepoFolder + value: compliance + steps: - checkout: self clean: true + path: $(repoFolder) - checkout: ComplianceRepo clean: true + path: $(complianceRepoFolder) - template: shouldSign.yml @@ -119,5 +128,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(repoRoot)/tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml index 6bf7814d93b..9634c3bdb57 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac.yml @@ -6,7 +6,7 @@ jobs: displayName: Build macOS ${{ parameters.buildArchitecture }} condition: succeeded() pool: - vmImage: internal-macos-10.14 + vmImage: macos-latest variables: # Turn off Homebrew analytics - name: HOMEBREW_NO_ANALYTICS @@ -28,13 +28,15 @@ jobs: - pwsh: | # create folder - sudo mkdir /PowerShell + sudo mkdir "$(Agent.TempDirectory)/PowerShell" # make the current user the owner - sudo chown $env:USER /PowerShell - displayName: 'Create /PowerShell' + sudo chown $env:USER "$(Agent.TempDirectory)/PowerShell" + displayName: 'Create $(Agent.TempDirectory)/PowerShell' - template: cloneToOfficialPath.yml + parameters: + nativePathRoot: '$(Agent.TempDirectory)' - pwsh: | tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap @@ -46,12 +48,14 @@ jobs: - pwsh: | $env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)' - $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults -Runtime 'osx-${{ parameters.buildArchitecture }}' + # Add -SkipReleaseChecks as a mitigation to unblock release. + # macos-10.15 does not allow creating a folder under root. Hence, moving the folder. + $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks $env:AzDevOpsFeedPAT2 = $null displayName: 'Build' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(Build.SourcesDirectory)/tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml new file mode 100644 index 00000000000..fe8c1a872ff --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml @@ -0,0 +1,70 @@ +parameters: + - name: PackageVersion + - name: PackagePath + - name: WinFxdPath + - name: LinuxFxdPath + - name: GenAPIToolPath + - name: ListOfFiles + type: object + default: + - Microsoft.Management.Infrastructure.CimCmdlets.dll + - Microsoft.PowerShell.Commands.Diagnostics.dll + - Microsoft.PowerShell.Commands.Management.dll + - Microsoft.PowerShell.Commands.Utility.dll + - Microsoft.PowerShell.ConsoleHost.dll + - Microsoft.PowerShell.CoreCLR.Eventing.dll + - Microsoft.PowerShell.Security.dll + - Microsoft.PowerShell.SDK.dll + - Microsoft.WSMan.Management.dll + - Microsoft.WSMan.Runtime.dll + - System.Management.Automation.dll + +steps: +- ${{ each value in parameters.ListOfFiles }}: + - pwsh: | + $FileName = '${{ value }}' + $FileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($FileName) + $FilePackagePath = Join-Path -Path '${{ parameters.PackagePath }}' -ChildPath $FileBaseName + $CGManifestPath = Join-Path -Path '${{ parameters.PackagePath }}' -ChildPath 'CGManifest' + Write-Verbose -Verbose "FileName to package: $FileName" + Write-Verbose -Verbose "FilePackage path: $FilePackagePath" + Write-Verbose -Verbose "CGManifest path: $CGManifestPath" + # Set SBOM package name + $vstsCommandString = "vso[task.setvariable variable=SbomFilePackageName]${FileBaseName}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + # Set SBOM package path variable + $vstsCommandString = "vso[task.setvariable variable=SbomFilePackagePath]${FilePackagePath}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + # Set CGManifest path variable + $vstsCommandString = "vso[task.setvariable variable=CGManifestPath]${CGManifestPath}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + # Create Nuget package sources + Import-Module -Name $env:REPOROOT\build.psm1 + Import-Module -Name $env:REPOROOT\tools\packaging + Find-DotNet + New-ILNugetPackageSource -File $FileName -PackagePath '${{ parameters.PackagePath }}' -PackageVersion '${{ parameters.PackageVersion }}' -WinFxdBinPath '${{ parameters.WinFxdPath }}' -LinuxFxdBinPath '${{ parameters.LinuxFxdPath }}' -GenAPIToolPath '${{ parameters.GenAPIToolPath }}' -CGManifestPath $CGManifestPath + displayName: 'Create NuGet Package source for single file' + + - template: Sbom.yml@ComplianceRepo + parameters: + BuildDropPath: $(SbomFilePackagePath) + Build_Repository_Uri: 'https://github.com/powershell/powershell' + PackageName: $(SbomFilePackageName) + PackageVersion: ${{ parameters.PackageVersion }} + sourceScanPath: $(CGManifestPath) + displayName: SBOM for NuGetPkg + + - pwsh: | + $FileName = '${{ value }}' + $FileBaseName = [System.IO.Path]::GetFileNameWithoutExtension($FileName) + $FilePackagePath = Join-Path -Path '${{ parameters.PackagePath }}' -ChildPath $FileBaseName + Write-Verbose -Verbose "FileName to package: $FileName" + Write-Verbose -Verbose "FilePackage path: $FilePackagePath" + Import-Module -Name $env:REPOROOT\build.psm1 + Import-Module -Name $env:REPOROOT\tools\packaging + Find-DotNet + New-ILNugetPackageFromSource -FileName $FileName -PackageVersion '${{ parameters.PackageVersion }}' -PackagePath '${{ parameters.PackagePath }}' + displayName: 'Create NuGet Package for single file' diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index 467968b5a3a..6994562d128 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -10,7 +10,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals PSMMS2019-Secure timeoutInMinutes: 90 @@ -51,10 +51,11 @@ jobs: Write-Host "##$vstsCommandString" displayName: 'Find SDK version from global.json' - - task: DotNetCoreInstaller@0 - displayName: 'Use .NET Core SDK from global.json' - inputs: - version: '$(SDKVersion)' + - pwsh: | + Import-Module "$env:REPOROOT/build.psm1" -Force + # We just need .NET but we fixed this in an urgent situation. + Start-PSBootStrap -Verbose + displayName: Bootstrap - task: DownloadBuildArtifacts@0 displayName: 'Download PowerShell build artifacts - finalResults' @@ -118,12 +119,14 @@ jobs: - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' - - pwsh: | - Import-Module $env:REPOROOT\build.psm1 - Import-Module $env:REPOROOT\tools\packaging - Find-Dotnet - New-ILNugetPackage -PackagePath "$(PackagePath)" -PackageVersion "$(Version)" -WinFxdBinPath '$(winFxdPath)' -LinuxFxdBinPath '$(linuxFxdPath)' -GenAPIToolPath "$(GenAPIToolPath)" - displayName: 'Create Nuget Package Folders' + # Create nuget packages along with SBOM manifests. + - template: nuget-pkg-sbom.yml + parameters: + PackageVersion: $(Version) + PackagePath: $(PackagePath) + WinFxdPath: $(winFxdPath) + LinuxFxdPath: $(linuxFxdPath) + GenAPIToolPath: $(GenAPIToolPath) - pwsh: | Get-ChildItem $(linuxFxdPath) diff --git a/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml b/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml index 6e968cca572..ce6eff7cb2a 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml @@ -1,10 +1,19 @@ steps: +- checkout: self + clean: true + - download: none - template: release-SetReleaseTagAndContainerName.yml - pwsh: | - $azcopy = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + Install-AzCopy + displayName: Install AzCopy + +- pwsh: | + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + $azcopy = Find-AzCopy & $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:$(StorageAccountKey) displayName: Download Azure Artifacts diff --git a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml index 34ea848ca7e..e7dc0685a47 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml @@ -10,6 +10,7 @@ jobs: - job: ${{ parameters.jobName }} displayName: ${{ parameters.displayName }} pool: + # test vmImage: ${{ parameters.imageName }} steps: - checkout: self @@ -66,7 +67,7 @@ jobs: - pwsh: | $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force - Start-PSBootstrap -Force + Start-PSBootstrap $toolPath = New-Item -ItemType Directory "$(System.DefaultWorkingDirectory)/toolPath" | Select-Object -ExpandProperty FullName @@ -91,13 +92,24 @@ jobs: - pwsh: | Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force - Start-PSBootstrap -Force + Start-PSBootstrap $exeName = if ($IsWindows) { "pwsh.exe" } else { "pwsh" } $toolPath = "$(System.DefaultWorkingDirectory)/toolPath/${{ parameters.globalToolExeName }}" - $env:DOTNET_ROOT = if ($IsWindows) { "$env:LOCALAPPDATA\Microsoft\dotnet" } else { "$env:HOME/.dotnet" } + $source = (get-command -Type Application -Name dotnet | Select-Object -First 1 -ExpandProperty source) + $target = (Get-ChildItem $source).target + + # If we find a symbolic link for dotnet, then we need to split the filename off the target. + if ($target) { + Write-Verbose -Verbose "Splitting target: $target" + $target = Split-Path $target + } + + Write-Verbose -Verbose "target is set as $target" + + $env:DOTNET_ROOT = (resolve-path -Path (Join-Path (split-path $source) $target)).ProviderPath Write-Verbose -Verbose "DOTNET_ROOT: $env:DOTNET_ROOT" Get-ChildItem $env:DOTNET_ROOT diff --git a/tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml b/tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml index 93bbbcc06d1..c883bc0ff0b 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-MsixBundle.yml @@ -1,8 +1,12 @@ jobs: - job: CreateMSIXBundle displayName: Create .msixbundle file + pool: - vmImage: windows-latest + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-Secure + variables: - group: msixTools - group: 'Azure Blob variable group' @@ -61,10 +65,16 @@ jobs: Write-Host "##$vstsCommandString" displayName: Create MsixBundle + - pwsh: | + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + Install-AzCopy + displayName: Install AzCopy + - pwsh: | ## We use AzCopy v8.1 in our release pipeline, see the documentation at: ## https://docs.microsoft.com/en-us/previous-versions/azure/storage/storage-use-azcopy - $azcopy = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + $azcopy = Find-AzCopy & $azcopy /Source:$(BundleDir) /Dest:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion)-private /DestKey:$(StorageAccountKey) /Pattern:*.msixbundle /Y displayName: Upload MSIX Bundle package to Az Blob diff --git a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml index 8b7663f1c37..416072aac2e 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml @@ -7,6 +7,7 @@ jobs: - job: ${{ parameters.jobName }} displayName: ${{ parameters.displayName }} pool: + # testing vmImage: ${{ parameters.imageName }} variables: - group: AzDevOpsArtifacts @@ -95,7 +96,7 @@ jobs: - pwsh: | $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force - Start-PSBootstrap -Force + Start-PSBootstrap $localLocation = "$(Pipeline.Workspace)/releasePipeline/finalResults" $xmlElement = @" diff --git a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml index 55bf592c339..799e220626d 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml @@ -1,8 +1,12 @@ jobs: - job: UpdateDepsFiles displayName: Update deps files + pool: - vmImage: windows-latest + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-Secure + variables: - group: 'Azure Blob variable group' steps: diff --git a/tools/releaseBuild/azureDevOps/templates/release-ValidateFxdPackage.yml b/tools/releaseBuild/azureDevOps/templates/release-ValidateFxdPackage.yml index 687b9d526d4..a4d473cf3b8 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-ValidateFxdPackage.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-ValidateFxdPackage.yml @@ -8,6 +8,7 @@ jobs: - job: ${{ parameters.jobName }} displayName: ${{ parameters.displayName }} pool: + # testing vmImage: ${{ parameters.imageName }} steps: - checkout: self diff --git a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml index 65eed1ef9e0..4778ebcaa1c 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml @@ -15,7 +15,13 @@ steps: displayName: Set Release Name - pwsh: | - $azcopy = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + Install-AzCopy + displayName: Install AzCopy + +- pwsh: | + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + $azcopy = Find-AzCopy & $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:$(StorageAccountKey) displayName: Download Azure Artifacts diff --git a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml index 44040f9ebb0..fa80c1d0c84 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml @@ -12,7 +12,13 @@ steps: displayName: Set Release Name - pwsh: | - $azcopy = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + Install-AzCopy + displayName: Install AzCopy + +- pwsh: | + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + $azcopy = Find-AzCopy & $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:$(StorageAccountKey) displayName: Download Azure Artifacts diff --git a/tools/releaseBuild/azureDevOps/templates/testartifacts.yml b/tools/releaseBuild/azureDevOps/templates/testartifacts.yml index 1bd715f1a46..38d9ee0d158 100644 --- a/tools/releaseBuild/azureDevOps/templates/testartifacts.yml +++ b/tools/releaseBuild/azureDevOps/templates/testartifacts.yml @@ -8,6 +8,7 @@ jobs: displayName: Build test artifacts condition: succeeded() pool: + # testing vmImage: ubuntu-latest steps: - checkout: self diff --git a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml index e6f07d55ce5..ffa7aa2a0e3 100644 --- a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml +++ b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml @@ -8,7 +8,10 @@ jobs: displayName: Build and Publish VPack - ${{ parameters.architecture }} condition: succeeded() - pool: PowerShell1ES + pool: + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-Secure steps: - checkout: self clean: true @@ -18,7 +21,13 @@ jobs: ReleaseTagVar: $(ReleaseTagVar) - pwsh: | - $azcopy = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe" + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + Install-AzCopy + displayName: Install AzCopy + + - pwsh: | + Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1' + $azcopy = Find-AzCopy Write-Host "running: $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:****** /Pattern:PowerShell-$(Version)-win-${{ parameters.architecture }}.zip /Z:$(AGENT.TEMPDIRECTORY)" & $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:$(StorageAccountKey) /Pattern:PowerShell-$(Version)-win-${{ parameters.architecture }}.zip /Z:$(AGENT.TEMPDIRECTORY) diff --git a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml index b21ab208f1b..53947655d90 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml @@ -12,7 +12,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals PSMMS2019-Secure steps: - checkout: self diff --git a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml index de08cb25824..16554ae7d41 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml @@ -16,7 +16,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals PSMMS2019-Secure variables: - name: runCodesignValidationInjection value: false @@ -76,5 +76,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(PowerShellRoot)' + sourceScanPath: '$(PowerShellRoot)\tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index dd547da6a60..66bf7af7b3d 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -10,18 +10,26 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals PSMMS2019-Secure variables: - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE value: 1 - group: ESRP + - name: repoFolder + value: PowerShell + - name: repoRoot + value: $(Agent.BuildDirectory)\$(repoFolder) + - name: complianceRepoFolder + value: compliance steps: - checkout: self clean: true + path: $(repoFolder) - checkout: ComplianceRepo clean: true + path: $(complianceRepoFolder) - template: SetVersionVariables.yml parameters: @@ -124,5 +132,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(repoRoot)\tools' snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index f2b0e091f0c..956eb19d855 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -16,7 +16,7 @@ jobs: pool: name: PowerShell1ES demands: - - ImageOverride -equals MMS2019 + - ImageOverride -equals PSMMS2019-Secure variables: - name: BuildConfiguration value: ${{ parameters.BuildConfiguration }} @@ -42,7 +42,7 @@ jobs: - template: shouldSign.yml - - powershell: | + - pwsh: | $pkgFilter = if ( '$(Architecture)' -eq 'arm' ) { "arm32" } else { '$(Architecture)' } if ($env:BuildConfiguration -eq 'minSize') { $pkgFilter += '-gc' } @@ -60,25 +60,6 @@ jobs: - template: cloneToOfficialPath.yml - pwsh: | - # cleanup previous install - if((Test-Path "${env:ProgramFiles(x86)}\WiX Toolset xcopy")) { - Remove-Item "${env:ProgramFiles(x86)}\WiX Toolset xcopy" -Recurse -Force - } - - $toolsDir = New-Item -ItemType Directory -Path '$(Build.ArtifactStagingDirectory)\tools' - $wixUri = 'https://github.com/wixtoolset/wix3/releases/download/wix3112rtm/wix311-binaries.zip' - Invoke-RestMethod -Uri $wixUri -OutFile '$(Build.ArtifactStagingDirectory)\tools\wix.zip' -MaximumRetryCount 5 -RetryIntervalSec 10 - - Import-Module '$(PowerShellRoot)/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/wix.psm1' - Install-WixZip -zipPath '$(Build.ArtifactStagingDirectory)\tools\wix.zip' - - $msixUrl = '$(makeappUrl)' - Invoke-RestMethod -Uri $msixUrl -OutFile '\makeappx.zip' - Expand-Archive '\makeappx.zip' -destination '\' -Force - - displayName: Install packaging tools - - - powershell: | $zipPathString = '$(System.ArtifactsDirectory)\Symbols\results\*$(PkgFilter).zip' Write-Verbose -Verbose "Zip Path: $zipPathString" $zipPath = Get-Item $zipPathString @@ -208,12 +189,12 @@ jobs: shouldSign: $(SHOULD_SIGN) displayName: Sign ThirdParty binaries - - powershell: | + - pwsh: | Get-ChildItem '$(System.ArtifactsDirectory)\thirdPartySigned\*' displayName: Capture ThirdParty Signed files condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - powershell: | + - pwsh: | Import-Module $(PowerShellRoot)/build.psm1 -Force Import-Module $(PowerShellRoot)/tools/packaging -Force $signedFilesPath = '$(System.ArtifactsDirectory)\thirdPartySigned' @@ -227,8 +208,11 @@ jobs: parameters: BuildDropPath: '$(System.ArtifactsDirectory)\$(SymbolsFolder)' Build_Repository_Uri: $(Github_Build_Repository_Uri) + PackageName: PowerShell Windows ${{ parameters.Architecture }} ${{ parameters.BuildConfiguration }} + PackageVersion: $(Version) + sourceScanPath: '$(PowerShellRoot)\tools' - - powershell: | + - pwsh: | Import-Module $(PowerShellRoot)/build.psm1 -Force Import-Module $(PowerShellRoot)/tools/packaging -Force @@ -269,7 +253,7 @@ jobs: $(PowerShellRoot)/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1 -BuildZip $signedPkg -location '$(PowerShellRoot)' -destination '$(System.ArtifactsDirectory)\pkgSigned' -Runtime $runtime -ReleaseTag '$(ReleaseTagVar)' @params displayName: 'Build Windows Universal - $(Architecture) Package' - - powershell: | + - pwsh: | Get-ChildItem '$(System.ArtifactsDirectory)\pkgSigned' | ForEach-Object { $packagePath = $_.FullName Write-Host "Uploading $packagePath" @@ -357,20 +341,15 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(PowerShellRoot)\tools' snapshotForceEnabled: true - - powershell: | + - pwsh: | if ((Test-Path "\PowerShell")) { Remove-Item -Path "\PowerShell" -Force -Recurse -Verbose } else { Write-Verbose -Verbose -Message "No cleanup required." } - - if((Test-Path "${env:ProgramFiles(x86)}\WiX Toolset xcopy")) { - Write-Verbose -Verbose "Cleaning up Wix tools" - Remove-Item "${env:ProgramFiles(x86)}\WiX Toolset xcopy" -Recurse -Force - } displayName: Clean up local Clone condition: always() diff --git a/tools/releaseBuild/azureDevOps/vpackRelease.yml b/tools/releaseBuild/azureDevOps/vpackRelease.yml index 2d502f23925..875e2d25c3c 100644 --- a/tools/releaseBuild/azureDevOps/vpackRelease.yml +++ b/tools/releaseBuild/azureDevOps/vpackRelease.yml @@ -24,8 +24,11 @@ stages: - job: rename displayName: Name the build condition: succeeded() - pool: - vmImage: 'windows-latest' + + pool: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-Secure + steps: - checkout: self clean: true