diff --git a/.pipelines/PowerShell-vPack-Official.yml b/.pipelines/PowerShell-vPack-Official.yml index d0e5535a552..3b151127cb1 100644 --- a/.pipelines/PowerShell-vPack-Official.yml +++ b/.pipelines/PowerShell-vPack-Official.yml @@ -1,6 +1,9 @@ trigger: none parameters: # parameters are shown up in ADO UI in a build queue time +- name: OfficialBuild + type: boolean + default: true - name: 'createVPack' displayName: 'Create and Submit VPack' type: boolean @@ -21,24 +24,8 @@ parameters: # parameters are shown up in ADO UI in a build queue time displayName: 'Enable debug output' type: boolean default: false -- name: 'architecture' - type: string - displayName: 'Select the vpack architecture:' - values: - - x64 - - x86 - - arm64 - default: x64 -- name: 'VPackPublishOverride' - type: string - displayName: 'VPack Publish Override Version (can leave blank):' - default: ' ' -- name: 'ReleaseTagVar' - type: string - displayName: 'Release Tag Var:' - default: 'fromBranch' -name: vPack_${{ parameters.architecture }}_$(date:yyMM).$(date:dd)$(rev:rrr) +name: vPack_$(Build.SourceBranchName)_Prod.${{ parameters.OfficialBuild }}_Create.${{ parameters.createVPack }}_Name.${{ parameters.vPackName}}_$(date:yyyyMMdd).$(rev:rr) variables: - name: CDP_DEFINITION_BUILD_COUNT @@ -77,17 +64,8 @@ resources: name: OneBranch.Pipelines/GovernedTemplates ref: refs/heads/main - pipelines: - - pipeline: PSPackagesOfficial - source: 'PowerShell-Packages-Official' - trigger: - branches: - include: - - master - - releases/* - extends: - template: v2/Microsoft.Official.yml@templates + template: ${{ variables.templateFile }} parameters: platform: name: 'windows_undocked' # windows undocked @@ -117,35 +95,116 @@ extends: enabled: false tsaOptionsFile: .config/tsaoptions.json stages: - - stage: main + - stage: BuildStage jobs: - - job: main + - job: BuildJob pool: type: windows + strategy: + matrix: + x86: + architecture: x86 + + x64: + architecture: x64 + + arm64: + architecture: arm64 + variables: + ArtifactPlatform: 'windows' + ob_artifactBaseName: drop_build_$(architecture) ob_outputDirectory: '$(BUILD.SOURCESDIRECTORY)\out' ob_createvpack_enabled: ${{ parameters.createVPack }} - ob_createvpack_packagename: 'PowerShell.${{ parameters.architecture }}' - ob_createvpack_description: PowerShell ${{ parameters.architecture }} $(version) ob_createvpack_owneralias: tplunk - ob_createvpack_versionAs: string - ob_createvpack_version: '$(version)' + ob_createvpack_versionAs: parts ob_createvpack_propsFile: true ob_createvpack_verbose: true + ob_createvpack_packagename: '${{ parameters.vPackName }}.$(architecture)' + ob_createvpack_description: PowerShell $(architecture) $(version) + # I think the variables reload after we transition back to the host so this works. 🤷‍♂️ + ob_createvpack_majorVer: $(pwshMajorVersion) + ob_createvpack_minorVer: $(pwshMinorVersion) + ob_createvpack_patchVer: $(pwshPatchVersion) + ${{ if ne(variables['pwshPrereleaseVersion'], '') }}: + ob_createvpack_prereleaseVer: $(pwshPrereleaseVersion) + ${{ else }}: + ob_createvpack_prereleaseVer: $(Build.SourceVersion) steps: + - checkout: self + displayName: Checkout source code - during restore + clean: true + path: s + env: + ob_restore_phase: true + - template: .pipelines/templates/SetVersionVariables.yml@self parameters: ReleaseTagVar: $(ReleaseTagVar) CreateJson: yes UseJson: no + - pwsh: | + $version = '$(Version)' + Write-Verbose -Verbose "Version: $version" + if(!$version) { + throw "Version is not set." + } + + $mainVersionParts = $version -split '-' + + Write-Verbose -Verbose "mainVersionParts: $($mainVersionParts[0]) ; $($mainVersionParts[1])" + $versionParts = $mainVersionParts[0] -split '[.]'; + $major = $versionParts[0] + $minor = $versionParts[1] + $patch = $versionParts[2] + + $previewPart = $mainVersionParts[1] + Write-Verbose -Verbose "previewPart: $previewPart" + + Write-Host "major: $major; minor: $minor; patch: $patch;" + + $vstsCommandString = "vso[task.setvariable variable=pwshMajorVersion]$major" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + + $vstsCommandString = "vso[task.setvariable variable=pwshMinorVersion]$minor" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + + $vstsCommandString = "vso[task.setvariable variable=pwshPatchVersion]$patch" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + if($previewPart) { + $vstsCommandString = "vso[task.setvariable variable=pwshPrereleaseVersion]$previewPart" + } else { + Write-Verbose -Verbose "No prerelease part found in version string." + } + displayName: Set ob_createvpack_*Ver + env: + ob_restore_phase: true + + # Validate pwsh*Version variables + - pwsh: | + $variables = @("pwshMajorVersion", "pwshMinorVersion", "pwshPatchVersion") + foreach ($var in $variables) { + if (-not (get-item "Env:\$var" -ErrorAction SilentlyContinue).value) { + throw "Required variable '`$env:$var' is not set." + } + } + displayName: Validate pwsh*Version variables + env: + ob_restore_phase: true + - pwsh: | if($env:RELEASETAGVAR -match '-') { throw "Don't release a preview build without coordinating with Windows Engineering Build Tools Team" } displayName: Stop any preview release + env: + ob_restore_phase: true - task: UseDotNet@2 displayName: 'Use .NET Core sdk' @@ -154,88 +213,115 @@ extends: version: 3.1.x installationPath: $(Agent.ToolsDirectory)/dotnet + ### BUILD ### + + - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + parameters: + repoRoot: $(repoRoot) + + - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step. + env: + ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + inputs: + Enabled: true + AnalyzeInPipeline: false # Do not upload results + Language: csharp + + - task: UseDotNet@2 + displayName: 'Install .NET based on global.json' + inputs: + useGlobalJson: true + workingDirectory: $(repoRoot) + env: + ob_restore_phase: true + - pwsh: | - $packageArtifactName = 'drop_windows_package_package_win_${{ parameters.architecture }}' - $vstsCommandString = "vso[task.setvariable variable=PackageArtifactName]$packageArtifactName" - Write-Host "sending " + $vstsCommandString + # Need to set PowerShellRoot variable for obp-file-signing template + $vstsCommandString = "vso[task.setvariable variable=PowerShellRoot]$(repoRoot)" + Write-Host ("sending " + $vstsCommandString) Write-Host "##$vstsCommandString" - $packageArtifactPath = '$(Pipeline.Workspace)\PSPackagesOfficial' - $vstsCommandString = "vso[task.setvariable variable=PackageArtifactPath]$packageArtifactPath" - Write-Host "sending " + $vstsCommandString + $Architecture = '$(Architecture)' + $runtime = switch ($Architecture) + { + "x64" { "win7-x64" } + "x86" { "win7-x86" } + "arm64" { "win-arm64" } + } + + $params = @{} + if ($env:BuildConfiguration -eq 'minSize') { + $params['ForMinimalSize'] = $true + } + + $vstsCommandString = "vso[task.setvariable variable=Runtime]$runtime" + Write-Host ("sending " + $vstsCommandString) Write-Host "##$vstsCommandString" - displayName: 'Set package artifact variables' - - download: PSPackagesOfficial - artifact: $(PackageArtifactName) - displayName: Download package + Write-Verbose -Message "Building PowerShell with Runtime: $runtime for '$env:BuildConfiguration' configuration" + Import-Module -Name $(repoRoot)/build.psm1 -Force + $buildWithSymbolsPath = New-Item -ItemType Directory -Path "$(Pipeline.Workspace)/Symbols_$Architecture" -Force - - pwsh: 'Get-ChildItem $(PackageArtifactPath)\* -recurse | Select-Object -ExpandProperty Name' - displayName: 'Capture Artifact Listing' + Start-PSBootstrap -Scenario Package + $null = New-Item -ItemType Directory -Path $buildWithSymbolsPath -Force -Verbose - - pwsh: | - $message = @() - $packages = Get-ChildItem $(PackageArtifactPath)\* -recurse -include *.zip, *.msi - - if($packages.count -eq 0) {throw "No packages found in $(PackageArtifactPath)"} - - $packages | ForEach-Object { - if($_.Name -notmatch 'PowerShell-\d+\.\d+\.\d+\-([a-z]*.\d+\-)?win\-(fxdependent|x64|arm64|x86|fxdependentWinDesktop)\.(msi|zip){1}') - { - $messageInstance = "$($_.Name) is not a valid package name" - $message += $messageInstance - Write-Warning $messageInstance - } + $ReleaseTagParam = @{} + + if ($env:RELEASETAGVAR) { + $ReleaseTagParam['ReleaseTag'] = $env:RELEASETAGVAR } - if($message.count -gt 0){throw ($message | out-string)} - displayName: 'Validate Zip and MSI Package Names' + Start-PSBuild -Runtime $runtime -Configuration Release -Output $buildWithSymbolsPath -Clean -PSModuleRestore @params @ReleaseTagParam - - pwsh: | - Get-ChildItem $(PackageArtifactPath)\* -recurse -include *.zip | ForEach-Object { - if($_.Name -match 'PowerShell-\d+\.\d+\.\d+\-([a-z]*.\d+\-)?win\-(${{ parameters.architecture }})\.(zip){1}') - { - Expand-Archive -Path $_.FullName -DestinationPath $(ob_outputDirectory) - } - } - displayName: 'Extract Zip to ob_outputDirectory' + $refFolderPath = Join-Path $buildWithSymbolsPath 'ref' + Write-Verbose -Verbose "refFolderPath: $refFolderPath" + $outputPath = Join-Path '$(ob_outputDirectory)' 'psoptions' + $null = New-Item -ItemType Directory -Path $outputPath -Force + $psOptPath = "$outputPath/psoptions.json" + Save-PSOptions -PSOptionsPath $psOptPath + + Write-Verbose -Verbose "Completed building PowerShell for '$env:BuildConfiguration' configuration" + displayName: Build Windows Universal - $(Architecture) -$(BuildConfiguration) Symbols folder + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + + - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. + env: + ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + inputs: + sourceScanPath: '$(repoRoot)\src' + ob_restore_phase: true + + - template: /.pipelines/templates/obp-file-signing.yml@self + parameters: + binPath: '$(Pipeline.Workspace)/Symbols_$(Architecture)' + SigningProfile: $(windows_build_tools_cert_id) + OfficialBuild: false + vPackScenario: true + + ### END OF BUILD ### - pwsh: | - Write-Verbose "VPack Version: $(ob_createvpack_version)" -Verbose - Get-ChildItem -Path $(ob_outputDirectory)\* -Recurse - Get-Content $(ob_outputdirectory)\preview.json -ErrorAction SilentlyContinue | Write-Host + Get-ChildItem env:/ob_createvpack_*Ver + Get-ChildItem -Path "$(Pipeline.Workspace)\Symbols_$(Architecture)\*" -Recurse + Get-Content "$(Pipeline.Workspace)\PowerShell\preview.json" -ErrorAction SilentlyContinue | Write-Host displayName: Debug Output Directory and Version condition: succeededOrFailed() - - pwsh: | - Write-Host "Using VPackPublishOverride variable" - $vpackVersion = '${{ parameters.VPackPublishOverride }}' - $vstsCommandString = "vso[task.setvariable variable=ob_createvpack_version]$vpackVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - condition: ne('${{ parameters.VPackPublishOverride }}', ' ') - displayName: 'Set ob_createvpack_version with VPackPublishOverride' - - pwsh: | Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose displayName: Capture Environment condition: succeededOrFailed() - pwsh: | - Write-Verbose "VPack Version: $(ob_createvpack_version)" -Verbose - $vpackFiles = Get-ChildItem -Path $(ob_outputDirectory)\* -Recurse + $vpackFiles = Get-ChildItem -Path "$(Pipeline.Workspace)\Symbols_$(Architecture)\*" -Recurse if($vpackFiles.Count -eq 0) { - throw "No files found in $(ob_outputDirectory)" + throw "No files found in $(Pipeline.Workspace)\Symbols_$(Architecture)" } $vpackFiles displayName: Debug Output Directory and Version condition: succeededOrFailed() - - - task: onebranch.pipeline.signing@1 - displayName: 'Onebranch Signing' - inputs: - command: 'sign' - signing_environment: 'azure-ado' - cp_code: $(windows_build_tools_cert_id) - files_to_sign: '**/*.exe;**/System.Management.Automation.dll' - search_root: $(ob_outputDirectory) diff --git a/.pipelines/templates/obp-file-signing.yml b/.pipelines/templates/obp-file-signing.yml index 1fccecaa062..3793078503e 100644 --- a/.pipelines/templates/obp-file-signing.yml +++ b/.pipelines/templates/obp-file-signing.yml @@ -1,6 +1,9 @@ parameters: binPath: '$(ob_outputDirectory)' globalTool: 'false' + SigningProfile: 'external_distribution' + OfficialBuild: true + vPackScenario: false steps: - pwsh: | @@ -80,7 +83,7 @@ steps: displayName: Sign 1st party files inputs: command: 'sign' - signing_profile: external_distribution + signing_profile: ${{ parameters.SigningProfile }} files_to_sign: '**\*.psd1;**\*.psm1;**\*.ps1xml;**\*.ps1;**\*.dll;**\*.exe;**\pwsh' search_root: $(Pipeline.Workspace)/toBeSigned @@ -101,12 +104,15 @@ steps: Write-Verbose -Verbose "Signed Path: $(Pipeline.Workspace)/toBeSigned" + $officialBuild = [System.Convert]::ToBoolean('${{ parameters.OfficialBuild }}') ## copy all files to be signed to build folder - Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath '$(Pipeline.Workspace)/toBeSigned' + Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath '$(Pipeline.Workspace)/toBeSigned' -OfficialBuild $officialBuild $dlls = Get-ChildItem $BuildPath/*.dll, $BuildPath/*.exe -Recurse $signatures = $dlls | Get-AuthenticodeSignature - $missingSignatures = $signatures | Where-Object { $_.status -eq 'notsigned' -or $_.SignerCertificate.Issuer -notmatch '^CN=Microsoft.*'}| select-object -ExpandProperty Path + $officialIssuerPattern = '^CN=(Microsoft Code Signing PCA|Microsoft Root Certificate Authority|Microsoft Corporation).*' + $testCert = '^CN=(Microsoft|TestAzureEngBuildCodeSign).*' + $missingSignatures = $signatures | Where-Object { $_.status -eq 'notsigned' -or $_.SignerCertificate.Issuer -notmatch $testCert -or $_.SignerCertificate.Issuer -notmatch $officialIssuerPattern} | select-object -ExpandProperty Path Write-Verbose -verbose "to be signed:`r`n $($missingSignatures | Out-String)" @@ -143,11 +149,20 @@ steps: displayName: Capture ThirdParty Signed files - pwsh: | + $officialBuild = [System.Convert]::ToBoolean('${{ parameters.OfficialBuild }}') + $vPackScenario = [System.Convert]::ToBoolean('${{ parameters.vPackScenario }}') Import-Module '$(PowerShellRoot)/build.psm1' -Force Import-Module '$(PowerShellRoot)/tools/packaging' -Force $isGlobalTool = '${{ parameters.globalTool }}' -eq 'true' - if (-not $isGlobalTool) { + if ($vPackScenario) { + Write-Verbose -Verbose -Message "vPackScenario is true, copying to $(ob_outputDirectory)" + $pathForUpload = New-Item -ItemType Directory -Path '$(ob_outputDirectory)' -Force + Write-Verbose -Verbose -Message "pathForUpload: $pathForUpload" + Copy-Item -Path '${{ parameters.binPath }}\*' -Destination $pathForUpload -Recurse -Force -Verbose + Write-Verbose -Verbose -Message "Files copied to $pathForUpload" + } + elseif (-not $isGlobalTool) { $pathForUpload = New-Item -ItemType Directory -Path '$(ob_outputDirectory)/Signed-$(Runtime)' -Force Write-Verbose -Verbose -Message "pathForUpload: $pathForUpload" Copy-Item -Path '${{ parameters.binPath }}\*' -Destination $pathForUpload -Recurse -Force -Verbose @@ -159,7 +174,7 @@ steps: Write-Verbose "Copying third party signed files to the build folder" $thirdPartySignedFilesPath = (Get-Item '$(Pipeline.Workspace)/thirdPartyToBeSigned').FullName - Update-PSSignedBuildFolder -BuildPath $pathForUpload -SignedFilesPath $thirdPartySignedFilesPath + Update-PSSignedBuildFolder -BuildPath $pathForUpload -SignedFilesPath $thirdPartySignedFilesPath -OfficialBuild $officialBuild displayName: 'Copy signed files for upload' diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index f3b4a16b318..6b78d94ab0b 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -887,7 +887,8 @@ function Update-PSSignedBuildFolder [string]$BuildPath, [Parameter(Mandatory)] [string]$SignedFilesPath, - [string[]] $RemoveFilter = ('*.pdb', '*.zip', '*.r2rmap') + [string[]] $RemoveFilter = ('*.pdb', '*.zip', '*.r2rmap'), + [bool]$OfficialBuild = $true ) $BuildPathNormalized = (Get-Item $BuildPath).FullName @@ -943,8 +944,21 @@ function Update-PSSignedBuildFolder if ($IsWindows) { $signature = Get-AuthenticodeSignature -FilePath $signedFilePath - if ($signature.Status -ne 'Valid') { + + if ($signature.Status -ne 'Valid' -and $OfficialBuild) { + Write-Host "Certificate Issuer: $($signature.SignerCertificate.Issuer)" + Write-Host "Certificate Subject: $($signature.SignerCertificate.Subject)" Write-Error "Invalid signature for $signedFilePath" + } elseif ($OfficialBuild -eq $false) { + if ($signature.Status -eq 'NotSigned') { + Write-Warning "File is not signed: $signedFilePath" + } elseif ($signature.SignerCertificate.Issuer -notmatch '^CN=(Microsoft|TestAzureEngBuildCodeSign|Windows Internal Build Tools).*') { + Write-Warning "File signed with test certificate: $signedFilePath" + Write-Host "Certificate Issuer: $($signature.SignerCertificate.Issuer)" + Write-Host "Certificate Subject: $($signature.SignerCertificate.Subject)" + } else { + Write-Verbose -Verbose "File properly signed: $signedFilePath" + } } } else