From edadefbb04d1a43c3572e9d92d3d9510600ed70b Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 7 Apr 2025 13:24:09 -0700 Subject: [PATCH] Update test result processing to use NUnitXml format and enhance logging for better clarity (#25288) --- .github/actions/test/nix/action.yml | 2 +- .../test/process-pester-results/action.yml | 37 +--------- .../process-pester-results.ps1 | 68 +++++++++++++++++++ .github/actions/test/windows/action.yml | 20 ++++-- build.psm1 | 57 ++++++++++------ 5 files changed, 125 insertions(+), 59 deletions(-) create mode 100644 .github/actions/test/process-pester-results/process-pester-results.ps1 diff --git a/.github/actions/test/nix/action.yml b/.github/actions/test/nix/action.yml index cf586f894a0..b338c398340 100644 --- a/.github/actions/test/nix/action.yml +++ b/.github/actions/test/nix/action.yml @@ -101,7 +101,7 @@ runs: chmod a+x $pwshPath $options.Output = $pwshPath Set-PSOptions $options - Invoke-CITest -Purpose '${{ inputs.purpose }}' -TagSet '${{ inputs.tagSet }}' -TitlePrefix '${{ inputs.buildName }}' -OutputFormat JUnitXml + Invoke-CITest -Purpose '${{ inputs.purpose }}' -TagSet '${{ inputs.tagSet }}' -TitlePrefix '${{ inputs.buildName }}' -OutputFormat NUnitXml shell: pwsh - name: Convert, Publish, and Upload Pester Test Results diff --git a/.github/actions/test/process-pester-results/action.yml b/.github/actions/test/process-pester-results/action.yml index e1072ec08ca..27b94f6ebcb 100644 --- a/.github/actions/test/process-pester-results/action.yml +++ b/.github/actions/test/process-pester-results/action.yml @@ -10,46 +10,13 @@ inputs: required: false default: "${{ runner.workspace }}/testResults" type: string - ctrfFolder: - required: false - default: ctrf - type: string runs: using: composite steps: - name: Log Summary - run: | - if (-not $env:GITHUB_STEP_SUMMARY) { - Write-Error "GITHUB_STEP_SUMMARY is not set. Ensure this workflow is running in a GitHub Actions environment." - exit 1 - } - - $testCaseCount = 0 - $testErrorCount = 0 - $testFailureCount = 0 - $testDisabledCount = 0 - Get-ChildItem -Path "${{ inputs.testResultsFolder }}/*.xml" -Recurse | ForEach-Object { - $results = [xml] (get-content $_.FullName) - $testCaseCount += $results.testsuites.tests - $testErrorCount += $results.testsuites.errors - $testFailureCount += $results.testsuites.failures - $testDisabledCount += $results.testsuites.disabled - } - - @" - - # Summary of ${{ inputs.name }} - - - Total Tests: $testCaseCount - - Total Errors: $testErrorCount - - Total Failures: $testFailureCount - - Total Disabled: $testDisabledCount - - "@ | Out-File -FilePath $ENV:GITHUB_STEP_SUMMARY -Append - - Write-Host "Summary written to $ENV:GITHUB_STEP_SUMMARY" - Get-Content $ENV:GITHUB_STEP_SUMMARY + run: |- + & "$env:GITHUB_ACTION_PATH/process-pester-results.ps1" -Name '${{ inputs.name }}' -TestResultsFolder '${{ inputs.testResultsFolder }}' shell: pwsh - name: Upload testResults artifact diff --git a/.github/actions/test/process-pester-results/process-pester-results.ps1 b/.github/actions/test/process-pester-results/process-pester-results.ps1 new file mode 100644 index 00000000000..523de3bebaa --- /dev/null +++ b/.github/actions/test/process-pester-results/process-pester-results.ps1 @@ -0,0 +1,68 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +param( + [parameter(Mandatory)] + [string]$Name, + [parameter(Mandatory)] + [string]$TestResultsFolder +) + +Import-Module "$PSScriptRoot/../../../../build.psm1" + +if (-not $env:GITHUB_STEP_SUMMARY) { + Write-Error "GITHUB_STEP_SUMMARY is not set. Ensure this workflow is running in a GitHub Actions environment." + exit 1 +} + +$testCaseCount = 0 +$testErrorCount = 0 +$testFailureCount = 0 +$testNotRunCount = 0 +$testInconclusiveCount = 0 +$testIgnoredCount = 0 +$testSkippedCount = 0 +$testInvalidCount = 0 + +Get-ChildItem -Path "${TestResultsFolder}/*.xml" -Recurse | ForEach-Object { + $results = [xml] (get-content $_.FullName) + + $testCaseCount += [int]$results.'test-results'.total + $testErrorCount += [int]$results.'test-results'.errors + $testFailureCount += [int]$results.'test-results'.failures + $testNotRunCount += [int]$results.'test-results'.'not-run' + $testInconclusiveCount += [int]$results.'test-results'.inconclusive + $testIgnoredCount += [int]$results.'test-results'.ignored + $testSkippedCount += [int]$results.'test-results'.skipped + $testInvalidCount += [int]$results.'test-results'.invalid +} + +@" + +# Summary of $Name + +- Total Tests: $testCaseCount +- Total Errors: $testErrorCount +- Total Failures: $testFailureCount +- Total Not Run: $testNotRunCount +- Total Inconclusive: $testInconclusiveCount +- Total Ignored: $testIgnoredCount +- Total Skipped: $testSkippedCount +- Total Invalid: $testInvalidCount + +"@ | Out-File -FilePath $ENV:GITHUB_STEP_SUMMARY -Append + +Write-Log "Summary written to $ENV:GITHUB_STEP_SUMMARY" + +Write-LogGroupStart -Title 'Test Results' +Get-Content $ENV:GITHUB_STEP_SUMMARY +Write-LogGroupEnd -Title 'Test Results' + +if ($testErrorCount -gt 0 -or $testFailureCount -gt 0) { + Write-Error "There were $testErrorCount/$testFailureCount errors/failures in the test results." + exit 1 +} +if ($testCaseCount -eq 0) { + Write-Error "No test cases were run." + exit 1 +} diff --git a/.github/actions/test/windows/action.yml b/.github/actions/test/windows/action.yml index d2af55ce5a9..734e30208f0 100644 --- a/.github/actions/test/windows/action.yml +++ b/.github/actions/test/windows/action.yml @@ -20,15 +20,25 @@ runs: steps: - name: Capture Environment if: success() || failure() - run: 'Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose' + run: |- + Import-Module ./build.psm1 + Write-LogGroupStart -Title 'Environment' + Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + Write-LogGroupEnd -Title 'Environment' shell: pwsh + - name: Download Build Artifacts uses: actions/download-artifact@v4 with: path: "${{ github.workspace }}" + - name: Capture Artifacts Directory continue-on-error: true - run: Get-ChildItem "${{ github.workspace }}\build\*" -Recurse + run: |- + Import-Module ./build.psm1 + Write-LogGroupStart -Title 'Artifacts Directory' + Get-ChildItem "${{ github.workspace }}/build/*" -Recurse + Write-LogGroupEnd -Title 'Artifacts Directory' shell: pwsh - uses: actions/setup-dotnet@v4 @@ -38,7 +48,8 @@ runs: - name: Bootstrap shell: powershell run: |- - # Remove "Program Files\dotnet" from the env variable PATH, so old SDKs won't affect us. + Import-Module ./build.psm1 + Write-LogGroupStart -Title 'Bootstrap' Write-Host "Old Path:" Write-Host $env:Path $dotnetPath = Join-Path $env:SystemDrive 'Program Files\dotnet' @@ -49,6 +60,7 @@ runs: # Bootstrap Import-Module .\tools\ci.psm1 Invoke-CIInstall + Write-LogGroupEnd -Title 'Bootstrap' - name: Test if: success() @@ -60,7 +72,7 @@ runs: $path = split-path -path $options.Output $rootPath = split-Path -path $path Expand-Archive -Path '${{ github.workspace }}\build\build.zip' -DestinationPath $rootPath -Force - Invoke-CITest -Purpose '${{ inputs.purpose }}' -TagSet '${{ inputs.tagSet }}' -OutputFormat JUnitXml + Invoke-CITest -Purpose '${{ inputs.purpose }}' -TagSet '${{ inputs.tagSet }}' -OutputFormat NUnitXml shell: pwsh - name: Convert, Publish, and Upload Pester Test Results diff --git a/build.psm1 b/build.psm1 index 73becb3f0d0..b371891d706 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1231,6 +1231,7 @@ function Get-PesterTag { # testing PowerShell remote custom connections. function Publish-CustomConnectionTestModule { + Write-LogGroupStart -Title "Publish-CustomConnectionTestModule" $sourcePath = "${PSScriptRoot}/test/tools/NamedPipeConnection" $outPath = "${PSScriptRoot}/test/tools/NamedPipeConnection/out/Microsoft.PowerShell.NamedPipeConnection" $publishPath = "${PSScriptRoot}/test/tools/Modules" @@ -1255,6 +1256,8 @@ function Publish-CustomConnectionTestModule finally { Pop-Location } + + Write-LogGroupEnd -Title "Publish-CustomConnectionTestModule" } function Publish-PSTestTools { @@ -1264,6 +1267,7 @@ function Publish-PSTestTools { $runtime ) + Write-LogGroupStart -Title "Publish-PSTestTools" Find-Dotnet $tools = @( @@ -1335,6 +1339,7 @@ function Publish-PSTestTools { # Publish the Microsoft.PowerShell.NamedPipeConnection module Publish-CustomConnectionTestModule + Write-LogGroupEnd -Title "Publish-PSTestTools" } function Get-ExperimentalFeatureTests { @@ -1822,12 +1827,16 @@ function Show-PSPesterError throw 'Unknown Show-PSPester parameter set' } - Write-Log -isError -message ("Description: " + $description) - Write-Log -isError -message ("Name: " + $name) - Write-Log -isError -message "message:" - Write-Log -isError -message $message - Write-Log -isError -message "stack-trace:" - Write-Log -isError -message $stack_trace + # Empty line at the end is intentional formatting + Write-Log -isError -message @" +Description: $description +Name: $name +message: +$message +stack-trace: +$stack_trace + +"@ } @@ -1867,13 +1876,17 @@ function Test-XUnitTestResults $message = $failure.failure.message $stack_trace = $failure.failure.'stack-trace' - Write-Log -isError -message ("Description: " + $description) - Write-Log -isError -message ("Name: " + $name) - Write-Log -isError -message "message:" - Write-Log -isError -message $message - Write-Log -isError -message "stack-trace:" - Write-Log -isError -message $stack_trace - Write-Log -isError -message " " + # Empty line at the end is intentional formatting + Write-Log -isError -message @" + Description: $description + Name: $name + message: + $message + stack-trace: + $stack_trace + +"@ + } throw "$($results.assemblies.assembly.failed) tests failed" @@ -1909,7 +1922,8 @@ function Test-PSPesterResults $x = [xml](Get-Content -Raw $testResultsFile) if ([int]$x.'test-results'.failures -gt 0) { - Write-Log -isError -message "TEST FAILURES" + Write-LogGroupStart -Title 'TEST FAILURES' + # switch between methods, SelectNode is not available on dotnet core if ( "System.Xml.XmlDocumentXPathExtensions" -as [Type] ) { @@ -1923,6 +1937,8 @@ function Test-PSPesterResults { Show-PSPesterError -testFailure $testfail } + + Write-LogGroupEnd -Title 'TEST FAILURES' throw "$($x.'test-results'.failures) tests in $TestArea failed" } } @@ -1943,11 +1959,12 @@ function Test-PSPesterResults } elseif ($ResultObject.FailedCount -gt 0) { - Write-Log -isError -message 'TEST FAILURES' + Write-LogGroupStart -Title 'TEST FAILURES' $ResultObject.TestResult | Where-Object {$_.Passed -eq $false} | ForEach-Object { Show-PSPesterError -testFailureObject $_ } + Write-LogGroupEnd -Title 'TEST FAILURES' throw "$($ResultObject.FailedCount) tests in $TestArea failed" } @@ -2694,10 +2711,12 @@ function script:Write-Log ) if ($isError) { - Write-Host -Foreground Red $message - if($env:GITHUB_WORKFLOW) - { - Write-Host "::error::${message}" + if ($env:GITHUB_WORKFLOW) { + # https://github.com/actions/toolkit/issues/193#issuecomment-605394935 + $escapedMessage = $message -replace "`n", "%0A" -replace "`r" + Write-Host "::error::${escapedMessage}" + } else { + Write-Host -Foreground Red $message } } else