Outerloop Tests #708
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Executes quarantined tests in the outerloop | |
| name: Outerloop Tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 */2 * * *' # Every 2 hours | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| generate_tests_matrix: | |
| name: Generate test runsheet | |
| runs-on: windows-latest | |
| if: ${{ github.repository_owner == 'dotnet' }} | |
| outputs: | |
| runsheet: ${{ steps.generate_tests_matrix.outputs.runsheet }} | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| # We need to build the whole solution, so that we can interrogate each test project | |
| # and find out whether it contains any quarantined tests. | |
| - name: Build the solution | |
| run: | | |
| ./build.cmd -restore -build -c Release -ci /p:CI=false /p:GeneratePackageOnBuild=false /p:InstallBrowsersForPlaywright=false | |
| - name: Generate test runsheet | |
| id: generate_tests_matrix | |
| run: | | |
| ./build.cmd -test /p:TestRunnerName=QuarantinedTestRunsheetBuilder /p:RunQuarantinedTests=true -c Release -ci /p:CI=false /p:Restore=false /p:Build=false /bl:./artifacts/log/Release/runsheet.binlog | |
| - name: Upload logs, and test results | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 | |
| with: | |
| name: logs-runsheet | |
| path: | | |
| ${{ github.workspace }}/artifacts/log/*/runsheet.binlog | |
| ${{ github.workspace }}/artifacts/log/*/TestLogs/** | |
| ${{ github.workspace }}/artifacts/tmp/*/combined_runsheet.json | |
| retention-days: 5 | |
| run_tests: | |
| name: Test | |
| needs: generate_tests_matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tests: ${{ fromJson(needs.generate_tests_matrix.outputs.runsheet) }} | |
| runs-on: ${{ matrix.tests.os }} # Use the OS from the matrix | |
| if: ${{ github.repository_owner == 'dotnet' }} | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Trust HTTPS development certificate (Linux) | |
| if: matrix.tests.os == 'ubuntu-latest' | |
| run: ./dotnet.sh dev-certs https --trust | |
| - name: Test ${{ matrix.tests.label }} | |
| run: | | |
| ${{ matrix.tests.command }} | |
| - name: Upload logs, and test results | |
| if: always() | |
| uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 | |
| with: | |
| name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-logs | |
| path: | | |
| ${{ github.workspace }}/artifacts/log/*/TestLogs/**/*.log | |
| ${{ github.workspace }}/artifacts/TestResults/*/*.trx | |
| # Longer retention time to allow scanning runs for quarantined test results | |
| retention-days: 30 | |
| results: | |
| if: always() | |
| runs-on: ubuntu-latest | |
| name: Final Results | |
| needs: run_tests | |
| steps: | |
| # get all the test-job-result* artifacts into a single directory | |
| - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 | |
| with: | |
| pattern: '*-logs' | |
| path: ${{ github.workspace }}/artifacts/all-logs | |
| - name: Process logs and post results | |
| if: ${{ always() && github.repository_owner == 'dotnet' }} | |
| shell: pwsh | |
| run: | | |
| $logDirectory = "${{ github.workspace }}/artifacts/all-logs" | |
| $trxFiles = Get-ChildItem -Path $logDirectory -Filter *.trx -Recurse | |
| $testResults = @() # Initialize an array to store test results | |
| foreach ($trxFile in $trxFiles) { | |
| # Determine the OS based on the file path | |
| if ($trxFile.FullName -match "ubuntu") { | |
| $OS = "ubuntu" | |
| } elseif ($trxFile.FullName -match "windows") { | |
| $OS = "windows" | |
| } elseif ($trxFile.FullName -match "macos") { | |
| $OS = "macos" | |
| } else { | |
| $OS = "unknown" | |
| } | |
| # Load the .trx file as XML | |
| try { | |
| # Attempt to load the .trx file as XML | |
| $xmlContent = [xml](Get-Content -Path $trxFile.FullName) | |
| # Extract test results from the XML | |
| foreach ($testResult in $xmlContent.TestRun.Results.UnitTestResult) { | |
| $testName = $testResult.testName | |
| $outcome = $testResult.outcome | |
| $duration = $testResult.duration | |
| # Map outcome to emoji | |
| switch ($outcome) { | |
| "Passed" { $emoji = "✅" } | |
| "Failed" { $emoji = "❌" } | |
| default { $emoji = "❔" } | |
| } | |
| # Normalize the duration to a consistent format (hh:mm:ss.fff) | |
| $normalizedDuration = [TimeSpan]::Parse($duration).ToString("mm\:ss\.fff") | |
| # Add the test result to the array | |
| $testResults += [PSCustomObject]@{ | |
| TestName = $testName | |
| Outcome = $outcome | |
| OutcomeIcon = $emoji | |
| Duration = $normalizedDuration | |
| OS = $OS | |
| } | |
| } | |
| } catch { | |
| Write-Host "::error::Failed to process $($trxFile.FullName): $($_.Exception.Message)" | |
| } | |
| } | |
| if ($testResults.Length -lt 1) { | |
| Write-Host "::notice::Tests Summary: no quaratined tests found" | |
| return; | |
| } | |
| # Sort the test results by test name | |
| $testResults = $testResults | Sort-Object -Property TestName | |
| # Calculate summary statistics | |
| $totalTests = $testResults.Count | |
| $passedTests = ($testResults | Where-Object { $_.Outcome -eq "Passed" }).Count | |
| $failedTests = ($testResults | Where-Object { $_.Outcome -eq "Failed" }).Count | |
| $skippedTests = ($testResults | Where-Object { $_.Outcome -eq "NotExecuted" }).Count | |
| # Add the summary to the annotation | |
| $summary = "total: $totalTests, passed: $passedTests, failed: $failedTests, skipped: $skippedTests" | |
| if ($failedTests -gt 0) { | |
| Write-Host "::error::Tests Summary: $summary" | |
| } else { | |
| Write-Host "::notice::Tests Summary: $summary" | |
| } | |
| # Format the test results as a console-friendly table | |
| $tableHeader = "{0,-12} {1,-10} {2,-140} {3,-16}" -f "Result", "OS", "Test Name", "Duration" | |
| $tableSeparator = "-" * 190 | |
| $tableRows = $testResults | ForEach-Object { "{0,-12} {1,-10} {2,-140} {3,-16}" -f "$($_.OutcomeIcon) $($_.Outcome)", $_.OS, $_.TestName, $_.Duration } | |
| $table = "$tableHeader`n$tableSeparator`n" + ($tableRows -join "`n") + "`n$tableSeparator`n" | |
| Write-Host "`nTest Results:`n`n$table" | |
| # Optionally, save the results to a file for further processing | |
| $outputPath = "${{ github.workspace }}/artifacts/summary.log" | |
| $table | Out-File -FilePath $outputPath -Encoding utf8 | |
| Write-Host "Test results saved to $outputPath" |