From 4eec701ef7c171ff90ccd27876937e2054e3e37e Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 3 Apr 2025 10:10:36 -0700 Subject: [PATCH 1/2] Cherry-pick PR #25275 with conflicts for manual resolution --- .../infrastructure/path-filters/action.yml | 88 +++++++++++++++++ .github/actions/test/nix/action.yml | 27 +++++- .../test/process-pester-results/action.yml | 64 ++++++------- .github/workflows/linux-ci.yml | 28 +++--- .github/workflows/macos-ci.yml | 23 ++--- .github/workflows/windows-ci.yml | 23 ++--- build.psm1 | 95 ++++++++++++++++++- 7 files changed, 262 insertions(+), 86 deletions(-) create mode 100644 .github/actions/infrastructure/path-filters/action.yml diff --git a/.github/actions/infrastructure/path-filters/action.yml b/.github/actions/infrastructure/path-filters/action.yml new file mode 100644 index 00000000000..58255fab55c --- /dev/null +++ b/.github/actions/infrastructure/path-filters/action.yml @@ -0,0 +1,88 @@ +name: Path Filters +description: 'Path Filters' +inputs: + GITHUB_TOKEN: + description: 'GitHub token' + required: true +outputs: + source: + description: 'Source code changes (composite of all changes)' + value: ${{ steps.filter.outputs.source }} + githubChanged: + description: 'GitHub workflow changes' + value: ${{ steps.filter.outputs.githubChanged }} + toolsChanged: + description: 'Tools changes' + value: ${{ steps.filter.outputs.toolsChanged }} + propsChanged: + description: 'Props changes' + value: ${{ steps.filter.outputs.propsChanged }} + testsChanged: + description: 'Tests changes' + value: ${{ steps.filter.outputs.testsChanged }} + mainSourceChanged: + description: 'Main source code changes (any changes in src/)' + value: ${{ steps.filter.outputs.mainSourceChanged }} + buildModuleChanged: + description: 'Build module changes' + value: ${{ steps.filter.outputs.buildModuleChanged }} +runs: + using: composite + steps: + - name: Check if GitHubWorkflowChanges is present + id: filter + uses: actions/github-script@v7.0.1 + with: + github-token: ${{ inputs.GITHUB_TOKEN }} + script: | + // Fetch the list of files changed in the PR + let files = []; + let page = 1; + let fetchedFiles; + do { + fetchedFiles = await github.rest.pulls.listFiles({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + per_page: 100, + page: page++ + }); + files = files.concat(fetchedFiles.data); + } while (fetchedFiles.data.length > 0); + + const actionsChanged = files.some(file => file.filename.startsWith('.github/actions')); + const workflowsChanged = files.some(file => file.filename.startsWith('.github/workflows')); + const githubChanged = actionsChanged || workflowsChanged; + + const toolsCiPsm1Changed = files.some(file => file.filename.startsWith('tools/ci.psm1')); + const toolsBuildCommonChanged = files.some(file => file.filename.startsWith('tools/buildCommon/')); + const toolsChanged = toolsCiPsm1Changed || toolsBuildCommonChanged; + + const propsChanged = files.some(file => file.filename.endsWith('.props')); + + const testsChanged = files.some(file => file.filename.startsWith('test/powershell/') || file.filename.startsWith('test/tools/') || file.filename.startsWith('test/xUnit/')); + + const mainSourceChanged = files.some(file => file.filename.startsWith('src/')); + + const buildModuleChanged = files.some(file => file.filename.startsWith('build.psm1')); + + const source = mainSourceChanged || toolsChanged || githubChanged || propsChanged || testsChanged; + + core.setOutput('toolsChanged', toolsChanged); + core.setOutput('githubChanged', githubChanged); + core.setOutput('propsChanged', propsChanged); + core.setOutput('testsChanged', testsChanged); + core.setOutput('mainSourceChanged', mainSourceChanged); + core.setOutput('buildModuleChanged', buildModuleChanged); + core.setOutput('source', source); + + - name: Capture outputs + run: | + Write-Verbose -Verbose "source: ${{ steps.filter.outputs.source }}" + Write-Verbose -Verbose "github: ${{ steps.filter.outputs.githubChanged }}" + Write-Verbose -Verbose "tools: ${{ steps.filter.outputs.toolsChanged }}" + Write-Verbose -Verbose "props: ${{ steps.filter.outputs.propsChanged }}" + Write-Verbose -Verbose "tests: ${{ steps.filter.outputs.testsChanged }}" + Write-Verbose -Verbose "mainSource: ${{ steps.filter.outputs.mainSourceChanged }}" + Write-Verbose -Verbose "buildModule: ${{ steps.filter.outputs.buildModuleChanged }}" + shell: pwsh diff --git a/.github/actions/test/nix/action.yml b/.github/actions/test/nix/action.yml index 03c44a151c7..cf586f894a0 100644 --- a/.github/actions/test/nix/action.yml +++ b/.github/actions/test/nix/action.yml @@ -20,26 +20,39 @@ runs: steps: - name: Capture Environment if: success() || failure() - run: 'Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose' + run: |- + Import-Module ./build.psm1 + Write-LogGroupStart -Title 'Environment' + Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + Write-LogGroupEnd -Title 'Environment' shell: pwsh + - name: Download Build Artifacts uses: actions/download-artifact@v4 with: path: "${{ github.workspace }}" + - name: Capture Artifacts Directory continue-on-error: true - run: Get-ChildItem "${{ github.workspace }}/build/*" -Recurse + run: |- + Import-Module ./build.psm1 + Write-LogGroupStart -Title 'Artifacts Directory' + Get-ChildItem "${{ github.workspace }}/build/*" -Recurse + Write-LogGroupEnd -Title 'Artifacts Directory' shell: pwsh - + - uses: actions/setup-dotnet@v4 with: global-json-file: ./global.json - + - name: Bootstrap shell: pwsh run: |- + Import-Module ./build.psm1 + Write-LogGroupStart -Title 'Bootstrap' Import-Module ./tools/ci.psm1 Invoke-CIInstall -SkipUser + Write-LogGroupEnd -Title 'Bootstrap' - name: Extract Files uses: actions/github-script@v7.0.0 @@ -68,7 +81,11 @@ runs: - name: Capture Extracted Build ZIP continue-on-error: true - run: Get-ChildItem "${{ github.workspace }}/bins/*" -Recurse -ErrorAction SilentlyContinue + run: |- + Import-Module ./build.psm1 + Write-LogGroupStart -Title 'Extracted Build ZIP' + Get-ChildItem "${{ github.workspace }}/bins/*" -Recurse -ErrorAction SilentlyContinue + Write-LogGroupEnd -Title 'Extracted Build ZIP' shell: pwsh - name: Test diff --git a/.github/actions/test/process-pester-results/action.yml b/.github/actions/test/process-pester-results/action.yml index 758bbdfc353..e1072ec08ca 100644 --- a/.github/actions/test/process-pester-results/action.yml +++ b/.github/actions/test/process-pester-results/action.yml @@ -18,36 +18,39 @@ inputs: runs: using: composite steps: - - name: Convert JUnit to CTRF - run: |- + - name: Log Summary + run: | + if (-not $env:GITHUB_STEP_SUMMARY) { + Write-Error "GITHUB_STEP_SUMMARY is not set. Ensure this workflow is running in a GitHub Actions environment." + exit 1 + } + + $testCaseCount = 0 + $testErrorCount = 0 + $testFailureCount = 0 + $testDisabledCount = 0 Get-ChildItem -Path "${{ inputs.testResultsFolder }}/*.xml" -Recurse | ForEach-Object { - npx --yes junit-to-ctrf $_.FullName --output ./${{ inputs.ctrfFolder }}/$($_.BaseName).json --tool Pester + $results = [xml] (get-content $_.FullName) + $testCaseCount += $results.testsuites.tests + $testErrorCount += $results.testsuites.errors + $testFailureCount += $results.testsuites.failures + $testDisabledCount += $results.testsuites.disabled } - shell: pwsh - # this task only takes / as directory separators - - name: Publish Test Report - uses: ctrf-io/github-test-reporter@v1 - with: - report-path: './${{ inputs.ctrfFolder }}/*.json' - exit-on-fail: true - summary-report: true - test-report: false - test-list-report: false - failed-report: false - fail-rate-report: false - flaky-report: false - flaky-rate-report: false - failed-folded-report: true - previous-results-report: false - ai-report: true - skipped-report: false - suite-folded-report: false - suite-list-report: false - pull-request-report: false - commit-report: false - custom-report: false - if: always() + @" + + # Summary of ${{ inputs.name }} + + - Total Tests: $testCaseCount + - Total Errors: $testErrorCount + - Total Failures: $testFailureCount + - Total Disabled: $testDisabledCount + + "@ | Out-File -FilePath $ENV:GITHUB_STEP_SUMMARY -Append + + Write-Host "Summary written to $ENV:GITHUB_STEP_SUMMARY" + Get-Content $ENV:GITHUB_STEP_SUMMARY + shell: pwsh - name: Upload testResults artifact if: always() @@ -55,10 +58,3 @@ runs: with: name: junit-pester-${{ inputs.name }} path: ${{ runner.workspace }}/testResults - - - name: Upload ctrf artifact - if: always() - uses: actions/upload-artifact@v4 - with: - name: ctrf-pester-${{ inputs.name }} - path: ${{ inputs.ctrfFolder }} diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index d7d76310846..dfc3d4c7e4b 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -9,6 +9,7 @@ on: branches: - master - release/** + - github-mirror paths: - "**" - "!.github/ISSUE_TEMPLATE/**" @@ -19,6 +20,7 @@ on: branches: - master - release/** + - github-mirror # Path filters for PRs need to go into the changes job concurrency: @@ -43,30 +45,30 @@ jobs: # Required permissions permissions: pull-requests: read + contents: read + # Set job outputs to values from filter step outputs: source: ${{ steps.filter.outputs.source }} steps: - name: checkout +<<<<<<< HEAD uses: actions/checkout@v4.1.0 # For pull requests it's not necessary to checkout the code - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.2.0 id: filter +======= + uses: actions/checkout@v4 +>>>>>>> f60193b9d (Make GitHub Workflows work in the internal mirror (#25275)) + with: + persist-credentials: false + + - name: Change Detection + id: filter + uses: "./.github/actions/infrastructure/path-filters" with: - list-files: json - filters: .github/action-filters.yml - - - name: Capture outputs - run: | - "source: ${{ steps.filter.outputs.source }}" - "github: ${{ steps.filter.outputs.github }}" - "tools: ${{ steps.filter.outputs.tools }}" - "props: ${{ steps.filter.outputs.props }}" - "tests: ${{ steps.filter.outputs.tests }}" - "mainSource: ${{ steps.filter.outputs.mainSource }}" - "buildModule: ${{ steps.filter.outputs.buildModule }}" - shell: pwsh + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ci_build: name: Build PowerShell diff --git a/.github/workflows/macos-ci.yml b/.github/workflows/macos-ci.yml index d7d5dc76d0b..9184dc088f0 100644 --- a/.github/workflows/macos-ci.yml +++ b/.github/workflows/macos-ci.yml @@ -7,6 +7,7 @@ on: branches: - master - release/** + - github-mirror paths: - "**" - "!.github/ISSUE_TEMPLATE/**" @@ -17,6 +18,7 @@ on: branches: - master - release/** + - github-mirror # Path filters for PRs need to go into the changes job concurrency: @@ -34,6 +36,7 @@ env: __SuppressAnsiEscapeSequences: 1 nugetMultiFeedWarnLevel: none system_debug: 'false' + jobs: changes: name: Change Detection @@ -42,6 +45,8 @@ jobs: # Required permissions permissions: pull-requests: read + contents: read + # Set job outputs to values from filter step outputs: source: ${{ steps.filter.outputs.source }} @@ -49,23 +54,11 @@ jobs: - name: checkout uses: actions/checkout@v4.1.0 - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.2.0 + - name: Change Detection id: filter + uses: "./.github/actions/infrastructure/path-filters" with: - list-files: json - filters: .github/action-filters.yml - - - name: Capture outputs - run: | - "source: ${{ steps.filter.outputs.source }}" - "github: ${{ steps.filter.outputs.github }}" - "tools: ${{ steps.filter.outputs.tools }}" - "props: ${{ steps.filter.outputs.props }}" - "tests: ${{ steps.filter.outputs.tests }}" - "mainSource: ${{ steps.filter.outputs.mainSource }}" - "buildModule: ${{ steps.filter.outputs.buildModule }}" - shell: pwsh + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ci_build: name: Build PowerShell diff --git a/.github/workflows/windows-ci.yml b/.github/workflows/windows-ci.yml index 9f2aef06a59..1e955c78be6 100644 --- a/.github/workflows/windows-ci.yml +++ b/.github/workflows/windows-ci.yml @@ -5,6 +5,7 @@ on: branches: - master - release/** + - github-mirror paths: - "**" - "!.vsts-ci/misc-analysis.yml" @@ -16,6 +17,8 @@ on: branches: - master - release/** + - github-mirror + # Path filters for PRs need to go into the changes job concurrency: @@ -43,6 +46,8 @@ jobs: # Required permissions permissions: pull-requests: read + contents: read + # Set job outputs to values from filter step outputs: source: ${{ steps.filter.outputs.source }} @@ -50,23 +55,11 @@ jobs: - name: checkout uses: actions/checkout@v4.1.0 - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.2.0 + - name: Change Detection id: filter + uses: "./.github/actions/infrastructure/path-filters" with: - list-files: json - filters: .github/action-filters.yml - - - name: Capture outputs - run: | - "source: ${{ steps.filter.outputs.source }}" - "github: ${{ steps.filter.outputs.github }}" - "tools: ${{ steps.filter.outputs.tools }}" - "props: ${{ steps.filter.outputs.props }}" - "tests: ${{ steps.filter.outputs.tests }}" - "mainSource: ${{ steps.filter.outputs.mainSource }}" - "buildModule: ${{ steps.filter.outputs.buildModule }}" - shell: pwsh + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ci_build: name: Build PowerShell diff --git a/build.psm1 b/build.psm1 index e882e2f797c..73becb3f0d0 100644 --- a/build.psm1 +++ b/build.psm1 @@ -179,6 +179,8 @@ function Get-EnvironmentInformation $environment += @{'IsUbuntu16' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '16.04'} $environment += @{'IsUbuntu18' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '18.04'} $environment += @{'IsUbuntu20' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '20.04'} + $environment += @{'IsUbuntu22' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '22.04'} + $environment += @{'IsUbuntu24' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '24.04'} $environment += @{'IsCentOS' = $LinuxInfo.ID -match 'centos' -and $LinuxInfo.VERSION_ID -match '7'} $environment += @{'IsFedora' = $LinuxInfo.ID -match 'fedora' -and $LinuxInfo.VERSION_ID -ge 24} $environment += @{'IsOpenSUSE' = $LinuxInfo.ID -match 'opensuse'} @@ -2693,6 +2695,10 @@ function script:Write-Log if ($isError) { Write-Host -Foreground Red $message + if($env:GITHUB_WORKFLOW) + { + Write-Host "::error::${message}" + } } else { @@ -2701,6 +2707,59 @@ function script:Write-Log #reset colors for older package to at return to default after error message on a compilation error [console]::ResetColor() } + +function script:Write-LogGroup { + param + ( + [Parameter(Position = 0, Mandatory)] + [ValidateNotNullOrEmpty()] + [string[]] $Message, + [Parameter(Mandatory)] + [string] $Title + ) + + + Write-LogGroupStart -Title $Title + + foreach ($line in $Message) { + Write-Log -Message $line + } + + Write-LogGroupEnd -Title $Title +} + +$script:logGroupColor = [System.ConsoleColor]::Cyan + +function script:Write-LogGroupStart { + param + ( + [Parameter(Mandatory)] + [string] $Title + ) + + if ($env:GITHUB_WORKFLOW) { + Write-Host "::group::${Title}" + } + else { + Write-Host -ForegroundColor $script:logGroupColor "=== BEGIN: $Title ===" + } +} + +function script:Write-LogGroupEnd { + param + ( + [Parameter(Mandatory)] + [string] $Title + ) + + if ($env:GITHUB_WORKFLOW) { + Write-Host "::endgroup::" + } + else { + Write-Host -ForegroundColor $script:logGroupColor "==== END: $Title ====" + } +} + function script:precheck([string]$command, [string]$missedMessage) { $c = Get-Command $command -ErrorAction Ignore if (-not $c) { @@ -3619,22 +3678,50 @@ function Set-PipelineNugetAuthentication { function Set-CorrectLocale { + Write-LogGroupStart -Title "Set-CorrectLocale" + if (-not $IsLinux) { + Write-LogGroupEnd -Title "Set-CorrectLocale" return } $environment = Get-EnvironmentInformation - if ($environment.IsUbuntu -and $environment.IsUbuntu20) - { + if ($environment.IsUbuntu16 -or $environment.IsUbuntu18) { + Write-Verbose -Message "Don't set locale before Ubuntu 20" -Verbose + Write-LogGroupEnd -Title "Set-CorrectLocale" + Write-Locale + return + } + + if ($environment.IsUbuntu) { + Write-Log -Message "Setting locale to en_US.UTF-8" $env:LC_ALL = 'en_US.UTF-8' $env:LANG = 'en_US.UTF-8' sudo locale-gen $env:LANG - sudo update-locale + if ($environment.IsUbuntu20) { + Write-Log -Message "Updating locale for Ubuntu 20" + sudo update-locale + } else { + Write-Log -Message "Updating locale for Ubuntu 22 and newer" + sudo update-locale LANG=$env:LANG LC_ALL=$env:LC_ALL + } + } + + Write-LogGroupEnd -Title "Set-CorrectLocale" + Write-Locale + +} + +function Write-Locale { + if (-not $IsLinux -and -not $IsMacOS) { + Write-Verbose -Message "only supported on Linux and macOS" -Verbose + return } # Output the locale to log it - locale + $localOutput = & locale + Write-LogGroup -Title "Capture Locale" -Message $localOutput } function Install-AzCopy { From 84477e65d81867e4b0f29cbb6ef24f7839006ac9 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Fri, 11 Apr 2025 14:11:22 -0700 Subject: [PATCH 2/2] Update .github/workflows/linux-ci.yml --- .github/workflows/linux-ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index dfc3d4c7e4b..eda1ea56167 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -52,15 +52,7 @@ jobs: source: ${{ steps.filter.outputs.source }} steps: - name: checkout -<<<<<<< HEAD - uses: actions/checkout@v4.1.0 - - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.2.0 - id: filter -======= uses: actions/checkout@v4 ->>>>>>> f60193b9d (Make GitHub Workflows work in the internal mirror (#25275)) with: persist-credentials: false