From de7f155e530dca90ee9b36e8824ceeb733bfe642 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Oct 2025 15:18:23 -0700 Subject: [PATCH] Replace fpm with native rpmbuild for RPM package generation (#26233) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: TravisEz13 <10873629+TravisEz13@users.noreply.github.com> Co-authored-by: Travis Plunk Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../actions/test/linux-packaging/action.yml | 13 + .github/workflows/linux-ci.yml | 3 +- build.psm1 | 21 +- .../linux/package-validation.tests.ps1 | 91 +++++ tools/ci.psm1 | 26 +- tools/packaging/packaging.psm1 | 326 ++++++++++++++++-- 6 files changed, 440 insertions(+), 40 deletions(-) create mode 100644 test/packaging/linux/package-validation.tests.ps1 diff --git a/.github/actions/test/linux-packaging/action.yml b/.github/actions/test/linux-packaging/action.yml index 736bddfa7a7..b7bbdf37185 100644 --- a/.github/actions/test/linux-packaging/action.yml +++ b/.github/actions/test/linux-packaging/action.yml @@ -62,6 +62,9 @@ runs: # Create the artifacts staging directory New-Item -ItemType Directory -Path "$env:BUILD_ARTIFACTSTAGINGDIRECTORY" -Force | Out-Null + # Import packaging module to ensure RPM packaging changes are loaded + Import-Module ./build.psm1 -Force + Import-Module ./tools/packaging/packaging.psm1 -Force Import-Module ./tools/ci.psm1 Restore-PSOptions -PSOptionsPath '${{ runner.workspace }}/build/psoptions.json' $options = (Get-PSOptions) @@ -75,6 +78,16 @@ runs: Invoke-CIFinish shell: pwsh + - name: Validate Package Names + run: |- + # Run Pester tests to validate package names + Import-Module Pester -Force + $testResults = Invoke-Pester -Path ./test/packaging/linux/package-validation.tests.ps1 -PassThru + if ($testResults.FailedCount -gt 0) { + throw "Package validation tests failed" + } + shell: pwsh + - name: Upload deb packages uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index b2c6d01f8cb..78bbc62276d 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -53,6 +53,7 @@ jobs: # Set job outputs to values from filter step outputs: source: ${{ steps.filter.outputs.source }} + packagingChanged: ${{ steps.filter.outputs.packagingChanged }} steps: - name: checkout uses: actions/checkout@v5 @@ -239,7 +240,7 @@ jobs: needs: - ci_build - changes - if: ${{ needs.changes.outputs.source == 'true' }} + if: ${{ needs.changes.outputs.packagingChanged == 'true' }} runs-on: ubuntu-latest steps: - name: checkout diff --git a/build.psm1 b/build.psm1 index 98d080ecc99..88878bdd635 100644 --- a/build.psm1 +++ b/build.psm1 @@ -2401,11 +2401,24 @@ function Start-PSBootstrap { } # Install [fpm](https://github.com/jordansissel/fpm) + # Note: fpm is now only needed for DEB and macOS packages; RPM packages use rpmbuild directly if ($Scenario -eq 'Both' -or $Scenario -eq 'Package') { - Install-GlobalGem -Sudo $sudo -GemName "dotenv" -GemVersion "2.8.1" - Install-GlobalGem -Sudo $sudo -GemName "ffi" -GemVersion "1.16.3" - Install-GlobalGem -Sudo $sudo -GemName "fpm" -GemVersion "1.15.1" - Install-GlobalGem -Sudo $sudo -GemName "rexml" -GemVersion "3.2.5" + # Install fpm on Debian-based systems, macOS, and Mariner (where DEB packages are built) + if (($environment.IsLinux -and ($environment.IsDebianFamily -or $environment.IsMariner)) -or $environment.IsMacOS) { + Install-GlobalGem -Sudo $sudo -GemName "dotenv" -GemVersion "2.8.1" + Install-GlobalGem -Sudo $sudo -GemName "ffi" -GemVersion "1.16.3" + Install-GlobalGem -Sudo $sudo -GemName "fpm" -GemVersion "1.15.1" + Install-GlobalGem -Sudo $sudo -GemName "rexml" -GemVersion "3.2.5" + } + + # For RPM-based systems, ensure rpmbuild is available + if ($environment.IsLinux -and ($environment.IsRedHatFamily -or $environment.IsSUSEFamily -or $environment.IsMariner)) { + Write-Verbose -Verbose "Checking for rpmbuild..." + if (!(Get-Command rpmbuild -ErrorAction SilentlyContinue)) { + Write-Warning "rpmbuild not found. Installing rpm-build package..." + Start-NativeExecution -sb ([ScriptBlock]::Create("$sudo $PackageManager install -y rpm-build")) -IgnoreExitcode + } + } } } diff --git a/test/packaging/linux/package-validation.tests.ps1 b/test/packaging/linux/package-validation.tests.ps1 new file mode 100644 index 00000000000..241863de45d --- /dev/null +++ b/test/packaging/linux/package-validation.tests.ps1 @@ -0,0 +1,91 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +Describe "Linux Package Name Validation" { + BeforeAll { + # Determine artifacts directory (GitHub Actions or Azure DevOps) + $artifactsDir = if ($env:GITHUB_ACTIONS -eq 'true') { + "$env:GITHUB_WORKSPACE/../packages" + } else { + $env:SYSTEM_ARTIFACTSDIRECTORY + } + + if (-not $artifactsDir) { + throw "Artifacts directory not found. GITHUB_WORKSPACE or SYSTEM_ARTIFACTSDIRECTORY must be set." + } + + Write-Verbose "Artifacts directory: $artifactsDir" -Verbose + } + + Context "RPM Package Names" { + It "Should have valid RPM package names" { + $rpmPackages = Get-ChildItem -Path $artifactsDir -Recurse -Filter *.rpm -ErrorAction SilentlyContinue + + if ($rpmPackages.Count -eq 0) { + Set-ItResult -Skipped -Because "No RPM packages found in artifacts directory" + return + } + + $invalidPackages = @() + # Regex pattern for valid RPM package names. + # Breakdown: + # ^powershell\- : Starts with 'powershell-' + # (preview-|lts-)? : Optionally 'preview-' or 'lts-' + # \d+\.\d+\.\d+ : Version number (e.g., 7.6.0) + # (_[a-z]*\.\d+)? : Optional underscore, letters, dot, and digits (e.g., _alpha.1) + # -1\. : Literal '-1.' + # (preview\.\d+\.)? : Optional 'preview.' and digits, followed by a dot + # (rh|cm)\. : Either 'rh.' or 'cm.' + # (x86_64|aarch64)\.rpm$ : Architecture and file extension + $rpmPackageNamePattern = 'powershell\-(preview-|lts-)?\d+\.\d+\.\d+(_[a-z]*\.\d+)?-1\.(preview\.\d+\.)?(rh|cm)\.(x86_64|aarch64)\.rpm' + + foreach ($package in $rpmPackages) { + if ($package.Name -notmatch $rpmPackageNamePattern) { + $invalidPackages += "$($package.Name) is not a valid RPM package name" + Write-Warning "$($package.Name) is not a valid RPM package name" + } + } + + if ($invalidPackages.Count -gt 0) { + throw ($invalidPackages | Out-String) + } + + $rpmPackages.Count | Should -BeGreaterThan 0 + } + } + + Context "Tar.Gz Package Names" { + It "Should have valid tar.gz package names" { + $tarPackages = Get-ChildItem -Path $artifactsDir -Recurse -Filter *.tar.gz -ErrorAction SilentlyContinue + + if ($tarPackages.Count -eq 0) { + Set-ItResult -Skipped -Because "No tar.gz packages found in artifacts directory" + return + } + + $invalidPackages = @() + foreach ($package in $tarPackages) { + # Pattern matches: powershell-7.6.0-preview.6-linux-x64.tar.gz or powershell-7.6.0-linux-x64.tar.gz + # Also matches various runtime configurations + if ($package.Name -notmatch 'powershell-(lts-)?\d+\.\d+\.\d+\-([a-z]*.\d+\-)?(linux|osx|linux-musl)+\-(x64\-fxdependent|x64|arm32|arm64|x64\-musl-noopt\-fxdependent)\.(tar\.gz)') { + $invalidPackages += "$($package.Name) is not a valid tar.gz package name" + Write-Warning "$($package.Name) is not a valid tar.gz package name" + } + } + + if ($invalidPackages.Count -gt 0) { + throw ($invalidPackages | Out-String) + } + + $tarPackages.Count | Should -BeGreaterThan 0 + } + } + + Context "Package Existence" { + It "Should find at least one package in artifacts directory" { + $allPackages = Get-ChildItem -Path $artifactsDir -Recurse -Include *.rpm, *.tar.gz, *.deb -ErrorAction SilentlyContinue + + $allPackages.Count | Should -BeGreaterThan 0 -Because "At least one package should exist in the artifacts directory" + } + } +} diff --git a/tools/ci.psm1 b/tools/ci.psm1 index 7b13ded1811..9e95e68c843 100644 --- a/tools/ci.psm1 +++ b/tools/ci.psm1 @@ -874,16 +874,36 @@ function New-LinuxPackage $packageObj = $package } - Write-Log -message "Artifacts directory: ${env:BUILD_ARTIFACTSTAGINGDIRECTORY}" - Copy-Item $packageObj.FullName -Destination "${env:BUILD_ARTIFACTSTAGINGDIRECTORY}" -Force + # Determine artifacts directory (GitHub Actions or Azure DevOps) + $artifactsDir = if ($env:GITHUB_ACTIONS -eq 'true') { + "${env:GITHUB_WORKSPACE}/../packages" + } else { + "${env:BUILD_ARTIFACTSTAGINGDIRECTORY}" + } + + # Ensure artifacts directory exists + if (-not (Test-Path $artifactsDir)) { + New-Item -ItemType Directory -Path $artifactsDir -Force | Out-Null + } + + Write-Log -message "Artifacts directory: $artifactsDir" + Copy-Item $packageObj.FullName -Destination $artifactsDir -Force } if ($IsLinux) { + # Determine artifacts directory (GitHub Actions or Azure DevOps) + $artifactsDir = if ($env:GITHUB_ACTIONS -eq 'true') { + "${env:GITHUB_WORKSPACE}/../packages" + } else { + "${env:BUILD_ARTIFACTSTAGINGDIRECTORY}" + } + # Create and package Raspbian .tgz + # Build must be clean for Raspbian Start-PSBuild -PSModuleRestore -Clean -Runtime linux-arm -Configuration 'Release' $armPackage = Start-PSPackage @packageParams -Type tar-arm -SkipReleaseChecks - Copy-Item $armPackage -Destination "${env:BUILD_ARTIFACTSTAGINGDIRECTORY}" -Force + Copy-Item $armPackage -Destination $artifactsDir -Force } } diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 8e7fb3c4e08..894323a4b98 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -1245,40 +1245,124 @@ function New-UnixPackage { # Setup package dependencies $Dependencies = @(Get-PackageDependencies @packageDependenciesParams) - $Arguments = @() - - - $Arguments += Get-FpmArguments ` - -Name $Name ` - -Version $packageVersion ` - -Iteration $Iteration ` - -Description $Description ` - -Type $Type ` - -Dependencies $Dependencies ` - -AfterInstallScript $AfterScriptInfo.AfterInstallScript ` - -AfterRemoveScript $AfterScriptInfo.AfterRemoveScript ` - -Staging $Staging ` - -Destination $Destination ` - -ManGzipFile $ManGzipInfo.GzipFile ` - -ManDestination $ManGzipInfo.ManFile ` - -LinkInfo $Links ` - -AppsFolder $AppsFolder ` - -Distribution $DebDistro ` - -HostArchitecture $HostArchitecture ` - -ErrorAction Stop - # Build package try { - if ($PSCmdlet.ShouldProcess("Create $type package")) { - Write-Log "Creating package with fpm $Arguments..." - try { - $Output = Start-NativeExecution { fpm $Arguments } + if ($Type -eq 'rpm') { + # Use rpmbuild directly for RPM packages + if ($PSCmdlet.ShouldProcess("Create RPM package with rpmbuild")) { + Write-Log "Creating RPM package with rpmbuild..." + + # Create rpmbuild directory structure + $rpmBuildRoot = Join-Path $env:HOME "rpmbuild" + $specsDir = Join-Path $rpmBuildRoot "SPECS" + $rpmsDir = Join-Path $rpmBuildRoot "RPMS" + + New-Item -ItemType Directory -Path $specsDir -Force | Out-Null + New-Item -ItemType Directory -Path $rpmsDir -Force | Out-Null + + # Generate RPM spec file + $specContent = New-RpmSpec ` + -Name $Name ` + -Version $packageVersion ` + -Iteration $Iteration ` + -Description $Description ` + -Dependencies $Dependencies ` + -AfterInstallScript $AfterScriptInfo.AfterInstallScript ` + -AfterRemoveScript $AfterScriptInfo.AfterRemoveScript ` + -Staging $Staging ` + -Destination $Destination ` + -ManGzipFile $ManGzipInfo.GzipFile ` + -ManDestination $ManGzipInfo.ManFile ` + -LinkInfo $Links ` + -Distribution $DebDistro ` + -HostArchitecture $HostArchitecture + + $specFile = Join-Path $specsDir "$Name.spec" + $specContent | Out-File -FilePath $specFile -Encoding ascii + Write-Verbose "Generated spec file: $specFile" -Verbose + + # Log the spec file content + if ($env:GITHUB_ACTIONS -eq 'true') { + Write-Host "::group::RPM Spec File Content" + Write-Host $specContent + Write-Host "::endgroup::" + } else { + Write-Verbose "RPM Spec File Content:`n$specContent" -Verbose + } + + # Build RPM package + try { + # Use bash to properly handle rpmbuild arguments + # Add --target for cross-architecture builds + $targetArch = "" + if ($HostArchitecture -ne "x86_64" -and $HostArchitecture -ne "noarch") { + $targetArch = "--target $HostArchitecture" + } + $buildCmd = "rpmbuild -bb --quiet $targetArch --define '_topdir $rpmBuildRoot' --buildroot '$rpmBuildRoot/BUILDROOT' '$specFile'" + Write-Verbose "Running: $buildCmd" -Verbose + $Output = bash -c $buildCmd 2>&1 + $exitCode = $LASTEXITCODE + + if ($exitCode -ne 0) { + throw "rpmbuild failed with exit code $exitCode" + } + + # Find the generated RPM + $rpmFile = Get-ChildItem -Path (Join-Path $rpmsDir $HostArchitecture) -Filter "*.rpm" -ErrorAction Stop | + Sort-Object -Property LastWriteTime -Descending | + Select-Object -First 1 + + if ($rpmFile) { + # Copy RPM to current location + Copy-Item -Path $rpmFile.FullName -Destination $CurrentLocation -Force + $Output = @("Created package {:path=>""$($rpmFile.Name)""}") + } else { + throw "RPM file not found after build" + } + } + catch { + Write-Verbose -Message "!!!Handling error in rpmbuild!!!" -Verbose -ErrorAction SilentlyContinue + if ($Output) { + Write-Verbose -Message "$Output" -Verbose -ErrorAction SilentlyContinue + } + Get-Error -InputObject $_ + throw + } } - catch { - Write-Verbose -Message "!!!Handling error in FPM!!!" -Verbose -ErrorAction SilentlyContinue - Write-Verbose -Message "$Output" -Verbose -ErrorAction SilentlyContinue - Get-Error -InputObject $_ - throw + } else { + # Use fpm for DEB and macOS packages + $Arguments = @() + + $Arguments += Get-FpmArguments ` + -Name $Name ` + -Version $packageVersion ` + -Iteration $Iteration ` + -Description $Description ` + -Type $Type ` + -Dependencies $Dependencies ` + -AfterInstallScript $AfterScriptInfo.AfterInstallScript ` + -AfterRemoveScript $AfterScriptInfo.AfterRemoveScript ` + -Staging $Staging ` + -Destination $Destination ` + -ManGzipFile $ManGzipInfo.GzipFile ` + -ManDestination $ManGzipInfo.ManFile ` + -LinkInfo $Links ` + -AppsFolder $AppsFolder ` + -Distribution $DebDistro ` + -HostArchitecture $HostArchitecture ` + -ErrorAction Stop + + if ($PSCmdlet.ShouldProcess("Create $type package")) { + Write-Log "Creating package with fpm $Arguments..." + try { + $Output = Start-NativeExecution { fpm $Arguments } + } + catch { + Write-Verbose -Message "!!!Handling error in FPM!!!" -Verbose -ErrorAction SilentlyContinue + Write-Verbose -Message "$Output" -Verbose -ErrorAction SilentlyContinue + Get-Error -InputObject $_ + throw + } } } } finally { @@ -1295,6 +1379,16 @@ function New-UnixPackage { Start-NativeExecution -sb ([ScriptBlock]::Create("$sudo mv $hack_dest $symlink_dest")) -VerboseOutputOnError } } + + # Clean up rpmbuild directory if it was created + if ($Type -eq 'rpm') { + $rpmBuildRoot = Join-Path $env:HOME "rpmbuild" + if (Test-Path $rpmBuildRoot) { + Write-Verbose "Cleaning up rpmbuild directory: $rpmBuildRoot" -Verbose + Remove-Item -Path $rpmBuildRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } + if ($AfterScriptInfo.AfterInstallScript) { Remove-Item -ErrorAction 'silentlycontinue' $AfterScriptInfo.AfterInstallScript -Force } @@ -1439,6 +1533,165 @@ Class LinkInfo [string] $Destination } +function New-RpmSpec +{ + param( + [Parameter(Mandatory,HelpMessage='Package Name')] + [String]$Name, + + [Parameter(Mandatory,HelpMessage='Package Version')] + [String]$Version, + + [Parameter(Mandatory)] + [String]$Iteration, + + [Parameter(Mandatory,HelpMessage='Package description')] + [String]$Description, + + [Parameter(Mandatory,HelpMessage='Staging folder for installation files')] + [String]$Staging, + + [Parameter(Mandatory,HelpMessage='Install path on target machine')] + [String]$Destination, + + [Parameter(Mandatory,HelpMessage='The built and gzipped man file.')] + [String]$ManGzipFile, + + [Parameter(Mandatory,HelpMessage='The destination of the man file')] + [String]$ManDestination, + + [Parameter(Mandatory,HelpMessage='Symlink to powershell executable')] + [LinkInfo[]]$LinkInfo, + + [Parameter(Mandatory,HelpMessage='Packages required to install this package')] + [String[]]$Dependencies, + + [Parameter(Mandatory,HelpMessage='Script to run after the package installation.')] + [String]$AfterInstallScript, + + [Parameter(Mandatory,HelpMessage='Script to run after the package removal.')] + [String]$AfterRemoveScript, + + [String]$Distribution = 'rhel.7', + [string]$HostArchitecture + ) + + # RPM doesn't allow hyphens in version, so convert them to underscores + # e.g., "7.6.0-preview.6" becomes Version: 7.6.0_preview.6 + $rpmVersion = $Version -replace '-', '_' + + # Build Release field with distribution suffix (e.g., "1.cm" or "1.rh") + # Don't use RPM macros - build the full release string in PowerShell + $rpmRelease = "$Iteration.$Distribution" + + $specContent = @" +# RPM spec file for PowerShell +# Generated by PowerShell build system + +Name: $Name +Version: $rpmVersion +Release: $rpmRelease +Summary: PowerShell - Cross-platform automation and configuration tool/framework +License: MIT +URL: https://microsoft.com/powershell +AutoReq: no + +"@ + + # Only add BuildArch if not doing cross-architecture build + # For cross-arch builds, we'll rely on --target option + if ($HostArchitecture -eq "x86_64" -or $HostArchitecture -eq "noarch") { + $specContent += "BuildArch: $HostArchitecture`n`n" + } else { + # For cross-architecture builds, don't specify BuildArch in spec + # The --target option will handle the architecture + + # Disable automatic binary stripping for cross-arch builds + # The native /bin/strip on x86_64 cannot process ARM64 binaries and would fail with: + # "Unable to recognise the format of the input file" + # See: https://rpm-software-management.github.io/rpm/manual/macros.html + # __strip: This macro controls the command used for stripping binaries during the build process. + # /bin/true: A command that does nothing and always exits successfully, effectively bypassing the stripping process. + $specContent += "%define __strip /bin/true`n" + + # Disable debug package generation to prevent strip-related errors + # Debug packages require binary stripping which fails for cross-arch builds + # See: https://rpm-packaging-guide.github.io/#debugging + # See: https://docs.fedoraproject.org/en-US/packaging-guidelines/Debuginfo/#_useless_or_incomplete_debuginfo_packages_due_to_other_reasons + $specContent += "%global debug_package %{nil}`n`n" + } + + # Add dependencies + foreach ($dep in $Dependencies) { + $specContent += "Requires: $dep`n" + } + + $specContent += @" + +%description +$Description + +%prep +# No prep needed - files are already staged + +%build +# No build needed - binaries are pre-built + +%install +rm -rf `$RPM_BUILD_ROOT +mkdir -p `$RPM_BUILD_ROOT$Destination +mkdir -p `$RPM_BUILD_ROOT$(Split-Path -Parent $ManDestination) + +# Copy all files from staging to destination +cp -r $Staging/* `$RPM_BUILD_ROOT$Destination/ + +# Copy man page +cp $ManGzipFile `$RPM_BUILD_ROOT$ManDestination + +"@ + + # Add symlinks - we need to get the target of the temp symlink + foreach ($link in $LinkInfo) { + $linkDir = Split-Path -Parent $link.Destination + $specContent += "mkdir -p `$RPM_BUILD_ROOT$linkDir`n" + # For RPM, we copy the symlink itself (which fpm does by including it in the source) + # The symlink at $link.Source points to the actual target, so we'll copy it + # The -P flag preserves symlinks rather than copying their targets, which is critical for this operation. + $specContent += "cp -P $($link.Source) `$RPM_BUILD_ROOT$($link.Destination)`n" + } + + # Post-install script + $postInstallContent = Get-Content -Path $AfterInstallScript -Raw + $specContent += "`n%post`n" + $specContent += $postInstallContent + $specContent += "`n" + + # Post-uninstall script + $postUninstallContent = Get-Content -Path $AfterRemoveScript -Raw + $specContent += "%postun`n" + $specContent += $postUninstallContent + $specContent += "`n" + + # Files section + $specContent += "%files`n" + $specContent += "%defattr(-,root,root,-)`n" + $specContent += "$Destination/*`n" + $specContent += "$ManDestination`n" + + # Add symlinks to files + foreach ($link in $LinkInfo) { + $specContent += "$($link.Destination)`n" + } + + # Changelog with correct date format for RPM + $changelogDate = Get-Date -Format "ddd MMM dd yyyy" + $specContent += "`n%changelog`n" + $specContent += "* $changelogDate PowerShell Team - $rpmVersion-$rpmRelease`n" + $specContent += "- Automated build`n" + + return $specContent +} + function Get-FpmArguments { param( @@ -1651,7 +1904,16 @@ function Get-PackageDependencies function Test-Dependencies { - foreach ($Dependency in "fpm") { + # Note: RPM packages no longer require fpm; they use rpmbuild directly + # DEB packages still use fpm + $Dependencies = @() + + # Only check for fpm on Debian-based systems + if ($Environment.IsDebianFamily) { + $Dependencies += "fpm" + } + + foreach ($Dependency in $Dependencies) { if (!(precheck $Dependency "Package dependency '$Dependency' not found. Run Start-PSBootstrap -Scenario Package")) { # These tools are not added to the path automatically on OpenSUSE 13.2 # try adding them to the path and re-tesing first