From 859c411d05b9f3f7779395536bfac87f84271430 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 12:16:40 -0700 Subject: [PATCH 01/64] Update set-buildvariable to support isOutput --- tools/ci.psm1 | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tools/ci.psm1 b/tools/ci.psm1 index f4ed3c78945..8548c48864d 100644 --- a/tools/ci.psm1 +++ b/tools/ci.psm1 @@ -48,21 +48,26 @@ Function Set-BuildVariable [Parameter(Mandatory=$true)] [string] - $Value + $Value, + + [switch] + $IsOutput ) - if($env:TF_BUILD) - { + $IsOutputString = $IsOutput ? 'true' : 'false' + $command = "vso[task.setvariable variable=$Name;isOutput=$IsOutputString]$Value" + + # always log command to make local debugging easier + Write-Verbose -Message "sending command: $command" -Verbose + + if ($env:TF_BUILD) { # In VSTS - Write-Host "##vso[task.setvariable variable=$Name;]$Value" + Write-Host "##$command" # The variable will not show up until the next task. - # Setting in the current session for the same behavior as the CI - Set-Item env:/$name -Value $Value - } - else - { - Set-Item env:/$name -Value $Value } + + # Setting in the current session for the same behavior as the CI and to make it show up in the same task + Set-Item env:/$name -Value $Value } # Emulates running all of CI but locally From b775b25284fd7ff0efdc7a2584dbbc324a6403db Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 12:17:05 -0700 Subject: [PATCH 02/64] stop running static analysis on release branches --- .vsts-ci/misc-analysis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.vsts-ci/misc-analysis.yml b/.vsts-ci/misc-analysis.yml index e1f2d8ebfc6..b7d451ea199 100644 --- a/.vsts-ci/misc-analysis.yml +++ b/.vsts-ci/misc-analysis.yml @@ -5,14 +5,12 @@ trigger: branches: include: - master - - release* - feature* pr: branches: include: - master - - release* - feature* resources: From f0a8ccf37b06b95a94ab5d759c783fc1eebc9ab5 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 12:17:39 -0700 Subject: [PATCH 03/64] placeholder to get container info --- .vsts-ci/linux.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 9faa155f74d..081c923531e 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -59,6 +59,19 @@ stages: pool: ubuntu-20.04 jobName: linux_build displayName: linux Build + - job: getContainer + displayName: Get Container + steps: + - checkout: self + clean: true + - pwsh: | + Import-Module ./tools/ci.psm1 + $containerName = 'mcr.microsoft.com/powershell/test-deps/preview-mariner-1.0' + Set-BuildVariable -Name containerName -Value $containerName -IsOutput + $containerName = 'mariner1' + Set-BuildVariable -Name containerBuildName -Value $containerName -IsOutput + name: getContainer + displayName: Get Container - stage: TestLinux displayName: Test for Linux From 1ee59cc19ace6c0148901d9207cbab8671448c4e Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 12:18:16 -0700 Subject: [PATCH 04/64] remove codecov --- .vsts-ci/linux.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 081c923531e..33e7c229db4 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -115,17 +115,3 @@ stages: - template: linux/templates/packaging.yml parameters: pool: ubuntu-20.04 - -- stage: CodeCovTestPackage - displayName: CodeCoverage and Test Packages - dependsOn: [] # by specifying an empty array, this stage doesn't depend on the stage before it - jobs: - - job: CodeCovTestPackage - displayName: CodeCoverage and Test Packages - pool: - vmImage: ubuntu-20.04 - steps: - - pwsh: | - Import-Module .\tools\ci.psm1 - New-CodeCoverageAndTestPackage - displayName: CodeCoverage and Test Package From 7f1564d03b87fad2388d3807af7f3104dcec4045 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 12:18:47 -0700 Subject: [PATCH 05/64] Add one container job --- .vsts-ci/linux.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 33e7c229db4..172d68015ad 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -75,6 +75,7 @@ stages: - stage: TestLinux displayName: Test for Linux + dependsOn: [BuildLinux] jobs: - template: templates/nix-test.yml parameters: @@ -108,6 +109,14 @@ stages: parameters: pool: ubuntu-20.04 + - template: templates/nix-test.yml + parameters: + name: $[stageDependencies.BuildLinux.getContainer.outputs['getContainer.containerBuildName']] + pool: ubuntu-20.04 + purpose: UnelevatedPesterTests + tagSet: CI + containerName: $[stageDependencies.BuildLinux.getContainer.outputs['getContainer.containerName']] + - stage: PackageLinux displayName: Package Linux dependsOn: ["BuildLinux"] From 215e05fea0fd7645e97dce555b1065d8e9f6bd56 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 12:19:22 -0700 Subject: [PATCH 06/64] make *nix test support containers --- .vsts-ci/templates/nix-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index f2f04d9dd5c..ad5e5b560f4 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -4,6 +4,7 @@ parameters: purpose: '' tagSet: 'CI' name: 'mac' + containerName: '' jobs: - job: ${{ parameters.name }}_test_${{ parameters.purpose }}_${{ parameters.tagSet }} @@ -12,6 +13,9 @@ jobs: pool: vmImage: ${{ parameters.pool }} + ${{ if and(ne(parameters.containerName, ''), not(contains(parameters.containerName, 'alpine'))) }}: + container: ${{ parameters.containerName }} + displayName: ${{ parameters.name }} Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }} steps: From 8e189d508f25aaf74cae332685df0df6118dc757 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 12:20:25 -0700 Subject: [PATCH 07/64] rename existing jobs to Ubuntu --- .vsts-ci/linux.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 172d68015ad..8343b64cb36 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -79,28 +79,28 @@ stages: jobs: - template: templates/nix-test.yml parameters: - name: Linux + name: Ubuntu pool: ubuntu-20.04 purpose: UnelevatedPesterTests tagSet: CI - template: templates/nix-test.yml parameters: - name: Linux + name: Ubuntu pool: ubuntu-20.04 purpose: ElevatedPesterTests tagSet: CI - template: templates/nix-test.yml parameters: - name: Linux + name: Ubuntu pool: ubuntu-20.04 purpose: UnelevatedPesterTests tagSet: Others - template: templates/nix-test.yml parameters: - name: Linux + name: Ubuntu pool: ubuntu-20.04 purpose: ElevatedPesterTests tagSet: Others From f712d18870975fff7bfe346cabf49f44997174f1 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 12:20:45 -0700 Subject: [PATCH 08/64] move xUnit verify to end --- .vsts-ci/linux.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 8343b64cb36..ff335fb51ff 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -105,10 +105,6 @@ stages: purpose: ElevatedPesterTests tagSet: Others - - template: templates/verify-xunit.yml - parameters: - pool: ubuntu-20.04 - - template: templates/nix-test.yml parameters: name: $[stageDependencies.BuildLinux.getContainer.outputs['getContainer.containerBuildName']] @@ -117,6 +113,10 @@ stages: tagSet: CI containerName: $[stageDependencies.BuildLinux.getContainer.outputs['getContainer.containerName']] + - template: templates/verify-xunit.yml + parameters: + pool: ubuntu-20.04 + - stage: PackageLinux displayName: Package Linux dependsOn: ["BuildLinux"] From 38d6e8d72a059a003cc56d18b72394220868a6fb Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 12:23:49 -0700 Subject: [PATCH 09/64] don't use an output for jobname --- .vsts-ci/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index ff335fb51ff..1c55dc6ff01 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -107,7 +107,7 @@ stages: - template: templates/nix-test.yml parameters: - name: $[stageDependencies.BuildLinux.getContainer.outputs['getContainer.containerBuildName']] + name: container pool: ubuntu-20.04 purpose: UnelevatedPesterTests tagSet: CI From 191faba08737def2affcdb2b40ef983f0530b0be Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 12:25:37 -0700 Subject: [PATCH 10/64] set vmImage to get container info --- .vsts-ci/linux.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 1c55dc6ff01..a2be7f6ee7f 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -61,6 +61,8 @@ stages: displayName: linux Build - job: getContainer displayName: Get Container + pool: + vmImage: ubuntu-20.04 steps: - checkout: self clean: true From fc7059a68400544486bd0f12e3a1f0624422404b Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 12:43:58 -0700 Subject: [PATCH 11/64] try other format --- .vsts-ci/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index a2be7f6ee7f..24c7020fc45 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -113,7 +113,7 @@ stages: pool: ubuntu-20.04 purpose: UnelevatedPesterTests tagSet: CI - containerName: $[stageDependencies.BuildLinux.getContainer.outputs['getContainer.containerName']] + containerName: $[ stageDependencies.BuildLinux.outputs['getContainer.getContainer.containerName']] - template: templates/verify-xunit.yml parameters: From 10c0ab1228969e929f706560acce464cf6447105 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 13:03:12 -0700 Subject: [PATCH 12/64] split into two stages --- .vsts-ci/linux.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 24c7020fc45..5635676c1b2 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -75,8 +75,8 @@ stages: name: getContainer displayName: Get Container -- stage: TestLinux - displayName: Test for Linux +- stage: TestUbuntu + displayName: Test for Ubuntu dependsOn: [BuildLinux] jobs: - template: templates/nix-test.yml @@ -107,6 +107,14 @@ stages: purpose: ElevatedPesterTests tagSet: Others + - template: templates/verify-xunit.yml + parameters: + pool: ubuntu-20.04 + +- stage: TestContainer + displayName: Test in a container + dependsOn: [BuildLinux] + jobs: - template: templates/nix-test.yml parameters: name: container @@ -115,9 +123,6 @@ stages: tagSet: CI containerName: $[ stageDependencies.BuildLinux.outputs['getContainer.getContainer.containerName']] - - template: templates/verify-xunit.yml - parameters: - pool: ubuntu-20.04 - stage: PackageLinux displayName: Package Linux From a6ed36981502b45e2564bedde44dc8a02e267091 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 13:03:41 -0700 Subject: [PATCH 13/64] try using stage variables --- .vsts-ci/linux.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 5635676c1b2..fdc3fc2b8ca 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -114,6 +114,9 @@ stages: - stage: TestContainer displayName: Test in a container dependsOn: [BuildLinux] + variables: + containerName: $[ dependencies.BuildLinux.outputs['getContainer.getContainer.containerName']] + containerBuildName: $[ dependencies.BuildLinux.outputs['getContainer.getContainer.containerBuildName']] jobs: - template: templates/nix-test.yml parameters: @@ -121,8 +124,7 @@ stages: pool: ubuntu-20.04 purpose: UnelevatedPesterTests tagSet: CI - containerName: $[ stageDependencies.BuildLinux.outputs['getContainer.getContainer.containerName']] - + containerName: $(containerName) - stage: PackageLinux displayName: Package Linux From 7ad100575444753b90b65fcebe326521d4902b04 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 13:42:53 -0700 Subject: [PATCH 14/64] add conditon --- .vsts-ci/linux.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index fdc3fc2b8ca..9bdc5249e0f 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -59,7 +59,7 @@ stages: pool: ubuntu-20.04 jobName: linux_build displayName: linux Build - - job: getContainer + - job: getContainerJob displayName: Get Container pool: vmImage: ubuntu-20.04 @@ -72,7 +72,7 @@ stages: Set-BuildVariable -Name containerName -Value $containerName -IsOutput $containerName = 'mariner1' Set-BuildVariable -Name containerBuildName -Value $containerName -IsOutput - name: getContainer + name: getContainerTask displayName: Get Container - stage: TestUbuntu @@ -114,9 +114,10 @@ stages: - stage: TestContainer displayName: Test in a container dependsOn: [BuildLinux] + condition: and(succeeded(), ne(dependencies.BuildLinux.outputs['getContainerJob.getContainerTask.containerName'],'')) variables: - containerName: $[ dependencies.BuildLinux.outputs['getContainer.getContainer.containerName']] - containerBuildName: $[ dependencies.BuildLinux.outputs['getContainer.getContainer.containerBuildName']] + containerName: $[ dependencies.BuildLinux.outputs['getContainerJob.getContainerTask.containerName']] + containerBuildName: $[ dependencies.BuildLinux.outputs['getContainerJob.getContainerTask.containerBuildName']] jobs: - template: templates/nix-test.yml parameters: From b017cc6cbf749471b8f9983d4c24f3ace851cb85 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 14:00:23 -0700 Subject: [PATCH 15/64] add type to names --- .vsts-ci/linux.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 9bdc5249e0f..c0d53485277 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -51,7 +51,7 @@ resources: clean: true stages: -- stage: BuildLinux +- stage: BuildLinuxStage displayName: Build for Linux jobs: - template: templates/ci-build.yml @@ -77,7 +77,7 @@ stages: - stage: TestUbuntu displayName: Test for Ubuntu - dependsOn: [BuildLinux] + dependsOn: [BuildLinuxStage] jobs: - template: templates/nix-test.yml parameters: @@ -113,11 +113,11 @@ stages: - stage: TestContainer displayName: Test in a container - dependsOn: [BuildLinux] - condition: and(succeeded(), ne(dependencies.BuildLinux.outputs['getContainerJob.getContainerTask.containerName'],'')) + dependsOn: [BuildLinuxStage] + condition: and(succeeded(), ne(dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerTask.containerName'],'')) variables: - containerName: $[ dependencies.BuildLinux.outputs['getContainerJob.getContainerTask.containerName']] - containerBuildName: $[ dependencies.BuildLinux.outputs['getContainerJob.getContainerTask.containerBuildName']] + containerName: $[ dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerTask.containerName']] + containerBuildName: $[ dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerTask.containerBuildName']] jobs: - template: templates/nix-test.yml parameters: @@ -129,7 +129,7 @@ stages: - stage: PackageLinux displayName: Package Linux - dependsOn: ["BuildLinux"] + dependsOn: ["BuildLinuxStage"] jobs: - template: linux/templates/packaging.yml parameters: From 16b1369479d63d47191fa60964f374bd7b59bd5f Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 14:10:43 -0700 Subject: [PATCH 16/64] hard code image --- .vsts-ci/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index c0d53485277..6cee3d4e6ba 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -125,7 +125,7 @@ stages: pool: ubuntu-20.04 purpose: UnelevatedPesterTests tagSet: CI - containerName: $(containerName) + containerName: mcr.microsoft.com/powershell/test-deps/preview-mariner-1.0 - stage: PackageLinux displayName: Package Linux From 8323bde714d1610c48ba81cb68eee3b11a079bbe Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 14:24:24 -0700 Subject: [PATCH 17/64] fix container name --- .vsts-ci/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 6cee3d4e6ba..6bcb76b678f 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -125,7 +125,7 @@ stages: pool: ubuntu-20.04 purpose: UnelevatedPesterTests tagSet: CI - containerName: mcr.microsoft.com/powershell/test-deps/preview-mariner-1.0 + containerName: mcr.microsoft.com/powershell/test-deps:preview-mariner-1.0 - stage: PackageLinux displayName: Package Linux From c4b8e34fec084919fbe861c3740b78774cd06c26 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 14:41:03 -0700 Subject: [PATCH 18/64] add unzip --- .vsts-ci/templates/nix-test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index ad5e5b560f4..07eac8a01ea 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -42,6 +42,13 @@ jobs: Invoke-CIInstall -SkipUser displayName: Bootstrap + - pwsh: | + if (Get-Command tdnf -errorAction SilentlyContinue) { + tdnf install -y unzip + } + displayName: 'Install unzip using tdnf' + continueOnError: true + - task: ExtractFiles@1 displayName: 'Extract Build ZIP' inputs: From 287aaf6d053ff5ec304d2b292e504784eae131c2 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 14:41:25 -0700 Subject: [PATCH 19/64] get image list from docker repo --- .vsts-ci/linux.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 6bcb76b678f..1ccb27627f4 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -67,11 +67,18 @@ stages: - checkout: self clean: true - pwsh: | + git clone https://github.com/powerhell/powershell-docker.git + $matrix = ./powershell-docker/build.ps1 -GenerateMatrixJson -FullJson -Channel preview | ConvertFrom-Json + $linuxImages = $matrix.preview | + Where-Object {$_.osversion -notlike '*windows*' -and $_.osversion -notlike '*nano*' -and $_.osversion -notlike 'alpine*' -and $_.osversion -notlike '*arm*'} | + Select-Object -First 100 JobName, Taglist | + Sort-Object -property JobName + $marinerImage = $linuxImages | where-object {$_.JobName -like 'mariner*'} Import-Module ./tools/ci.psm1 - $containerName = 'mcr.microsoft.com/powershell/test-deps/preview-mariner-1.0' + $tag = $marinerImage.Taglist -split ';' | select-object -first 1 + $containerName = "mcr.microsoft.com/powershell/test-deps:$tag" Set-BuildVariable -Name containerName -Value $containerName -IsOutput - $containerName = 'mariner1' - Set-BuildVariable -Name containerBuildName -Value $containerName -IsOutput + Set-BuildVariable -Name containerBuildName -Value $marinerImage.JobName -IsOutput name: getContainerTask displayName: Get Container From 3cc5e29908dd5174abb5539a387558317d09d5a5 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 14:48:32 -0700 Subject: [PATCH 20/64] use azure devops checkout for docker --- .vsts-ci/linux.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 1ccb27627f4..c681039a2c3 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -47,8 +47,12 @@ variables: __SuppressAnsiEscapeSequences: 1 resources: -- repo: self - clean: true + repositories: + - repository: Docker + type: github + endpoint: PowerShell + name: PowerShell/PowerShell-Docker + ref: master stages: - stage: BuildLinuxStage @@ -66,15 +70,19 @@ stages: steps: - checkout: self clean: true + + - checkout: Docker + clean: true + - pwsh: | git clone https://github.com/powerhell/powershell-docker.git - $matrix = ./powershell-docker/build.ps1 -GenerateMatrixJson -FullJson -Channel preview | ConvertFrom-Json + $matrix = ./Docker/build.ps1 -GenerateMatrixJson -FullJson -Channel preview | ConvertFrom-Json $linuxImages = $matrix.preview | Where-Object {$_.osversion -notlike '*windows*' -and $_.osversion -notlike '*nano*' -and $_.osversion -notlike 'alpine*' -and $_.osversion -notlike '*arm*'} | Select-Object -First 100 JobName, Taglist | Sort-Object -property JobName $marinerImage = $linuxImages | where-object {$_.JobName -like 'mariner*'} - Import-Module ./tools/ci.psm1 + Import-Module ./PowerShell/tools/ci.psm1 $tag = $marinerImage.Taglist -split ';' | select-object -first 1 $containerName = "mcr.microsoft.com/powershell/test-deps:$tag" Set-BuildVariable -Name containerName -Value $containerName -IsOutput From 7fcd5104a33c4acf7138306877ac392199e9c8c8 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 14:49:05 -0700 Subject: [PATCH 21/64] continue on error for now --- .vsts-ci/linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index c681039a2c3..f8048f9c0e8 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -89,6 +89,7 @@ stages: Set-BuildVariable -Name containerBuildName -Value $marinerImage.JobName -IsOutput name: getContainerTask displayName: Get Container + continueOnError: true - stage: TestUbuntu displayName: Test for Ubuntu From cc286519cc0a7006183afdf7309d37dcddec8fe8 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 14:49:49 -0700 Subject: [PATCH 22/64] add another stage that tries to use the container --- .vsts-ci/linux.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index f8048f9c0e8..3d1bf3ae5cd 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -142,6 +142,13 @@ stages: purpose: UnelevatedPesterTests tagSet: CI containerName: mcr.microsoft.com/powershell/test-deps:preview-mariner-1.0 + - template: templates/nix-test.yml + parameters: + name: container + pool: ubuntu-20.04 + purpose: ElevatedPesterTests + tagSet: CI + containerName: $(containerName) - stage: PackageLinux displayName: Package Linux From 55380bd98d64086628e9f404cc05f402892ca8cb Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 15:03:14 -0700 Subject: [PATCH 23/64] finish move to azDevOps checkout --- .vsts-ci/linux.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 3d1bf3ae5cd..6b67a84b471 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -75,8 +75,7 @@ stages: clean: true - pwsh: | - git clone https://github.com/powerhell/powershell-docker.git - $matrix = ./Docker/build.ps1 -GenerateMatrixJson -FullJson -Channel preview | ConvertFrom-Json + $matrix = ./PowerShell-Docker/build.ps1 -GenerateMatrixJson -FullJson -Channel preview | ConvertFrom-Json $linuxImages = $matrix.preview | Where-Object {$_.osversion -notlike '*windows*' -and $_.osversion -notlike '*nano*' -and $_.osversion -notlike 'alpine*' -and $_.osversion -notlike '*arm*'} | Select-Object -First 100 JobName, Taglist | From 64724bf301ef8fb9a92932b0f8c1476383f6f9e9 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 15:12:22 -0700 Subject: [PATCH 24/64] use sudo to install --- .vsts-ci/templates/nix-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index 07eac8a01ea..736eb72cec6 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -44,7 +44,7 @@ jobs: - pwsh: | if (Get-Command tdnf -errorAction SilentlyContinue) { - tdnf install -y unzip + sudo tdnf install -y unzip } displayName: 'Install unzip using tdnf' continueOnError: true From 12923a1cb8002586d7a842de9ed7e348bb0e5970 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 15:12:42 -0700 Subject: [PATCH 25/64] put job name twice --- .vsts-ci/linux.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 6b67a84b471..5b0a3e8c528 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -129,10 +129,10 @@ stages: - stage: TestContainer displayName: Test in a container dependsOn: [BuildLinuxStage] - condition: and(succeeded(), ne(dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerTask.containerName'],'')) + condition: and(succeeded(), ne(dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerJob.getContainerTask.containerName'],'')) variables: - containerName: $[ dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerTask.containerName']] - containerBuildName: $[ dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerTask.containerBuildName']] + containerName: $[ dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerJob.getContainerTask.containerName']] + containerBuildName: $[ dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerJob.getContainerTask.containerBuildName']] jobs: - template: templates/nix-test.yml parameters: From a67f0c391c9da141d2e6265ff5a214ca438e2a63 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 15:19:48 -0700 Subject: [PATCH 26/64] comment out condition --- .vsts-ci/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 5b0a3e8c528..0cb34f9959f 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -129,7 +129,7 @@ stages: - stage: TestContainer displayName: Test in a container dependsOn: [BuildLinuxStage] - condition: and(succeeded(), ne(dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerJob.getContainerTask.containerName'],'')) + #condition: and(succeeded(), ne(dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerJob.getContainerTask.containerName'],'')) variables: containerName: $[ dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerJob.getContainerTask.containerName']] containerBuildName: $[ dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerJob.getContainerTask.containerBuildName']] From 10d86986b5d5f948bfa6c6e0dac7f572e159fa18 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 15:40:09 -0700 Subject: [PATCH 27/64] ignore that we may not be able to build --- tools/ci.psm1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci.psm1 b/tools/ci.psm1 index 8548c48864d..af1246f5948 100644 --- a/tools/ci.psm1 +++ b/tools/ci.psm1 @@ -16,7 +16,8 @@ if(Test-Path $dotNetPath) } # import build into the global scope so it can be used by packaging -Import-Module (Join-Path $repoRoot 'build.psm1') -Verbose -Scope Global +# argumentList $true says ignore tha we may not be able to build +Import-Module (Join-Path $repoRoot 'build.psm1') -Verbose -Scope Global -ArgumentList $true Import-Module (Join-Path $repoRoot 'tools\packaging') -Verbose -Scope Global # import the windows specific functcion only in Windows PowerShell or on Windows From b3a8ae7e82ea00531ce0f3a518bd288fc316eb0e Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 15:49:37 -0700 Subject: [PATCH 28/64] move finding the container into the stage --- .vsts-ci/linux.yml | 58 +++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 0cb34f9959f..f2ad69063dd 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -63,32 +63,6 @@ stages: pool: ubuntu-20.04 jobName: linux_build displayName: linux Build - - job: getContainerJob - displayName: Get Container - pool: - vmImage: ubuntu-20.04 - steps: - - checkout: self - clean: true - - - checkout: Docker - clean: true - - - pwsh: | - $matrix = ./PowerShell-Docker/build.ps1 -GenerateMatrixJson -FullJson -Channel preview | ConvertFrom-Json - $linuxImages = $matrix.preview | - Where-Object {$_.osversion -notlike '*windows*' -and $_.osversion -notlike '*nano*' -and $_.osversion -notlike 'alpine*' -and $_.osversion -notlike '*arm*'} | - Select-Object -First 100 JobName, Taglist | - Sort-Object -property JobName - $marinerImage = $linuxImages | where-object {$_.JobName -like 'mariner*'} - Import-Module ./PowerShell/tools/ci.psm1 - $tag = $marinerImage.Taglist -split ';' | select-object -first 1 - $containerName = "mcr.microsoft.com/powershell/test-deps:$tag" - Set-BuildVariable -Name containerName -Value $containerName -IsOutput - Set-BuildVariable -Name containerBuildName -Value $marinerImage.JobName -IsOutput - name: getContainerTask - displayName: Get Container - continueOnError: true - stage: TestUbuntu displayName: Test for Ubuntu @@ -134,6 +108,33 @@ stages: containerName: $[ dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerJob.getContainerTask.containerName']] containerBuildName: $[ dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerJob.getContainerTask.containerBuildName']] jobs: + - job: getContainerJob + displayName: Get Container + pool: + vmImage: ubuntu-20.04 + steps: + - checkout: self + clean: true + + - checkout: Docker + clean: true + + - pwsh: | + $matrix = ./PowerShell-Docker/build.ps1 -GenerateMatrixJson -FullJson -Channel preview | ConvertFrom-Json + $linuxImages = $matrix.preview | + Where-Object {$_.osversion -notlike '*windows*' -and $_.osversion -notlike '*nano*' -and $_.osversion -notlike 'alpine*' -and $_.osversion -notlike '*arm*'} | + Select-Object -First 100 JobName, Taglist | + Sort-Object -property JobName + $marinerImage = $linuxImages | where-object {$_.JobName -like 'mariner*'} + Import-Module ./PowerShell/tools/ci.psm1 + $tag = $marinerImage.Taglist -split ';' | select-object -first 1 + $containerName = "mcr.microsoft.com/powershell/test-deps:$tag" + Set-BuildVariable -Name containerName -Value $containerName -IsOutput + Set-BuildVariable -Name containerBuildName -Value $marinerImage.JobName -IsOutput + name: getContainerTask + displayName: Get Container + continueOnError: true + - template: templates/nix-test.yml parameters: name: container @@ -141,13 +142,16 @@ stages: purpose: UnelevatedPesterTests tagSet: CI containerName: mcr.microsoft.com/powershell/test-deps:preview-mariner-1.0 + - template: templates/nix-test.yml parameters: name: container pool: ubuntu-20.04 purpose: ElevatedPesterTests tagSet: CI - containerName: $(containerName) + containerName: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName']] + parentJobs: + - getContainerTask - stage: PackageLinux displayName: Package Linux From f57dfa304814e0e36d63db94e5644be31199e9eb Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 15:52:30 -0700 Subject: [PATCH 29/64] fix depenecy --- .vsts-ci/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index f2ad69063dd..736afa443fe 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -151,7 +151,7 @@ stages: tagSet: CI containerName: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName']] parentJobs: - - getContainerTask + - getContainerJob - stage: PackageLinux displayName: Package Linux From ef571810d67479a346aead13f0e62e850506fafe Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 16:04:34 -0700 Subject: [PATCH 30/64] fix mariner bootstrap --- .vsts-ci/templates/nix-test.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index 736eb72cec6..817bdf39367 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -37,18 +37,18 @@ jobs: displayName: 'Capture Artifacts Directory' continueOnError: true - - pwsh: | - Import-Module .\tools\ci.psm1 - Invoke-CIInstall -SkipUser - displayName: Bootstrap - - pwsh: | if (Get-Command tdnf -errorAction SilentlyContinue) { - sudo tdnf install -y unzip + sudo tdnf install -y unzip git } - displayName: 'Install unzip using tdnf' + displayName: 'Install unzip, git using tdnf' continueOnError: true + - pwsh: | + Import-Module .\tools\ci.psm1 + Invoke-CIInstall -SkipUser + displayName: Bootstrap + - task: ExtractFiles@1 displayName: 'Extract Build ZIP' inputs: From b4563f16462665f0f72469aff174fd071ea19bc4 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 16:09:02 -0700 Subject: [PATCH 31/64] try docker way of evaluating dependecies --- .vsts-ci/linux.yml | 2 +- .vsts-ci/templates/nix-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 736afa443fe..a2486047935 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -149,7 +149,7 @@ stages: pool: ubuntu-20.04 purpose: ElevatedPesterTests tagSet: CI - containerName: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName']] + containerName: dependencies.getContainerJob.outputs['getContainerTask.containerName'] parentJobs: - getContainerJob diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index 817bdf39367..e307d0806f6 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -14,7 +14,7 @@ jobs: vmImage: ${{ parameters.pool }} ${{ if and(ne(parameters.containerName, ''), not(contains(parameters.containerName, 'alpine'))) }}: - container: ${{ parameters.containerName }} + container: $[ ${{ parameters.containerName }} ] displayName: ${{ parameters.name }} Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }} From 9f376adc33281eff704f299b81de3418d16cd16b Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 16:12:05 -0700 Subject: [PATCH 32/64] trying random stuff --- .vsts-ci/templates/nix-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index e307d0806f6..817bdf39367 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -14,7 +14,7 @@ jobs: vmImage: ${{ parameters.pool }} ${{ if and(ne(parameters.containerName, ''), not(contains(parameters.containerName, 'alpine'))) }}: - container: $[ ${{ parameters.containerName }} ] + container: ${{ parameters.containerName }} displayName: ${{ parameters.name }} Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }} From 6b67596be33b4ea4ad91d760ac442d06f7dd6aca Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 16:21:18 -0700 Subject: [PATCH 33/64] try using a variable --- .vsts-ci/templates/nix-test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index 817bdf39367..ec7acf5eff7 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -8,13 +8,15 @@ parameters: jobs: - job: ${{ parameters.name }}_test_${{ parameters.purpose }}_${{ parameters.tagSet }} + variables: + - containerName: $[ ${{ parameters.containerName }} ] dependsOn: ${{ parameters.parentJobs }} pool: vmImage: ${{ parameters.pool }} - ${{ if and(ne(parameters.containerName, ''), not(contains(parameters.containerName, 'alpine'))) }}: - container: ${{ parameters.containerName }} + ${{ if and(ne(variables.containerName, ''), not(contains(variables.containerName, 'alpine'))) }}: + container: ${{ variables.containerName }} displayName: ${{ parameters.name }} Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }} From 82d50347a2e7418068fcd3fcb12d266a700a40ed Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 16:29:32 -0700 Subject: [PATCH 34/64] fix parameter syntax --- .vsts-ci/templates/nix-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index ec7acf5eff7..daa5bce2683 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -9,7 +9,8 @@ parameters: jobs: - job: ${{ parameters.name }}_test_${{ parameters.purpose }}_${{ parameters.tagSet }} variables: - - containerName: $[ ${{ parameters.containerName }} ] + - name: containerName + value: $[ ${{ parameters.containerName }} ] dependsOn: ${{ parameters.parentJobs }} pool: From 0d058518a1c56c15aa2272ba4a020e5696cddcca Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 16:34:44 -0700 Subject: [PATCH 35/64] allow either a static name or an expression --- .vsts-ci/templates/nix-test.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index daa5bce2683..9c4969d35ac 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -5,19 +5,23 @@ parameters: tagSet: 'CI' name: 'mac' containerName: '' + containerNameExpression: '' jobs: - job: ${{ parameters.name }}_test_${{ parameters.purpose }}_${{ parameters.tagSet }} - variables: - - name: containerName - value: $[ ${{ parameters.containerName }} ] + ${{ if ne(variables.containerNameExpression, '') }}: + variables: + - name: containerName + value: $[ ${{ parameters.containerNameExpression }} ] dependsOn: ${{ parameters.parentJobs }} pool: vmImage: ${{ parameters.pool }} + ${{ if and(ne(parameters.containerName, ''), not(contains(parameters.containerName, 'alpine'))) }}: + container: ${{ parameters.containerName }} ${{ if and(ne(variables.containerName, ''), not(contains(variables.containerName, 'alpine'))) }}: - container: ${{ variables.containerName }} + container: ${{ parameters.containerName }} displayName: ${{ parameters.name }} Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }} From 35226cc40b036c52448f65107a933f15974563c9 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 16:35:15 -0700 Subject: [PATCH 36/64] use expression param --- .vsts-ci/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index a2486047935..44bd3edbd95 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -149,7 +149,7 @@ stages: pool: ubuntu-20.04 purpose: ElevatedPesterTests tagSet: CI - containerName: dependencies.getContainerJob.outputs['getContainerTask.containerName'] + containerNameExpression: dependencies.getContainerJob.outputs['getContainerTask.containerName'] parentJobs: - getContainerJob From 8b385ef7a7f86ca6693edf84e158fb6c358d1ee0 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 16:48:57 -0700 Subject: [PATCH 37/64] delete unused variables --- .vsts-ci/linux.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 44bd3edbd95..c441864a194 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -103,10 +103,6 @@ stages: - stage: TestContainer displayName: Test in a container dependsOn: [BuildLinuxStage] - #condition: and(succeeded(), ne(dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerJob.getContainerTask.containerName'],'')) - variables: - containerName: $[ dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerJob.getContainerTask.containerName']] - containerBuildName: $[ dependencies.BuildLinuxStage.outputs['getContainerJob.getContainerJob.getContainerTask.containerBuildName']] jobs: - job: getContainerJob displayName: Get Container From 06a600188dd00be503556031911400ef2d6d15af Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 16:49:12 -0700 Subject: [PATCH 38/64] misc fixes --- .vsts-ci/templates/nix-test.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index 9c4969d35ac..93fe2962aea 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -9,10 +9,12 @@ parameters: jobs: - job: ${{ parameters.name }}_test_${{ parameters.purpose }}_${{ parameters.tagSet }} - ${{ if ne(variables.containerNameExpression, '') }}: + ${{ if ne(parameters.containerNameExpression, '') }}: variables: - name: containerName value: $[ ${{ parameters.containerNameExpression }} ] + - name: variableDefined + value: 'true' dependsOn: ${{ parameters.parentJobs }} pool: @@ -20,8 +22,8 @@ jobs: ${{ if and(ne(parameters.containerName, ''), not(contains(parameters.containerName, 'alpine'))) }}: container: ${{ parameters.containerName }} - ${{ if and(ne(variables.containerName, ''), not(contains(variables.containerName, 'alpine'))) }}: - container: ${{ parameters.containerName }} + ${{ if and(ne(parameters.containerNameExpression, ''), not(contains(variables.containerName, 'alpine'))) }}: + container: ${{ variables.containerName }} displayName: ${{ parameters.name }} Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }} From 27e103a0a48d0d62e134a0417dda02fea69335b2 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 17:00:24 -0700 Subject: [PATCH 39/64] try to eliminate expression parse error --- .vsts-ci/templates/nix-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index 93fe2962aea..0433767bbf4 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -22,7 +22,7 @@ jobs: ${{ if and(ne(parameters.containerName, ''), not(contains(parameters.containerName, 'alpine'))) }}: container: ${{ parameters.containerName }} - ${{ if and(ne(parameters.containerNameExpression, ''), not(contains(variables.containerName, 'alpine'))) }}: + ${{ if and(ne(variables.containerName, ''), not(contains(variables.containerName, 'alpine'))) }}: container: ${{ variables.containerName }} displayName: ${{ parameters.name }} Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }} From ac65b3845656827e0c67af507d9c8cf475c534e6 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 17:17:11 -0700 Subject: [PATCH 40/64] add a default expression --- .vsts-ci/templates/nix-test.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index 0433767bbf4..0ffba3e6c5d 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -5,16 +5,16 @@ parameters: tagSet: 'CI' name: 'mac' containerName: '' - containerNameExpression: '' + containerNameExpression: | + '' jobs: - job: ${{ parameters.name }}_test_${{ parameters.purpose }}_${{ parameters.tagSet }} - ${{ if ne(parameters.containerNameExpression, '') }}: - variables: - - name: containerName - value: $[ ${{ parameters.containerNameExpression }} ] - - name: variableDefined - value: 'true' + variables: + - name: containerName + value: $[ ${{ parameters.containerNameExpression }} ] + - name: variableDefined + value: 'true' dependsOn: ${{ parameters.parentJobs }} pool: From 969ad8277236c4cc5683ddda4438208f6799de8c Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 17:20:30 -0700 Subject: [PATCH 41/64] default to the static variable --- .vsts-ci/templates/nix-test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index 0ffba3e6c5d..d21f693988e 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -6,7 +6,7 @@ parameters: name: 'mac' containerName: '' containerNameExpression: | - '' + 'parameters.containerName' jobs: - job: ${{ parameters.name }}_test_${{ parameters.purpose }}_${{ parameters.tagSet }} @@ -20,8 +20,6 @@ jobs: pool: vmImage: ${{ parameters.pool }} - ${{ if and(ne(parameters.containerName, ''), not(contains(parameters.containerName, 'alpine'))) }}: - container: ${{ parameters.containerName }} ${{ if and(ne(variables.containerName, ''), not(contains(variables.containerName, 'alpine'))) }}: container: ${{ variables.containerName }} From e2ca78e2be04fcb6d3f714f0f16e813a547afb76 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 17:31:08 -0700 Subject: [PATCH 42/64] fix default expression --- .vsts-ci/templates/nix-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index d21f693988e..73c8baefd3a 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -6,7 +6,7 @@ parameters: name: 'mac' containerName: '' containerNameExpression: | - 'parameters.containerName' + parameters.containerName jobs: - job: ${{ parameters.name }}_test_${{ parameters.purpose }}_${{ parameters.tagSet }} From 3425bff3623a7d3da9b8986a9a834cb204b0396d Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 17:40:22 -0700 Subject: [PATCH 43/64] move expression into template - paring but then get unrecognized error --- .vsts-ci/linux.yml | 1 - .vsts-ci/templates/nix-test.yml | 19 ++++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index c441864a194..7ea78255661 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -145,7 +145,6 @@ stages: pool: ubuntu-20.04 purpose: ElevatedPesterTests tagSet: CI - containerNameExpression: dependencies.getContainerJob.outputs['getContainerTask.containerName'] parentJobs: - getContainerJob diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index 73c8baefd3a..3c5ff3d227e 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -5,16 +5,21 @@ parameters: tagSet: 'CI' name: 'mac' containerName: '' - containerNameExpression: | - parameters.containerName jobs: - job: ${{ parameters.name }}_test_${{ parameters.purpose }}_${{ parameters.tagSet }} - variables: - - name: containerName - value: $[ ${{ parameters.containerNameExpression }} ] - - name: variableDefined - value: 'true' + ${{ if ne(parameters.containerName, '') }}: + variables: + - name: containerName + value: ${{ parameters.containerName }} + - name: variableDefined + value: 'static' + ${{ else }}: + variables: + - name: containerName + value: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName'] ] + - name: variableDefined + value: 'expression' dependsOn: ${{ parameters.parentJobs }} pool: From 2ca09ba27653a84cf422b6a2740be1248c469e3f Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 17:55:35 -0700 Subject: [PATCH 44/64] simplify more --- .vsts-ci/linux.yml | 5 ++++- .vsts-ci/templates/nix-test.yml | 18 +++--------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 7ea78255661..62e9c73905b 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -137,7 +137,9 @@ stages: pool: ubuntu-20.04 purpose: UnelevatedPesterTests tagSet: CI - containerName: mcr.microsoft.com/powershell/test-deps:preview-mariner-1.0 + useContainer: yes + parentJobs: + - getContainerJob - template: templates/nix-test.yml parameters: @@ -145,6 +147,7 @@ stages: pool: ubuntu-20.04 purpose: ElevatedPesterTests tagSet: CI + useContainer: yes parentJobs: - getContainerJob diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index 3c5ff3d227e..4f98783dce6 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -4,29 +4,17 @@ parameters: purpose: '' tagSet: 'CI' name: 'mac' - containerName: '' + useContainer: 'no' jobs: - job: ${{ parameters.name }}_test_${{ parameters.purpose }}_${{ parameters.tagSet }} - ${{ if ne(parameters.containerName, '') }}: - variables: - - name: containerName - value: ${{ parameters.containerName }} - - name: variableDefined - value: 'static' - ${{ else }}: - variables: - - name: containerName - value: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName'] ] - - name: variableDefined - value: 'expression' dependsOn: ${{ parameters.parentJobs }} pool: vmImage: ${{ parameters.pool }} - ${{ if and(ne(variables.containerName, ''), not(contains(variables.containerName, 'alpine'))) }}: - container: ${{ variables.containerName }} + ${{ if ne(variables.useContainer, 'no') }}: + container: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName'] ] displayName: ${{ parameters.name }} Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }} From c4e09b4eadc5e12720d8b22cc6ce46d9a46a9755 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 18:57:19 -0700 Subject: [PATCH 45/64] make dependsOn more explicit --- .vsts-ci/templates/nix-test.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index 4f98783dce6..227eb23b024 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -8,11 +8,18 @@ parameters: jobs: - job: ${{ parameters.name }}_test_${{ parameters.purpose }}_${{ parameters.tagSet }} - dependsOn: - ${{ parameters.parentJobs }} + + ${{ if ne(variables.useContainer, 'no') }}: + dependsOn: + - getContainerJob + variables: + usingContainer: 'yes' + pool: vmImage: ${{ parameters.pool }} + condition: or(ne(variables.useContainer, 'no'), ne(dependencies.getContainerJob.outputs['getContainerTask.containerName'],'')) + ${{ if ne(variables.useContainer, 'no') }}: container: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName'] ] From a08d68808cda503ac18fa2fcc8f35256ef5ef1fc Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 19:00:41 -0700 Subject: [PATCH 46/64] oops --- .vsts-ci/templates/nix-test.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index 227eb23b024..c424d73614e 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -9,19 +9,17 @@ parameters: jobs: - job: ${{ parameters.name }}_test_${{ parameters.purpose }}_${{ parameters.tagSet }} - ${{ if ne(variables.useContainer, 'no') }}: + ${{ if ne(parameters.useContainer, 'no') }}: dependsOn: - getContainerJob variables: usingContainer: 'yes' + container: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName'] ] pool: vmImage: ${{ parameters.pool }} - condition: or(ne(variables.useContainer, 'no'), ne(dependencies.getContainerJob.outputs['getContainerTask.containerName'],'')) - - ${{ if ne(variables.useContainer, 'no') }}: - container: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName'] ] + condition: or(ne('${{ parameters.useContainer }}', 'no'), ne(dependencies.getContainerJob.outputs['getContainerTask.containerName'],'')) displayName: ${{ parameters.name }} Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }} From 3c703c897c6251e9871fd2f1698bbff228104d9e Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 19:16:58 -0700 Subject: [PATCH 47/64] split into two templates --- .vsts-ci/linux.yml | 12 +-- .vsts-ci/templates/nix-test.yml | 76 +------------------ .../templates/test/nix-container-test.yml | 29 +++++++ .vsts-ci/templates/test/nix-test-steps.yml | 66 ++++++++++++++++ 4 files changed, 102 insertions(+), 81 deletions(-) create mode 100644 .vsts-ci/templates/test/nix-container-test.yml create mode 100644 .vsts-ci/templates/test/nix-test-steps.yml diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 62e9c73905b..4bf7e265a05 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -105,7 +105,7 @@ stages: dependsOn: [BuildLinuxStage] jobs: - job: getContainerJob - displayName: Get Container + displayName: Choose a container pool: vmImage: ubuntu-20.04 steps: @@ -131,25 +131,19 @@ stages: displayName: Get Container continueOnError: true - - template: templates/nix-test.yml + - template: templates/test/nix-container-test.yml parameters: name: container pool: ubuntu-20.04 purpose: UnelevatedPesterTests tagSet: CI - useContainer: yes - parentJobs: - - getContainerJob - - template: templates/nix-test.yml + - template: templates/test/nix-container-test.yml parameters: name: container pool: ubuntu-20.04 purpose: ElevatedPesterTests tagSet: CI - useContainer: yes - parentJobs: - - getContainerJob - stage: PackageLinux displayName: Package Linux diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index c424d73614e..ab3985dacd6 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -1,87 +1,19 @@ parameters: pool: 'macOS-latest' - parentJobs: [] purpose: '' tagSet: 'CI' name: 'mac' - useContainer: 'no' jobs: - job: ${{ parameters.name }}_test_${{ parameters.purpose }}_${{ parameters.tagSet }} - ${{ if ne(parameters.useContainer, 'no') }}: - dependsOn: - - getContainerJob - variables: - usingContainer: 'yes' - container: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName'] ] - pool: vmImage: ${{ parameters.pool }} - condition: or(ne('${{ parameters.useContainer }}', 'no'), ne(dependencies.getContainerJob.outputs['getContainerTask.containerName'],'')) - displayName: ${{ parameters.name }} Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }} steps: - - pwsh: | - Get-ChildItem -Path env: - displayName: Capture Environment - condition: succeededOrFailed() - - - task: DownloadBuildArtifacts@0 - displayName: 'Download build artifacts' - inputs: - downloadType: specific - itemPattern: | - build/**/* - downloadPath: '$(System.ArtifactsDirectory)' - - - pwsh: | - Get-ChildItem "$(System.ArtifactsDirectory)\*" -Recurse - displayName: 'Capture Artifacts Directory' - continueOnError: true - - - pwsh: | - if (Get-Command tdnf -errorAction SilentlyContinue) { - sudo tdnf install -y unzip git - } - displayName: 'Install unzip, git using tdnf' - continueOnError: true - - - pwsh: | - Import-Module .\tools\ci.psm1 - Invoke-CIInstall -SkipUser - displayName: Bootstrap - - - task: ExtractFiles@1 - displayName: 'Extract Build ZIP' - inputs: - archiveFilePatterns: '$(System.ArtifactsDirectory)/build/build.zip' - destinationFolder: '$(System.ArtifactsDirectory)/bins' - - - bash: | - find "$(System.ArtifactsDirectory)/bins" -type d -exec chmod +rwx {} \; - find "$(System.ArtifactsDirectory)/bins" -type f -exec chmod +rw {} \; - displayName: 'Fix permissions' - continueOnError: true - - - pwsh: | - Get-ChildItem "$(System.ArtifactsDirectory)\bins\*" -Recurse -ErrorAction SilentlyContinue - displayName: 'Capture Extracted Build ZIP' - continueOnError: true - - - pwsh: | - Import-Module .\tools\ci.psm1 - Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' - $options = (Get-PSOptions) - $rootPath = '$(System.ArtifactsDirectory)\bins' - $originalRootPath = Split-Path -path $options.Output - $path = Join-Path -path $rootPath -ChildPath (split-path -leaf -path $originalRootPath) - $pwshPath = Join-Path -path $path -ChildPath 'pwsh' - chmod a+x $pwshPath - $options.Output = $pwshPath - Set-PSOptions $options - Invoke-CITest -Purpose '${{ parameters.purpose }}' -TagSet '${{ parameters.tagSet }}' - displayName: Test - condition: succeeded() + - template: ./test/nix-test-steps.yml + parameters: + purpose: ${{ parameters.purpose }} + tagSet: ${{ parameters.tagSet }} diff --git a/.vsts-ci/templates/test/nix-container-test.yml b/.vsts-ci/templates/test/nix-container-test.yml new file mode 100644 index 00000000000..146b500bc9a --- /dev/null +++ b/.vsts-ci/templates/test/nix-container-test.yml @@ -0,0 +1,29 @@ +parameters: + pool: 'macOS-latest' + purpose: '' + tagSet: 'CI' + name: 'mac' + +jobs: +- job: ${{ parameters.name }}_test_${{ parameters.purpose }}_${{ parameters.tagSet }} + + dependsOn: + - getContainerJob + + variables: + usingContainer: 'yes' + containerName: mcr.microsoft.com/powershell/test-deps:preview-mariner-1.0 + getContainerJob: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName'] ] + + container: $(containerName) + + pool: + vmImage: ${{ parameters.pool }} + + displayName: ${{ parameters.name }} Test - ${{ parameters.purpose }} - ${{ parameters.tagSet }} + + steps: + - template: ./nix-test-steps.yml + parameters: + purpose: ${{ parameters.purpose }} + tagSet: ${{ parameters.tagSet }} diff --git a/.vsts-ci/templates/test/nix-test-steps.yml b/.vsts-ci/templates/test/nix-test-steps.yml new file mode 100644 index 00000000000..707e99b3023 --- /dev/null +++ b/.vsts-ci/templates/test/nix-test-steps.yml @@ -0,0 +1,66 @@ +parameters: + purpose: '' + tagSet: 'CI' + +steps: + - pwsh: | + Get-ChildItem -Path env: + displayName: Capture Environment + condition: succeededOrFailed() + + - task: DownloadBuildArtifacts@0 + displayName: 'Download build artifacts' + inputs: + downloadType: specific + itemPattern: | + build/**/* + downloadPath: '$(System.ArtifactsDirectory)' + + - pwsh: | + Get-ChildItem "$(System.ArtifactsDirectory)\*" -Recurse + displayName: 'Capture Artifacts Directory' + continueOnError: true + + - pwsh: | + if (Get-Command tdnf -errorAction SilentlyContinue) { + sudo tdnf install -y unzip git + } + displayName: 'Install unzip, git using tdnf' + continueOnError: true + + - pwsh: | + Import-Module .\tools\ci.psm1 + Invoke-CIInstall -SkipUser + displayName: Bootstrap + + - task: ExtractFiles@1 + displayName: 'Extract Build ZIP' + inputs: + archiveFilePatterns: '$(System.ArtifactsDirectory)/build/build.zip' + destinationFolder: '$(System.ArtifactsDirectory)/bins' + + - bash: | + find "$(System.ArtifactsDirectory)/bins" -type d -exec chmod +rwx {} \; + find "$(System.ArtifactsDirectory)/bins" -type f -exec chmod +rw {} \; + displayName: 'Fix permissions' + continueOnError: true + + - pwsh: | + Get-ChildItem "$(System.ArtifactsDirectory)\bins\*" -Recurse -ErrorAction SilentlyContinue + displayName: 'Capture Extracted Build ZIP' + continueOnError: true + + - pwsh: | + Import-Module .\tools\ci.psm1 + Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' + $options = (Get-PSOptions) + $rootPath = '$(System.ArtifactsDirectory)\bins' + $originalRootPath = Split-Path -path $options.Output + $path = Join-Path -path $rootPath -ChildPath (split-path -leaf -path $originalRootPath) + $pwshPath = Join-Path -path $path -ChildPath 'pwsh' + chmod a+x $pwshPath + $options.Output = $pwshPath + Set-PSOptions $options + Invoke-CITest -Purpose '${{ parameters.purpose }}' -TagSet '${{ parameters.tagSet }}' + displayName: Test + condition: succeeded() From a0614b78f30b9c7ff70950a36d01b2ebf1e2e90a Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 19:38:08 -0700 Subject: [PATCH 48/64] =?UTF-8?q?set=20container=20statically=20?= =?UTF-8?q?=F0=9F=8E=89working?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vsts-ci/templates/test/nix-container-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/templates/test/nix-container-test.yml b/.vsts-ci/templates/test/nix-container-test.yml index 146b500bc9a..9c54401d2a9 100644 --- a/.vsts-ci/templates/test/nix-container-test.yml +++ b/.vsts-ci/templates/test/nix-container-test.yml @@ -15,7 +15,7 @@ jobs: containerName: mcr.microsoft.com/powershell/test-deps:preview-mariner-1.0 getContainerJob: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName'] ] - container: $(containerName) + container: mcr.microsoft.com/powershell/test-deps:preview-mariner-1.0 pool: vmImage: ${{ parameters.pool }} From 6fdaa7786746d5196bf0cbdc89db79eb36b456be Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 19:45:04 -0700 Subject: [PATCH 49/64] try runtime evaluation --- .vsts-ci/templates/test/nix-container-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/templates/test/nix-container-test.yml b/.vsts-ci/templates/test/nix-container-test.yml index 9c54401d2a9..bb2210f3911 100644 --- a/.vsts-ci/templates/test/nix-container-test.yml +++ b/.vsts-ci/templates/test/nix-container-test.yml @@ -15,7 +15,7 @@ jobs: containerName: mcr.microsoft.com/powershell/test-deps:preview-mariner-1.0 getContainerJob: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName'] ] - container: mcr.microsoft.com/powershell/test-deps:preview-mariner-1.0 + container: $[ variables.containerName ] pool: vmImage: ${{ parameters.pool }} From 35c8e49284aba5e5cb4def2adcbe179ba6222bd1 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 19:53:42 -0700 Subject: [PATCH 50/64] Fix ping fake host --- .../Test-Connection.Tests.ps1 | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/test/powershell/Modules/Microsoft.PowerShell.Management/Test-Connection.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Management/Test-Connection.Tests.ps1 index dbe91e86574..b8c9d14a1be 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Management/Test-Connection.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Management/Test-Connection.Tests.ps1 @@ -97,15 +97,8 @@ Describe "Test-Connection" -tags "CI", "RequireSudoOnUnix" { { Test-Connection "fakeHost" -Count 1 -ErrorAction Stop } | Should -Throw -ErrorId "TestConnectionException,Microsoft.PowerShell.Commands.TestConnectionCommand" # Error code = 11001 - Host not found. - $platform = Get-PlatformInfo - if ($platform.Platform -match "raspbian" -or ( $platform.Platform -match 'ubuntu' -and $platform.Version -eq '20.04')) { - $code = 11 - } elseif (!$IsWindows) { - $code = -131073 - } else { - $code = 11001 - } - $error[0].Exception.InnerException.ErrorCode | Should -Be $code + # Error code = -131073 - Invalid address + $error[0].Exception.InnerException.ErrorCode | Should -BeIn 11, -131073, 11001 } It "Force IPv4 with implicit PingOptions" { From ce3ca8519e616fd68d3e33359292531abd43f4a7 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 19:56:34 -0700 Subject: [PATCH 51/64] wsman doesn't work on mariner --- test/powershell/engine/Remoting/PSSession.Tests.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/powershell/engine/Remoting/PSSession.Tests.ps1 b/test/powershell/engine/Remoting/PSSession.Tests.ps1 index 25c7ab9c582..465710245ff 100644 --- a/test/powershell/engine/Remoting/PSSession.Tests.ps1 +++ b/test/powershell/engine/Remoting/PSSession.Tests.ps1 @@ -84,7 +84,8 @@ Describe "SkipCACheck and SkipCNCheck PSSession options are required for New-PSS ($platformInfo.Platform -match "alpine|raspbian") -or ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or - ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') + ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') -or + ($platformInfo.Platform -eq 'mariner') ) { Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, and CentOS 8" return From 41270f7c94250ab77e902c670ff1dbe8354e63be Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 20:04:55 -0700 Subject: [PATCH 52/64] add function to determine if WSMan availible --- .../engine/Remoting/PSSession.Tests.ps1 | 11 ++----- .../Remoting/RemoteSession.Basic.Tests.ps1 | 33 ++++--------------- .../Modules/HelpersCommon/HelpersCommon.psm1 | 16 +++++++++ 3 files changed, 24 insertions(+), 36 deletions(-) diff --git a/test/powershell/engine/Remoting/PSSession.Tests.ps1 b/test/powershell/engine/Remoting/PSSession.Tests.ps1 index 465710245ff..cbf7313eed0 100644 --- a/test/powershell/engine/Remoting/PSSession.Tests.ps1 +++ b/test/powershell/engine/Remoting/PSSession.Tests.ps1 @@ -79,15 +79,8 @@ Describe "SkipCACheck and SkipCNCheck PSSession options are required for New-PSS It "" -TestCases $testCases { param ($scriptBlock, $expectedErrorCode) - $platformInfo = Get-PlatformInfo - if ( - ($platformInfo.Platform -match "alpine|raspbian") -or - ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID - ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or - ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') -or - ($platformInfo.Platform -eq 'mariner') - ) { - Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, and CentOS 8" + if ( -not (Get-WsManSupport)) { + Set-ItResult -Skipped -Because "MI library not available for this platform" return } diff --git a/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 b/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 index d56ed1999ac..84e25a276ec 100644 --- a/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 +++ b/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 @@ -10,15 +10,8 @@ function GetRandomString() Describe "New-PSSession basic test" -Tag @("CI") { It "New-PSSession should not crash powershell" { - $platformInfo = Get-PlatformInfo - if ( - ($platformInfo.Platform -match "alpine|raspbian") -or - ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID - ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or - ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') -or - ($IsMacOS) - ) { - Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" + if ( -not (Get-WsManSupport)) { + Set-ItResult -Skipped -Because "MI library not available for this platform" return } @@ -29,15 +22,8 @@ Describe "New-PSSession basic test" -Tag @("CI") { Describe "Basic Auth over HTTP not allowed on Unix" -Tag @("CI") { It "New-PSSession should throw when specifying Basic Auth over HTTP on Unix" -Skip:($IsWindows) { - $platformInfo = Get-PlatformInfo - if ( - ($platformInfo.Platform -match "alpine|raspbian") -or - ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID - ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or - ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') -or - ($IsMacOS) - ) { - Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" + if ( -not (Get-WsManSupport)) { + Set-ItResult -Skipped -Because "MI library not available for this platform" return } @@ -53,15 +39,8 @@ Describe "Basic Auth over HTTP not allowed on Unix" -Tag @("CI") { # Skip this test for macOS because the latest OS release is incompatible with our shipped libmi for WinRM/OMI. It "New-PSSession should NOT throw a ConnectFailed exception when specifying Basic Auth over HTTPS on Unix" -Skip:($IsWindows) { - $platformInfo = Get-PlatformInfo - if ( - ($platformInfo.Platform -match "alpine|raspbian") -or - ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID - ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or - ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') -or - ($IsMacOS) - ) { - Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" + if ( -not (Get-WsManSupport)) { + Set-ItResult -Skipped -Because "MI library not available for this platform" return } diff --git a/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 b/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 index c971e786765..1cc05e03098 100644 --- a/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 +++ b/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 @@ -385,3 +385,19 @@ function Get-PlatformInfo { return "unknown" } } + +# return true if WsMan is supported on the current platform +function Get-WsManSupport { + $platformInfo = Get-PlatformInfo + if ( + ($platformInfo.Platform -match "alpine|raspbian") -or + ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID + ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or + ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') -or + ($platformInfo.Platform -eq 'mariner') -or + ($IsMacOS) + ) { + return $false + } + return $true +} From fc5a15d7a17a054dedf4ec7665a5cb37b44937aa Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 20:46:20 -0700 Subject: [PATCH 53/64] switch to queried variable --- .vsts-ci/templates/test/nix-container-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/templates/test/nix-container-test.yml b/.vsts-ci/templates/test/nix-container-test.yml index bb2210f3911..18a7636335c 100644 --- a/.vsts-ci/templates/test/nix-container-test.yml +++ b/.vsts-ci/templates/test/nix-container-test.yml @@ -15,7 +15,7 @@ jobs: containerName: mcr.microsoft.com/powershell/test-deps:preview-mariner-1.0 getContainerJob: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName'] ] - container: $[ variables.containerName ] + container: $[ variables.getContainerJob ] pool: vmImage: ${{ parameters.pool }} From 95a589744202a91dc0b843901407d4da216816f7 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Fri, 29 Apr 2022 20:46:32 -0700 Subject: [PATCH 54/64] expose new function --- test/tools/Modules/HelpersCommon/HelpersCommon.psd1 | 1 + 1 file changed, 1 insertion(+) diff --git a/test/tools/Modules/HelpersCommon/HelpersCommon.psd1 b/test/tools/Modules/HelpersCommon/HelpersCommon.psd1 index 3d501983806..43d2ada88cc 100644 --- a/test/tools/Modules/HelpersCommon/HelpersCommon.psd1 +++ b/test/tools/Modules/HelpersCommon/HelpersCommon.psd1 @@ -35,6 +35,7 @@ FunctionsToExport = @( 'Wait-FileToBePresent' 'Wait-UntilTrue' 'Get-PlatformInfo' + 'Get-WSManSupport' ) CmdletsToExport= @() From 8f615c2810b2b7c8a31b459881b37b77a97585e3 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Sat, 30 Apr 2022 11:58:25 -0700 Subject: [PATCH 55/64] make work with 5.1 --- tools/ci.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci.psm1 b/tools/ci.psm1 index af1246f5948..c06137a438d 100644 --- a/tools/ci.psm1 +++ b/tools/ci.psm1 @@ -55,7 +55,7 @@ Function Set-BuildVariable $IsOutput ) - $IsOutputString = $IsOutput ? 'true' : 'false' + $IsOutputString = if ($IsOutput) { 'true' } else { 'false' } $command = "vso[task.setvariable variable=$Name;isOutput=$IsOutputString]$Value" # always log command to make local debugging easier From 0a8dfba7c53332b314a637db1fdf781421467fb5 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Sat, 30 Apr 2022 11:59:44 -0700 Subject: [PATCH 56/64] add other container test jobs --- .vsts-ci/linux.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 4bf7e265a05..dd6c1f99a67 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -145,6 +145,20 @@ stages: purpose: ElevatedPesterTests tagSet: CI + - template: templates/test/nix-container-test.yml + parameters: + name: container + pool: ubuntu-20.04 + purpose: UnelevatedPesterTests + tagSet: Others + + - template: templates/test/nix-container-test.yml + parameters: + name: container + pool: ubuntu-20.04 + purpose: ElevatedPesterTests + tagSet: Others + - stage: PackageLinux displayName: Package Linux dependsOn: ["BuildLinuxStage"] From 5ba5be65c8aafd24b594fac58c319b0b957c3640 Mon Sep 17 00:00:00 2001 From: travis plunk Date: Sat, 30 Apr 2022 17:55:44 -0700 Subject: [PATCH 57/64] remove unused variables and set __INCONTAINER --- .vsts-ci/templates/test/nix-container-test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.vsts-ci/templates/test/nix-container-test.yml b/.vsts-ci/templates/test/nix-container-test.yml index 18a7636335c..87827ac2508 100644 --- a/.vsts-ci/templates/test/nix-container-test.yml +++ b/.vsts-ci/templates/test/nix-container-test.yml @@ -11,9 +11,8 @@ jobs: - getContainerJob variables: - usingContainer: 'yes' - containerName: mcr.microsoft.com/powershell/test-deps:preview-mariner-1.0 getContainerJob: $[ dependencies.getContainerJob.outputs['getContainerTask.containerName'] ] + __INCONTAINER: 1 container: $[ variables.getContainerJob ] From 42b7aafe98cb7e8ed3ef2d262a4e12e794daaa00 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 2 May 2022 11:13:40 -0700 Subject: [PATCH 58/64] Update .vsts-ci/linux.yml --- .vsts-ci/linux.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index dd6c1f99a67..b7ba9d505f6 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -117,6 +117,8 @@ stages: - pwsh: | $matrix = ./PowerShell-Docker/build.ps1 -GenerateMatrixJson -FullJson -Channel preview | ConvertFrom-Json + # Filter out containers that won't run on AzDevOps Linux host as an agent + # Windows, nano server, alpine, and any ARM image $linuxImages = $matrix.preview | Where-Object {$_.osversion -notlike '*windows*' -and $_.osversion -notlike '*nano*' -and $_.osversion -notlike 'alpine*' -and $_.osversion -notlike '*arm*'} | Select-Object -First 100 JobName, Taglist | From c876b4e867653a72f0ad44e09d2c9f448cb1492f Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 2 May 2022 11:20:02 -0700 Subject: [PATCH 59/64] Update .vsts-ci/linux.yml Co-authored-by: James Truher [MSFT] --- .vsts-ci/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index b7ba9d505f6..7f7c0f3d567 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -120,7 +120,7 @@ stages: # Filter out containers that won't run on AzDevOps Linux host as an agent # Windows, nano server, alpine, and any ARM image $linuxImages = $matrix.preview | - Where-Object {$_.osversion -notlike '*windows*' -and $_.osversion -notlike '*nano*' -and $_.osversion -notlike 'alpine*' -and $_.osversion -notlike '*arm*'} | + Where-Object {$_.osversion -notmatch 'windows|nano|alpine|arm'} | Select-Object -First 100 JobName, Taglist | Sort-Object -property JobName $marinerImage = $linuxImages | where-object {$_.JobName -like 'mariner*'} From 0b1e7375fc36fb3bbd96395bb9f2f89145c16abd Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 2 May 2022 11:21:29 -0700 Subject: [PATCH 60/64] Update .vsts-ci/linux.yml --- .vsts-ci/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 7f7c0f3d567..c0d100d15d3 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -121,7 +121,7 @@ stages: # Windows, nano server, alpine, and any ARM image $linuxImages = $matrix.preview | Where-Object {$_.osversion -notmatch 'windows|nano|alpine|arm'} | - Select-Object -First 100 JobName, Taglist | + Select-Object JobName, Taglist | Sort-Object -property JobName $marinerImage = $linuxImages | where-object {$_.JobName -like 'mariner*'} Import-Module ./PowerShell/tools/ci.psm1 From ec88d980ef75427aeffdbdfe2c6a01f9b98df04d Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 2 May 2022 11:23:17 -0700 Subject: [PATCH 61/64] Update .vsts-ci/linux.yml Co-authored-by: James Truher [MSFT] --- .vsts-ci/linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index c0d100d15d3..f5623ccb2a1 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -123,6 +123,7 @@ stages: Where-Object {$_.osversion -notmatch 'windows|nano|alpine|arm'} | Select-Object JobName, Taglist | Sort-Object -property JobName + # Force mariner for now, later we will add automated rotation by the month. $marinerImage = $linuxImages | where-object {$_.JobName -like 'mariner*'} Import-Module ./PowerShell/tools/ci.psm1 $tag = $marinerImage.Taglist -split ';' | select-object -first 1 From 165118e1b5a6ce77631ae0881ffbf4ccc0f29bfa Mon Sep 17 00:00:00 2001 From: travis plunk Date: Mon, 2 May 2022 14:40:39 -0700 Subject: [PATCH 62/64] remove install command now that images have the packages --- .vsts-ci/templates/test/nix-test-steps.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.vsts-ci/templates/test/nix-test-steps.yml b/.vsts-ci/templates/test/nix-test-steps.yml index 707e99b3023..c0329888006 100644 --- a/.vsts-ci/templates/test/nix-test-steps.yml +++ b/.vsts-ci/templates/test/nix-test-steps.yml @@ -21,13 +21,6 @@ steps: displayName: 'Capture Artifacts Directory' continueOnError: true - - pwsh: | - if (Get-Command tdnf -errorAction SilentlyContinue) { - sudo tdnf install -y unzip git - } - displayName: 'Install unzip, git using tdnf' - continueOnError: true - - pwsh: | Import-Module .\tools\ci.psm1 Invoke-CIInstall -SkipUser From 9bbaa4dc55a81a43229977fb9dbea3581d99833e Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 2 May 2022 14:43:34 -0700 Subject: [PATCH 63/64] Update test/tools/Modules/HelpersCommon/HelpersCommon.psm1 Co-authored-by: Aditya Patwardhan --- test/tools/Modules/HelpersCommon/HelpersCommon.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 b/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 index 1cc05e03098..0de614c7ed5 100644 --- a/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 +++ b/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 @@ -391,7 +391,7 @@ function Get-WsManSupport { $platformInfo = Get-PlatformInfo if ( ($platformInfo.Platform -match "alpine|raspbian") -or - ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID + ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -ne '9') -or ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') -or ($platformInfo.Platform -eq 'mariner') -or From 1ae439b94682631d25d6b08b8166fe41c4f76fb8 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 2 May 2022 14:56:05 -0700 Subject: [PATCH 64/64] Update test/tools/Modules/HelpersCommon/HelpersCommon.psm1 --- test/tools/Modules/HelpersCommon/HelpersCommon.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 b/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 index 0de614c7ed5..dbd77186d2f 100644 --- a/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 +++ b/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 @@ -391,7 +391,7 @@ function Get-WsManSupport { $platformInfo = Get-PlatformInfo if ( ($platformInfo.Platform -match "alpine|raspbian") -or - ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -ne '9') -or + ($platformInfo.Platform -eq "debian" -and $platformInfo.Version -ne '9') -or ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') -or ($platformInfo.Platform -eq 'mariner') -or