diff --git a/.github/actions/install-tools/action.yaml b/.github/actions/install-tools/action.yaml index 81680caaa..6996a02f1 100644 --- a/.github/actions/install-tools/action.yaml +++ b/.github/actions/install-tools/action.yaml @@ -11,4 +11,4 @@ runs: - run: "curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin" shell: bash - - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + - uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0 diff --git a/.github/workflows/commitlint.yaml b/.github/workflows/commitlint.yaml index 6d3382221..b2b2964c5 100644 --- a/.github/workflows/commitlint.yaml +++ b/.github/workflows/commitlint.yaml @@ -27,4 +27,7 @@ jobs: run: npm install --save-dev @commitlint/{config-conventional,cli} - name: Lint PR title - run: echo "${{ github.event.pull_request.title }}" | npx commitlint + env: + PR_TITLE: ${{ github.event.pull_request.title }} + run: | + echo "$PR_TITLE" | npx commitlint diff --git a/.github/workflows/nightly-uds-core.yaml b/.github/workflows/nightly-uds-core.yaml new file mode 100644 index 000000000..074bcedea --- /dev/null +++ b/.github/workflows/nightly-uds-core.yaml @@ -0,0 +1,47 @@ +name: UDS Core Smoke Test +on: + schedule: + - cron: '0 7 * * *' ## Every day at 0700 UTC + + workflow_dispatch: + +permissions: + contents: read + +# Abort prior jobs in the same workflow / PR +concurrency: + group: uds-core-${{ github.ref }} + cancel-in-progress: true + +jobs: + uds-core-test: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + + - name: Build UDS-CLI binary + run: make build-cli-linux-amd ARCH=amd64 + + - name: Make UDS-CLI executable + run: | + chmod +x build/uds + + - name: Setup K3d + uses: ./.github/actions/k3d + + - name: Deploy UDS Core bundle + # renovate: datasource=github-tags depName=defenseunicorns/uds-core versioning=semver + run: build/uds deploy ghcr.io/defenseunicorns/packages/uds/bundles/k3d-core-istio-dev:0.13.1 --confirm --no-progress + shell: bash + + - name: Validate UDS Core deployment + run: | + build/uds zarf tools wait-for gateway admin-gateway -n istio-admin-gateway --timeout 10s + build/uds zarf tools wait-for gateway passthrough-gateway -n istio-passthrough-gateway --timeout 10s + build/uds zarf tools wait-for gateway tenant-gateway -n istio-tenant-gateway --timeout 10s + shell: bash diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 58bbb98c2..1e841f559 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -48,7 +48,7 @@ jobs: fetch-depth: 0 - name: Download build artifacts - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2 + uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4 with: name: build-artifacts path: build/ @@ -98,7 +98,7 @@ jobs: uses: ./.github/actions/install-tools - name: Download build artifacts - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2 + uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4 with: name: build-artifacts path: build/ diff --git a/.github/workflows/scan-codeql.yaml b/.github/workflows/scan-codeql.yaml index f8dda29d6..6d21c834c 100644 --- a/.github/workflows/scan-codeql.yaml +++ b/.github/workflows/scan-codeql.yaml @@ -45,7 +45,7 @@ jobs: run: make build-cli-linux-amd - name: Initialize CodeQL - uses: github/codeql-action/init@e2e140ad1441662206e8f97754b166877dfa1c73 # v3.24.4 + uses: github/codeql-action/init@8a470fddafa5cbb6266ee11b37ef4d8aae19c571 # v3.24.6 env: CODEQL_EXTRACTOR_GO_BUILD_TRACING: on with: @@ -54,6 +54,6 @@ jobs: - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@e2e140ad1441662206e8f97754b166877dfa1c73 # v3.24.4 + uses: github/codeql-action/analyze@8a470fddafa5cbb6266ee11b37ef4d8aae19c571 # v3.24.6 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml index d0f23f067..6fa60c520 100644 --- a/.github/workflows/scorecard.yaml +++ b/.github/workflows/scorecard.yaml @@ -45,6 +45,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@e2e140ad1441662206e8f97754b166877dfa1c73 # v3.24.4 + uses: github/codeql-action/upload-sarif@8a470fddafa5cbb6266ee11b37ef4d8aae19c571 # v3.24.6 with: sarif_file: results.sarif diff --git a/Makefile b/Makefile index d91a26c4a..3f2b0978b 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ BUILD_ARGS := -s -w -X 'github.com/defenseunicorns/uds-cli/src/config.CLIVersion .PHONY: help help: ## Display this help information - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ + @grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ | sort | awk 'BEGIN {FS = ":.*?## "}; \ {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index 7543b2b77..7c8fab549 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ 1. [Zarf Integration](#zarf-integration) 1. [Bundle Overrides](docs/overrides.md) 1. [Bundle Anatomy](docs/anatomy.md) -1. [UDS Runner](docs/runner.md) +1. [Runner](docs/runner.md) ## Install Recommended installation method is with Brew: @@ -45,7 +45,7 @@ metadata: packages: - name: init repository: ghcr.io/defenseunicorns/packages/init - ref: v0.31.4 + ref: v0.32.3 optionalComponents: - git-server - name: podinfo diff --git a/adr/0002-runner.md b/adr/0002-runner.md index ab06c9995..940f19ce6 100644 --- a/adr/0002-runner.md +++ b/adr/0002-runner.md @@ -3,7 +3,7 @@ Date: 1 Feb 2024 ## Status -Accepted +[AMENDED](#amendment-1) ## Context @@ -25,3 +25,14 @@ After quickly gaining adoption across the organization, we have decided to make ## Consequences The UDS CLI team will own the UDS Runner functionality and is responsible for maintaining it. Furthermore, because the UDS Runner uses Zarf, the UDS CLI team will contribute to upstream Zarf Actions and common library functionality to support UDS Runner. + +# Amendment 1 + +Date: 1 March 2024 + +## Status + +Accepted + +## Context and Decision +In an effort to reduce the scope of UDS CLI and experiment with a new standalone project, the UDS Runner functionality will be moved to a new project tentatively named [maru-runner](https://github.com/defenseunicorns/maru-runner). This project will be maintained by the UDS CLI team for a short time but ownership will be eventually be transferred to a different team. Furthermore, UDS CLI will vendor the runner such that no breaking changes will be introduced for UDS CLI users. diff --git a/docs/runner.md b/docs/runner.md index d2008acca..d234e6138 100644 --- a/docs/runner.md +++ b/docs/runner.md @@ -1,375 +1,37 @@ # UDS Runner -UDS runner enables UDS Bundle developers to automate UDS builds and perform common shell tasks. It -uses [Zarf](https://zarf.dev/) under the hood to perform tasks and shares a syntax similar to `zarf.yaml` manifests. -Many [Zarf Actions features](https://docs.zarf.dev/docs/create-a-zarf-package/component-actions) are also available in -UDS runner. +UDS CLI contains vendors and configures the [maru-runner](https://github.com/defenseunicorns/maru-runner) build tool to make compiling and building UDS bundles simple. -## Table of Contents - -- [UDS Runner](#uds-runner) - - [Quickstart](#quickstart) - - [Key Concepts](#key-concepts) - - [Tasks](#tasks) - - [Actions](#actions) - - [Task](#task) - - [Cmd](#cmd) - - [Variables](#variables) - - [Files](#files) - - [Wait](#wait) - - [Includes](#includes) - - [Task Inputs and Reusable Tasks](#task-inputs-and-reusable-tasks) ## Quickstart -Create a file called `tasks.yaml` - -```yaml -variables: - - name: FOO - default: foo - -tasks: - - name: default - actions: - - cmd: echo "run default task" - - - name: example - actions: - - task: set-variable - - task: echo-variable - - - name: set-variable - actions: - - cmd: echo "bar" - setVariables: - - name: FOO - - - name: echo-variable - actions: - - cmd: echo ${FOO} +#### Running a Task +To run a task from a `tasks.yaml`: ``` - -From the same directory as the `tasks.yaml`, run the `example` task using: - -```bash -uds run example +uds run ``` -This will run the `example` tasks which in turn runs the `set-variable` and `echo-variable`. In this example, the text " -bar" should be printed to the screen twice. - -Optionally, you can specify the location and name of your `tasks.yaml` using the `--file` or `-f` flag: - -```bash -uds run example -f tmp/tasks.yaml +#### Running a Task from a specific tasks file ``` - -You can also view the tasks that are available to run using the `list` flag: - -```bash -uds run -f tmp/tasks.yaml --list +uds run -f ``` -## Key Concepts -### Tasks +The Maru [docs](https://github.com/defenseunicorns/maru-runner) describe how to build `tasks.yaml` files to configure the runner. The functionality in UDS CLI is mostly identical with the following exceptions -Tasks are the fundamental building blocks of the UDS runner and they define operations to be performed. The `tasks` key -at the root of `tasks.yaml` define a list of tasks to be run. This underlying operations performed by a task are defined -under the `actions` key: +### Variables Set with Environment Variables +When running a `tasks.yaml` with `uds run my-task` you can set variables using environment prefixed with `UDS_` -```yaml -tasks: - - name: all-the-tasks - actions: - - task: make-build-dir - - task: install-deps -``` - -In this example, the name of the task is "all-the-tasks", and it is composed of multiple sub-tasks to run. These sub-tasks -would also be defined in the list of `tasks`: - -```yaml -tasks: - - name: default - actions: - - cmd: echo "run default task" - - - name: all-the-tasks - actions: - - task: make-build-dir - - task: install-deps - - - name: make-build-dir - actions: - - cmd: mkdir -p build - - - name: install-deps - actions: - - cmd: go mod tidy -``` - -Using the UDS CLI, these tasks can be run individually: - -```bash -uds run all-the-tasks # runs all-the-tasks, which calls make-build-dir and install-deps -uds run make-build-dir # only runs make-build-dir -``` - -#### Default Tasks -In the above example, there is also a `default` task, which is special, optional, task that can be used for the most common entrypoint for your tasks. When trying to run the `default` task, you can omit the task name from the run command: - -```bash -uds run -``` - -### Actions - -Actions are the underlying operations that a task will perform. Each action under the `actions` key has a unique syntax. - -#### Task - -A task can reference a task, thus making tasks composable. - -```yaml -tasks: - - name: foo - actions: - - task: bar - - name: bar - actions: - - task: baz - - name: baz - actions: - - cmd: "echo task foo is composed of task bar which is composed of task baz!" -``` - -In this example, the task `foo` calls a task called `bar` which calls a task `baz` which prints some output to the -console. - -#### Cmd - -Actions can run arbitrary bash commands including in-line scripts, and the output of a command can be placed in a -variable using the `setVariables` key - -```yaml -tasks: - - name: foo - actions: - - cmd: echo -n 'dHdvIHdlZWtzIG5vIHByb2JsZW0=' | base64 -d - setVariables: - - name: FOO -``` - -This task will decode the base64 string and set the value as a variable named `FOO` that can be used in other tasks. - -Command blocks can have several other properties including: - -- `description`: description of the command - - `mute`: boolean value to mute the output of a command - - `dir`: the directory to run the command in - - `env`: list of environment variables to run for this `cmd` block only - - ```yaml - tasks: - - name: foo - actions: - - cmd: echo ${BAR} - env: - - BAR=bar - ``` - - - `maxRetries`: number of times to retry the command - - `maxTotalSeconds`: max number of seconds the command can run until it is killed; takes precendence - over `maxRetries` - -### Variables - -Variables can be defined in several ways: - -1. At the top of the `tasks.yaml` - - ```yaml - variables: - - name: FOO - default: foo - - tasks: ... - ``` - -1. As the output of a `cmd` - - ```yaml - variables: - - name: FOO - default: foo - tasks: - - name: foo - actions: - - cmd: uname -m - mute: true - setVariables: - - name: FOO - - cmd: echo ${FOO} - ``` - -1. As an environment variable prefixed with `UDS_`. In the example above, if you create an env var `UDS_FOO=bar`, then the`FOO` variable would be set to `bar`. - -1. Using the `--set` flag in the CLI : `uds run foo --set FOO=bar` - -To use a variable, reference it using `${VAR_NAME}` - -Note that variables also have the following attributes when setting them with YAML: - -- `sensitive`: boolean value indicating if a variable should be visible in output -- `default`: default value of a variable - - In the example above, if `FOO` did not have a default, and you have an environment variable `UDS_FOO=bar`, the default would get set to `bar`. - -#### Environment Variable Files - -To include a file containing environment variables that you'd like to load into a task, use the `envPath` key in the task. This will load all of the environment variables in the file into the task being called and its child tasks. - -```yaml -tasks: - - name: env - actions: - - cmd: echo $FOO - - cmd: echo $UDS_ARCH - - task: echo-env - - name: echo-env - envPath: ./path/to/.env - actions: - - cmd: echo different task $FOO -``` - - -#### Variable Precedence -Variable precedence is as follows, from least to most specific: -- Variable defaults set in YAML -- Environment variables prefixed with `UDS_` -- Variables set with the `--set` flag in the CLI - -That is to say, variables set via the `--set` flag take precedence over all other variables. The exception to this precedence order is when a variable is modified using `setVariable`, which will change the value of the variable during runtime. - -### Files - -The `files` key is used to copy local or remote files to the current working directory - -```yaml -tasks: - - name: copy-local - files: - - source: /tmp/foo - target: foo - - name: copy-remote - files: - - source: https://cataas.com/cat - target: cat.jpeg -``` - -Files blocks can also use the following attributes: - -- `executable`: boolean value indicating if the file is executable -- `shasum`: SHA string to verify the integrity of the file -- `symlinks`: list of strings referring to symlink the file to - -### Wait - -The `wait`key is used to block execution while waiting for a resource, including network responses and K8s operations - -```yaml -tasks: - - name: network-response - wait: - network: - protocol: https - address: 1.1.1.1 - code: 200 - - name: configmap-creation - wait: - cluster: - kind: configmap - name: simple-configmap - namespace: foo -``` - -### Includes - -The `includes` key is used to import tasks from either local or remote task files. This is useful for sharing common tasks across multiple task files. When importing a task from a local task file, the path is relative to the file you are currently in. When running a task, the tasks in the task file as well as the `includes` get processed to ensure there are no infinite loop references. - -```yaml -includes: - - local: ./path/to/tasks-to-import.yaml - - remote: https://raw.githubusercontent.com/defenseunicorns/uds-cli/main/src/test/tasks/remote-import-tasks.yaml - -tasks: - - name: import-local - actions: - - task: local:some-local-task - - name: import-remote - actions: - - task: remote:echo-var -``` - -Note that included task files can also include other task files, with the following restriction: - -- If a task file includes a remote task file, the included remote task file cannot include any local task files - -### Task Inputs and Reusable Tasks - -Although all tasks should be reusable, sometimes you may want to create a task that can be reused with different inputs. To create a reusable task that requires inputs, add an `inputs` key with a map of inputs to the task: - -```yaml -tasks: - - name: echo-var - inputs: - hello-input: - default: hello world - description: This is an input to the echo-var task - deprecated-input: - default: foo - description: this is a input from a previous version of this task - deprecatedMessage: this input is deprecated, use hello-input instead - actions: - # to use the input, reference it using INPUT_ in all caps - - cmd: echo $INPUT_HELLO_INPUT - - - name: use-echo-var - actions: - - task: echo-var - with: - # hello-input is the name of the input in the echo-var task, hello-unicorn is the value we want to pass in - hello-input: hello unicorn -``` - -In this example, the `echo-var` task takes an input called `hello-input` and prints it to the console; notice that the `input` can have a `default` value. The `use-echo-var` task calls `echo-var` with a different input value using the `with` key. In this case `"hello unicorn"` is passed to the `hello-input` input. - -Note that the `deprecated-input` input has a `deprecatedMessage` attribute. This is used to indicate that the input is deprecated and should not be used. If a task is run with a deprecated input, a warning will be printed to the console. - -#### Templates - -When creating a task with `inputs` you can use [Go templates](https://pkg.go.dev/text/template#hdr-Functions) in that task's `actions`. For example: +For example, running `UDS_FOO=bar uds run echo-foo` on the following task will echo `bar`. ```yaml +variables: + - name: FOO + default: foo tasks: - - name: length-of-inputs - inputs: - hello-input: - default: hello world - description: This is an input to the echo-var task - another-input: - default: another world - actions: - # index and len are go template functions, while .inputs is map representing the inputs to the task - - cmd: echo ${{ index .inputs "hello-input" | len }} - - cmd: echo ${{ index .inputs "another-input" | len }} - - - name: len - actions: - - task: length-of-inputs - with: - hello-input: hello unicorn + - name: echo-foo + - cmd: echo ${FOO} ``` -Running `uds run len` will print the length of the inputs to `hello-input` and `another-input` to the console. +### No Dependency on Zarf +Since UDS CLI also vendors [Zarf](https://github.com/defenseunicorns/zarf), there is no need to also have Zarf installed on your system. diff --git a/go.mod b/go.mod index a2d70f9ce..a22f12707 100644 --- a/go.mod +++ b/go.mod @@ -5,18 +5,19 @@ go 1.21.6 require ( github.com/AlecAivazis/survey/v2 v2.3.7 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b - github.com/defenseunicorns/zarf v0.32.3 + github.com/defenseunicorns/maru-runner v0.0.1 + github.com/defenseunicorns/zarf v0.32.4 github.com/fsnotify/fsnotify v1.7.0 github.com/goccy/go-yaml v1.11.3 github.com/mholt/archiver/v3 v3.5.1 github.com/mholt/archiver/v4 v4.0.0-alpha.8 github.com/opencontainers/image-spec v1.1.0 - github.com/pterm/pterm v0.12.78 + github.com/pterm/pterm v0.12.79 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 github.com/stretchr/testify v1.8.4 - golang.org/x/exp v0.0.0-20240213143201-ec583247a57a - golang.org/x/mod v0.15.0 + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 + golang.org/x/mod v0.16.0 golang.org/x/sync v0.6.0 helm.sh/helm/v3 v3.14.2 oras.land/oras-go/v2 v2.4.0 @@ -85,7 +86,7 @@ require ( github.com/alibabacloud-go/tea-xml v1.1.3 // indirect github.com/aliyun/credentials-go v1.3.1 // indirect github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9 // indirect - github.com/anchore/clio v0.0.0-20240202120828-3ef5b3b40ea3 // indirect + github.com/anchore/clio v0.0.0-20240209204744-cb94e40a4f65 // indirect github.com/anchore/fangs v0.0.0-20231201140849-5075d28d6d8b // indirect github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a // indirect github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb // indirect diff --git a/go.sum b/go.sum index b7021dde9..e944ae07e 100644 --- a/go.sum +++ b/go.sum @@ -364,8 +364,8 @@ github.com/aliyun/credentials-go v1.3.1 h1:uq/0v7kWrxmoLGpqjx7vtQ/s03f0zR//0br/x github.com/aliyun/credentials-go v1.3.1/go.mod h1:8jKYhQuDawt8x2+fusqa1Y6mPxemTsBEN04dgcAcYz0= github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9 h1:p0ZIe0htYOX284Y4axJaGBvXHU0VCCzLN5Wf5XbKStU= github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9/go.mod h1:3ZsFB9tzW3vl4gEiUeuSOMDnwroWxIxJelOOHUp8dSw= -github.com/anchore/clio v0.0.0-20240202120828-3ef5b3b40ea3 h1:DfXgeXMJi1v3zyzgkftWK4a4FUc0QUtRGOrTvYBCPMs= -github.com/anchore/clio v0.0.0-20240202120828-3ef5b3b40ea3/go.mod h1:CdT/JVbhkK6cPZFxBjwsX4lHIYJXg+XCv+T0hndWrSw= +github.com/anchore/clio v0.0.0-20240209204744-cb94e40a4f65 h1:u9XrEabKlGPsrmRvAER+kUKkwXiJfLyqGhmOTFsXjX4= +github.com/anchore/clio v0.0.0-20240209204744-cb94e40a4f65/go.mod h1:8Jr7CjmwFVcBPtkJdTpaAGHimoGJGfbExypjzOu87Og= github.com/anchore/fangs v0.0.0-20231201140849-5075d28d6d8b h1:L/djgY7ZbZ/38+wUtdkk398W3PIBJLkt1N8nU/7e47A= github.com/anchore/fangs v0.0.0-20231201140849-5075d28d6d8b/go.mod h1:TLcE0RE5+8oIx2/NPWem/dq1DeaMoC+fPEH7hoSzPLo= github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a h1:nJ2G8zWKASyVClGVgG7sfM5mwoZlZ2zYpIzN2OhjWkw= @@ -598,8 +598,10 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX8ATG8oKsE= github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c= -github.com/defenseunicorns/zarf v0.32.3 h1:/jjWK3B1VfgBe114O7suWD2gOp9AA6emHgtzGzRvpvo= -github.com/defenseunicorns/zarf v0.32.3/go.mod h1:AjMjTPMIQT+bMeQ2R2kw9Q7KBdFUjO1MZj+26npmXVk= +github.com/defenseunicorns/maru-runner v0.0.1 h1:0SPiaXbPKnv7bjsUW2f7rPiIMmd3YLfT9+wQe4810K8= +github.com/defenseunicorns/maru-runner v0.0.1/go.mod h1:3K+JeLpud+rb8vC+nPFaTNjhqW40++6qFKKVTBEEzQM= +github.com/defenseunicorns/zarf v0.32.4 h1:3foCaUHUtAu8YId49j3u+EVknaTB8ERaQ9J6Do+bAwc= +github.com/defenseunicorns/zarf v0.32.4/go.mod h1:f4H7al7qnj5VXfkUkB/CcepVW/DA/O5tvAy8TWv9aT8= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da h1:ZOjWpVsFZ06eIhnh4mkaceTiVoktdU67+M7KDHJ268M= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da/go.mod h1:B3tI9iGHi4imdLi4Asdha1Sc6feLMTfPLXh9IUYmysk= github.com/depcheck-test/depcheck-test v0.0.0-20220607135614-199033aaa936 h1:foGzavPWwtoyBvjWyKJYDYsyzy+23iBV7NKTwdk+LRY= @@ -1456,8 +1458,8 @@ github.com/pterm/pterm v0.12.31/go.mod h1:32ZAWZVXD7ZfG0s8qqHXePte42kdz8ECtRyEej github.com/pterm/pterm v0.12.33/go.mod h1:x+h2uL+n7CP/rel9+bImHD5lF3nM9vJj80k9ybiiTTE= github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5bUw8T8= github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s= -github.com/pterm/pterm v0.12.78 h1:QTWKaIAa4B32GKwqVXtu9m1DUMgWw3VRljMkMevX+b8= -github.com/pterm/pterm v0.12.78/go.mod h1:1v/gzOF1N0FsjbgTHZ1wVycRkKiatFvJSJC4IGaQAAo= +github.com/pterm/pterm v0.12.79 h1:lH3yrYMhdpeqX9y5Ep1u7DejyHy7NSQg9qrBjF9dFT4= +github.com/pterm/pterm v0.12.79/go.mod h1:1v/gzOF1N0FsjbgTHZ1wVycRkKiatFvJSJC4IGaQAAo= github.com/rakyll/hey v0.1.4 h1:hhc8GIqHN4+rPFZvkM9lkCQGi7da0sINM83xxpFkbPA= github.com/rakyll/hey v0.1.4/go.mod h1:nAOTOo+L52KB9SZq/M6J18kxjto4yVtXQDjU2HgjUPI= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= @@ -1799,8 +1801,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240213143201-ec583247a57a h1:HinSgX1tJRX3KsL//Gxynpw5CTOAIPhgL4W8PNiIpVE= -golang.org/x/exp v0.0.0-20240213143201-ec583247a57a/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1829,8 +1831,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/src/cmd/root.go b/src/cmd/root.go index 1072735b2..93268fc73 100644 --- a/src/cmd/root.go +++ b/src/cmd/root.go @@ -10,6 +10,9 @@ import ( "runtime/debug" "strings" + runnerCLI "github.com/defenseunicorns/maru-runner/src/cmd" + runnerConfig "github.com/defenseunicorns/maru-runner/src/config" + "github.com/defenseunicorns/uds-cli/src/config" "github.com/defenseunicorns/uds-cli/src/config/lang" "github.com/defenseunicorns/uds-cli/src/pkg/utils" @@ -75,7 +78,7 @@ func init() { } } - // only vendor zarf if specifically invoked + // vendored Zarf command if len(os.Args) > 1 && (os.Args[1] == "zarf" || os.Args[1] == "z") { zarfCmd := &cobra.Command{ Use: "zarf COMMAND", @@ -93,6 +96,27 @@ func init() { return } + // vendored run command + if len(os.Args) > 1 && (os.Args[1] == "run" || os.Args[1] == "r") { + runnerCmd := &cobra.Command{ + Use: "run", + Aliases: []string{"r"}, + Run: func(_ *cobra.Command, _ []string) { + os.Args = os.Args[1:] // grab 'run' and onward from the CLI args + runnerConfig.CmdPrefix = "uds" // use vendored Zarf inside the runner + runnerConfig.EnvPrefix = "uds" + runnerCLI.RootCmd().SetArgs(os.Args) + runnerCLI.Execute() + }, + DisableFlagParsing: true, + } + rootCmd.AddCommand(runnerCmd) + + // disable UDS log file for the runner because the runner has its own log file + config.SkipLogFile = true + return + } + initViper() v.SetDefault(V_LOG_LEVEL, "info") diff --git a/src/cmd/run.go b/src/cmd/run.go deleted file mode 100644 index 430f20266..000000000 --- a/src/cmd/run.go +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2023-Present The UDS Authors - -package cmd - -import ( - "fmt" - "os" - "strings" - - "github.com/defenseunicorns/uds-cli/src/config" - "github.com/defenseunicorns/uds-cli/src/config/lang" - "github.com/defenseunicorns/uds-cli/src/pkg/runner" - "github.com/defenseunicorns/uds-cli/src/types" - "github.com/defenseunicorns/zarf/src/cmd/common" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" - "github.com/pterm/pterm" - "github.com/spf13/cobra" -) - -// runCmd represents the run command -var runCmd = &cobra.Command{ - Use: "run [ TASK NAME ]", - Short: "run a task", - Long: `run a task from an tasks file`, - ValidArgsFunction: func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { - var tasksFile types.TasksFile - - if _, err := os.Stat(config.TaskFileLocation); os.IsNotExist(err) { - return []string{}, cobra.ShellCompDirectiveNoFileComp - } - - err := utils.ReadYaml(config.TaskFileLocation, &tasksFile) - if err != nil { - return []string{}, cobra.ShellCompDirectiveNoFileComp - } - - var taskNames []string - for _, task := range tasksFile.Tasks { - taskNames = append(taskNames, task.Name) - } - return taskNames, cobra.ShellCompDirectiveNoFileComp - }, - Args: func(_ *cobra.Command, args []string) error { - if len(args) > 1 && !config.ListTasks { - return fmt.Errorf("accepts 0 or 1 arg(s), received %d", len(args)) - } - return nil - }, - Run: func(_ *cobra.Command, args []string) { - var tasksFile types.TasksFile - - if _, err := os.Stat(config.TaskFileLocation); os.IsNotExist(err) { - message.Fatalf(err, "%s not found", config.TaskFileLocation) - } - - // Ensure uppercase keys from viper - v := common.GetViper() - config.SetRunnerVariables = helpers.TransformAndMergeMap( - v.GetStringMapString(common.VPkgCreateSet), config.SetRunnerVariables, strings.ToUpper) - - err := utils.ReadYaml(config.TaskFileLocation, &tasksFile) - if err != nil { - message.Fatalf(err, "Cannot unmarshal %s", config.TaskFileLocation) - } - - if config.ListTasks { - rows := [][]string{ - {"Name", "Description"}, - } - for _, task := range tasksFile.Tasks { - rows = append(rows, []string{task.Name, task.Description}) - } - err := pterm.DefaultTable.WithHasHeader().WithData(rows).Render() - if err != nil { - message.Fatal(err, "error listing tasks") - } - - os.Exit(0) - } - - taskName := "default" - if len(args) > 0 { - taskName = args[0] - } - if err := runner.Run(tasksFile, taskName, config.SetRunnerVariables); err != nil { - message.Fatalf(err, "Failed to run action: %s", err) - } - }, -} - -func init() { - initViper() - rootCmd.AddCommand(runCmd) - runFlags := runCmd.Flags() - runFlags.StringVarP(&config.TaskFileLocation, "file", "f", config.TasksYAML, lang.CmdRunFlag) - runFlags.BoolVar(&config.ListTasks, "list", false, lang.CmdRunList) - runFlags.StringToStringVar(&config.SetRunnerVariables, "set", nil, lang.CmdRunSetVarFlag) -} diff --git a/src/cmd/uds.go b/src/cmd/uds.go index e6630f08f..ce266a7ec 100644 --- a/src/cmd/uds.go +++ b/src/cmd/uds.go @@ -255,8 +255,7 @@ func configureZarf() { TempDirectory: config.CommonOptions.TempDirectory, OCIConcurrency: config.CommonOptions.OCIConcurrency, Confirm: config.CommonOptions.Confirm, - // todo: decouple Zarf cache? - CachePath: config.CommonOptions.CachePath, + CachePath: config.CommonOptions.CachePath, // use uds-cache instead of zarf-cache } } diff --git a/src/config/config.go b/src/config/config.go index ae9c6101e..c122fb200 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -47,6 +47,9 @@ const ( // UDSCache is the directory containing cached bundle layers UDSCache = ".uds-cache" + // UDSCacheLayers is the directory in the cache containing cached bundle layers + UDSCacheLayers = "layers" + // TasksYAML is the default name of the uds run cmd file TasksYAML = "tasks.yaml" diff --git a/src/pkg/bundle/common.go b/src/pkg/bundle/common.go index 2095af3b6..862769412 100644 --- a/src/pkg/bundle/common.go +++ b/src/pkg/bundle/common.go @@ -144,7 +144,10 @@ func (b *Bundle) ValidateBundleResources(bundle *types.UDSBundle, spinner *messa return err } if err := remote.Repo().Reference.ValidateReferenceAsDigest(); err != nil { - manifestDesc, _ := remote.ResolveRoot() + manifestDesc, err := remote.ResolveRoot() + if err != nil { + return err + } // todo: don't do this here, a "validate" fn shouldn't be modifying the bundle bundle.Packages[idx].Ref = pkg.Ref + "@sha256:" + manifestDesc.Digest.Encoded() } diff --git a/src/pkg/bundle/common_test.go b/src/pkg/bundle/common_test.go index 2ef1e5fea..3f17b88a5 100644 --- a/src/pkg/bundle/common_test.go +++ b/src/pkg/bundle/common_test.go @@ -27,7 +27,8 @@ func Test_validateBundleVars(t *testing.T) { }, }, wantErr: false, - }, { + }, + { name: "ImportDoesntMatchExport", description: "error when import doesn't match export", args: args{ @@ -37,7 +38,8 @@ func Test_validateBundleVars(t *testing.T) { }, }, wantErr: true, - }, { + }, + { name: "FirstPkgHasImport", description: "error when first pkg has an import", args: args{ diff --git a/src/pkg/bundle/deploy.go b/src/pkg/bundle/deploy.go index 972745256..cdb834fda 100644 --- a/src/pkg/bundle/deploy.go +++ b/src/pkg/bundle/deploy.go @@ -35,15 +35,6 @@ type ZarfOverrideMap map[string]map[string]map[string]interface{} var templatedVarRegex = regexp.MustCompile(`\${([^}]+)}`) // Deploy deploys a bundle -// -// : create a new provider -// : pull the bundle's metadata + sig -// : read the metadata into memory -// : validate the sig (if present) -// : loop through each package -// : : load the package into a fresh temp dir -// : : validate the sig (if present) -// : : deploy the package func (b *Bundle) Deploy() error { ctx := context.TODO() @@ -227,7 +218,7 @@ func (b *Bundle) loadVariables(pkg types.Package, bundleExportedVars map[string] } } - // Set variables in order or precendence (least specific to most specific) + // Set variables in order or precedence (least specific to most specific) // imported vars for _, imp := range pkg.Imports { pkgVars[strings.ToUpper(imp.Name)] = bundleExportedVars[imp.Package][imp.Name] @@ -356,6 +347,7 @@ func (b *Bundle) processOverrideVariables(overrideMap *map[string]map[string]*va var overrideVal interface{} // Ensuring variable name is upper case since comparisons are being done against upper case env and config variables v.Name = strings.ToUpper(v.Name) + // check for override in env vars if envVarOverride, exists := os.LookupEnv(strings.ToUpper(config.EnvVarPrefix + v.Name)); exists { if err := addOverrideValue(*overrideMap, componentName, chartName, v.Path, envVarOverride, nil); err != nil { diff --git a/src/pkg/bundle/deploy_test.go b/src/pkg/bundle/deploy_test.go new file mode 100644 index 000000000..f4d7b697f --- /dev/null +++ b/src/pkg/bundle/deploy_test.go @@ -0,0 +1,241 @@ +package bundle + +import ( + "os" + "reflect" + "testing" + + "github.com/defenseunicorns/uds-cli/src/types" +) + +func TestLoadVariablesPrecedence(t *testing.T) { + testCases := []struct { + name string + description string + pkg types.Package + bundle Bundle + bundleExportVars map[string]map[string]string + loadEnvVar bool + expectedPkgVars map[string]string + }{ + { + name: "--set flag precedence", + loadEnvVar: true, + pkg: types.Package{ + Name: "fooPkg", + Imports: []types.BundleVariableImport{ + { + Name: "foo", + Package: "bazPkg", + }, + }, + }, + bundle: Bundle{ + cfg: &types.BundleConfig{ + DeployOpts: types.BundleDeployOptions{ + Variables: map[string]map[string]interface{}{ + "fooPkg": { + "foo": "set from variables key in uds-config.yaml", + }, + }, + // set from uds-config.yaml + SharedVariables: map[string]interface{}{ + "foo": "set from shared key in uds-config.yaml", + }, + SetVariables: map[string]string{ + "foo": "set using --set flag", + }, + }, + }, + }, + bundleExportVars: map[string]map[string]string{ + "barPkg": { + "foo": "exported from another pkg", + }, + "bazPkg": { + "foo": "imported from a specific pkg", + }, + }, + expectedPkgVars: map[string]string{ + "FOO": "set using --set flag", + }, + }, + { + name: "env var precedence", + loadEnvVar: true, + pkg: types.Package{ + Name: "fooPkg", + Imports: []types.BundleVariableImport{ + { + Name: "foo", + Package: "bazPkg", + }, + }, + }, + bundle: Bundle{ + cfg: &types.BundleConfig{ + DeployOpts: types.BundleDeployOptions{ + Variables: map[string]map[string]interface{}{ + "fooPkg": { + "foo": "set from variables key in uds-config.yaml", + }, + }, + // set from uds-config.yaml + SharedVariables: map[string]interface{}{ + "foo": "set from shared key in uds-config.yaml", + }, + }, + }, + }, + bundleExportVars: map[string]map[string]string{ + "barPkg": { + "foo": "exported from another pkg", + }, + "bazPkg": { + "foo": "imported from a specific pkg", + }, + }, + expectedPkgVars: map[string]string{ + "FOO": "set using env var", + }, + }, + { + name: "uds-config variables key precedence", + pkg: types.Package{ + Name: "fooPkg", + Imports: []types.BundleVariableImport{ + { + Name: "foo", + Package: "bazPkg", + }, + }, + }, + bundle: Bundle{ + cfg: &types.BundleConfig{ + DeployOpts: types.BundleDeployOptions{ + Variables: map[string]map[string]interface{}{ + "fooPkg": { + "foo": "set from variables key in uds-config.yaml", + }, + }, + // set from uds-config.yaml + SharedVariables: map[string]interface{}{ + "foo": "set from shared key in uds-config.yaml", + }, + }, + }, + }, + bundleExportVars: map[string]map[string]string{ + "barPkg": { + "foo": "exported from another pkg", + }, + "bazPkg": { + "foo": "imported from a specific pkg", + }, + }, + expectedPkgVars: map[string]string{ + "FOO": "set from variables key in uds-config.yaml", + }, + }, + { + name: "uds-config shared key precedence", + pkg: types.Package{ + Name: "fooPkg", + Imports: []types.BundleVariableImport{ + { + Name: "foo", + Package: "bazPkg", + }, + }, + }, + bundle: Bundle{ + cfg: &types.BundleConfig{ + DeployOpts: types.BundleDeployOptions{ + // set from uds-config.yaml + SharedVariables: map[string]interface{}{ + "foo": "set from shared key in uds-config.yaml", + }, + }, + }, + }, + bundleExportVars: map[string]map[string]string{ + "barPkg": { + "foo": "exported from another pkg", + }, + "bazPkg": { + "foo": "imported from a specific pkg", + }, + }, + expectedPkgVars: map[string]string{ + "FOO": "set from shared key in uds-config.yaml", + }, + }, + { + name: "uds-config shared key precedence", + pkg: types.Package{ + Name: "fooPkg", + Imports: []types.BundleVariableImport{ + { + Name: "foo", + Package: "bazPkg", + }, + }, + }, + bundle: Bundle{ + cfg: &types.BundleConfig{ + DeployOpts: types.BundleDeployOptions{ + SharedVariables: nil, + }, + }, + }, + bundleExportVars: map[string]map[string]string{ + "barPkg": { + "foo": "exported from another pkg", + }, + "bazPkg": { + "foo": "imported from a specific pkg", + }, + }, + expectedPkgVars: map[string]string{ + "FOO": "imported from a specific pkg", + }, + }, + { + name: "uds-config global export precedence", + pkg: types.Package{ + Name: "fooPkg", + }, + bundle: Bundle{ + cfg: &types.BundleConfig{ + DeployOpts: types.BundleDeployOptions{ + SharedVariables: nil, + }, + }, + }, + bundleExportVars: map[string]map[string]string{ + "barPkg": { + "foo": "exported from another pkg", + }, + }, + expectedPkgVars: map[string]string{ + "FOO": "exported from another pkg", + }, + }, + } + + // Run test cases + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Set for select test cases to test precedence of env vars + os.Unsetenv("UDS_FOO") + if tc.loadEnvVar { + os.Setenv("UDS_FOO", "set using env var") + } + actualPkgVars := tc.bundle.loadVariables(tc.pkg, tc.bundleExportVars) + + if !reflect.DeepEqual(actualPkgVars, tc.expectedPkgVars) { + t.Errorf("Test case %s failed. Expected %v, got %v", tc.name, tc.expectedPkgVars, actualPkgVars) + } + }) + } +} diff --git a/src/pkg/bundle/provider.go b/src/pkg/bundle/provider.go index 214de09c8..0d5fe3c2a 100644 --- a/src/pkg/bundle/provider.go +++ b/src/pkg/bundle/provider.go @@ -34,7 +34,7 @@ type Provider interface { // LoadBundle loads a bundle into the temporary directory and returns a map of the bundle's files // // (currently only the remote provider utilizes the concurrency parameter) - LoadBundle(concurrency int) (types.PathMap, error) + LoadBundle(options types.BundlePullOptions, concurrency int) (*types.UDSBundle, types.PathMap, error) // CreateBundleSBOM creates a bundle-level SBOM from the underlying Zarf packages, if the Zarf package contains an SBOM CreateBundleSBOM(extractSBOM bool) error @@ -43,7 +43,7 @@ type Provider interface { PublishBundle(bundle types.UDSBundle, remote *oci.OrasRemote) error // getBundleManifest gets the bundle's root manifest - getBundleManifest() error + getBundleManifest() (*oci.ZarfOCIManifest, error) // ZarfPackageNameMap returns a map of the zarf package name specified in the uds-bundle.yaml to the actual zarf package name ZarfPackageNameMap() (map[string]string, error) @@ -52,20 +52,34 @@ type Provider interface { // NewBundleProvider returns a new bundler Provider based on the source type func NewBundleProvider(ctx context.Context, source, destination string) (Provider, error) { if helpers.IsOCIURL(source) { - provider := ociProvider{ctx: ctx, src: source, dst: destination} + op := ociProvider{ctx: ctx, src: source, dst: destination} platform := ocispec.Platform{ Architecture: config.GetArch(), OS: oci.MultiOS, } + // get remote client remote, err := oci.NewOrasRemote(source, platform) if err != nil { return nil, err } - provider.OrasRemote = remote - return &provider, nil + op.OrasRemote = remote + + // get root manifest + root, err := op.FetchRoot() + if err != nil { + return nil, err + } + op.rootManifest = root + + return &op, nil } if !utils.IsValidTarballPath(source) { return nil, fmt.Errorf("invalid tarball path: %s", source) } - return &tarballBundleProvider{ctx: ctx, src: source, dst: destination}, nil + tp := tarballBundleProvider{ctx: ctx, src: source, dst: destination} + err := tp.loadBundleManifest() + if err != nil { + return nil, err + } + return &tp, nil } diff --git a/src/pkg/bundle/pull.go b/src/pkg/bundle/pull.go index ed6bf690a..0322ff04e 100644 --- a/src/pkg/bundle/pull.go +++ b/src/pkg/bundle/pull.go @@ -21,11 +21,12 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// Pull pulls a bundle and saves it locally + caches it +// Pull pulls a bundle and saves it locally func (b *Bundle) Pull() error { + // use uds-cache/packages as the dst dir for the pull to get auto caching + // we use an ORAS ocistore to make that dir look like an OCI artifact cacheDir := filepath.Join(zarfConfig.GetAbsCachePath(), "packages") - // create the cache directory if it doesn't exist - if err := utils.CreateDirectory(cacheDir, 0755); err != nil { + if err := utils.CreateDirectory(cacheDir, 0o755); err != nil { return err } @@ -41,28 +42,12 @@ func (b *Bundle) Pull() error { return err } - // pull the bundle's metadata + sig - loadedMetadata, err := provider.LoadBundleMetadata() - if err != nil { - return err - } - if err := utils.ReadYaml(loadedMetadata[config.BundleYAML], &b.bundle); err != nil { - return err - } - - // validate the sig (if present) - if err := ValidateBundleSignature(loadedMetadata[config.BundleYAML], loadedMetadata[config.BundleYAMLSignature], b.cfg.PullOpts.PublicKeyPath); err != nil { - return err - } - // pull the bundle's uds-bundle.yaml and it's Zarf pkgs - // todo: refactor this fn, think about pulling the rootDesc first and getting the hashes from there - // today, we are getting the Zarf image manifest hashes from the uds-bundle.yaml - // in that logic we end up pulling the root manifest twice, once in LoadBundle and the other below in remote.ResolveRoot() - loaded, err := provider.LoadBundle(zarfConfig.CommonOptions.OCIConcurrency) + bundle, loaded, err := provider.LoadBundle(b.cfg.PullOpts, zarfConfig.CommonOptions.OCIConcurrency) if err != nil { return err } + b.bundle = *bundle // create a remote client just to resolve the root descriptor platform := ocispec.Platform{ diff --git a/src/pkg/bundle/remote.go b/src/pkg/bundle/remote.go index 74ad523f6..0ea38535e 100644 --- a/src/pkg/bundle/remote.go +++ b/src/pkg/bundle/remote.go @@ -43,19 +43,14 @@ type ociProvider struct { src string dst string *oci.OrasRemote - manifest *oci.ZarfOCIManifest + rootManifest *oci.ZarfOCIManifest } -func (op *ociProvider) getBundleManifest() error { - if op.manifest != nil { - return nil +func (op *ociProvider) getBundleManifest() (*oci.ZarfOCIManifest, error) { + if op.rootManifest != nil { + return op.rootManifest, nil } - root, err := op.FetchRoot() - if err != nil { - return err - } - op.manifest = root - return nil + return nil, fmt.Errorf("bundle root manifest not loaded") } // LoadBundleMetadata loads a remote bundle's metadata @@ -79,10 +74,6 @@ func (op *ociProvider) LoadBundleMetadata() (types.PathMap, error) { } loaded[rel] = absSha } - err = op.getBundleManifest() - if err != nil { - return nil, err - } return loaded, nil } @@ -149,55 +140,60 @@ func (op *ociProvider) CreateBundleSBOM(extractSBOM bool) error { return nil } -// LoadBundle loads a bundle's uds-bundle.yaml and Zarf packages from a remote source -func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) { - var layersToPull []ocispec.Descriptor - estimatedBytes := int64(0) - - if err := op.getBundleManifest(); err != nil { - return nil, err - } - - loaded, err := op.LoadBundleMetadata() // todo: remove? this seems redundant, can we pass the "loaded" var in +// LoadBundle loads a bundle from a remote source +func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.UDSBundle, types.PathMap, error) { + var bundle types.UDSBundle + // pull the bundle's metadata + sig + loaded, err := op.LoadBundleMetadata() if err != nil { - return nil, err + return nil, nil, err + } + if err := zarfUtils.ReadYaml(loaded[config.BundleYAML], &bundle); err != nil { + return nil, nil, err } - b, err := os.ReadFile(loaded[config.BundleYAML]) - if err != nil { - return nil, err + // validate the sig (if present) before pulling the whole bundle + if err := ValidateBundleSignature(loaded[config.BundleYAML], loaded[config.BundleYAMLSignature], opts.PublicKeyPath); err != nil { + return nil, nil, err } - var bundle types.UDSBundle - if err := goyaml.Unmarshal(b, &bundle); err != nil { - return nil, err + var layersToPull []ocispec.Descriptor + estimatedBytes := int64(0) + + // get the bundle's root manifest + rootManifest, err := op.getBundleManifest() + if err != nil { + return nil, nil, err } for _, pkg := range bundle.Packages { + // grab sha of zarf image manifest and pull it down sha := strings.Split(pkg.Ref, "@sha256:")[1] // this is where we use the SHA appended to the Zarf pkg inside the bundle - manifestDesc := op.manifest.Locate(sha) + manifestDesc := rootManifest.Locate(sha) if err != nil { - return nil, err + return nil, nil, err } manifestBytes, err := op.FetchLayer(manifestDesc) if err != nil { - return nil, err + return nil, nil, err } + // unmarshal the zarf image manifest and add it to the layers to pull var manifest oci.ZarfOCIManifest if err := json.Unmarshal(manifestBytes, &manifest); err != nil { - return nil, err + return nil, nil, err } layersToPull = append(layersToPull, manifestDesc) progressBar := message.NewProgressBar(int64(len(manifest.Layers)), fmt.Sprintf("Verifying layers in Zarf package: %s", pkg.Name)) + // go through the layers in the zarf image manifest and check if they exist in the remote for _, layer := range manifest.Layers { ok, err := op.Repo().Blobs().Exists(op.ctx, layer) progressBar.Add(1) estimatedBytes += layer.Size if err != nil { - return nil, err + return nil, nil, err } // if the layer exists in the remote, add it to the layers to pull if ok { @@ -209,13 +205,13 @@ func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) { store, err := ocistore.NewWithContext(op.ctx, op.dst) if err != nil { - return nil, err + return nil, nil, err } // grab the bundle root manifest and add it to the layers to pull rootDesc, err := op.ResolveRoot() if err != nil { - return nil, err + return nil, nil, err } layersToPull = append(layersToPull, rootDesc) @@ -232,7 +228,7 @@ func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) { _, err = oras.Copy(op.ctx, op.Repo(), op.Repo().Reference.String(), store, op.Repo().Reference.String(), copyOpts) if err != nil { doneSaving <- 1 - return nil, err + return nil, nil, err } doneSaving <- 1 @@ -243,7 +239,7 @@ func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) { loaded[sha] = filepath.Join(op.dst, config.BlobsDir, sha) } - return loaded, nil + return &bundle, loaded, nil } func (op *ociProvider) PublishBundle(_ types.UDSBundle, _ *oci.OrasRemote) error { @@ -337,7 +333,8 @@ func CheckOCISourcePath(source string) (string, error) { // ZarfPackageNameMap returns the uds bundle zarf package name to actual zarf package name mappings from the oci provider func (op *ociProvider) ZarfPackageNameMap() (map[string]string, error) { - if err := op.getBundleManifest(); err != nil { + rootManifest, err := op.getBundleManifest() + if err != nil { return nil, err } @@ -359,7 +356,7 @@ func (op *ociProvider) ZarfPackageNameMap() (map[string]string, error) { nameMap := make(map[string]string) for _, pkg := range bundle.Packages { sha := strings.Split(pkg.Ref, "@sha256:")[1] // this is where we use the SHA appended to the Zarf pkg inside the bundle - manifestDesc := op.manifest.Locate(sha) + manifestDesc := rootManifest.Locate(sha) nameMap[manifestDesc.Annotations[config.UDSPackageNameAnnotation]] = manifestDesc.Annotations[config.ZarfPackageNameAnnotation] } return nameMap, nil diff --git a/src/pkg/bundle/remove.go b/src/pkg/bundle/remove.go index f6c5876ca..6896bfcc9 100644 --- a/src/pkg/bundle/remove.go +++ b/src/pkg/bundle/remove.go @@ -81,14 +81,18 @@ func (b *Bundle) Remove() error { } func removePackages(packagesToRemove []types.Package, b *Bundle, zarfPackageNameMap map[string]string) error { - // Get deployed packages deployedPackageNames := GetDeployedPackageNames() for i := len(packagesToRemove) - 1; i >= 0; i-- { pkg := packagesToRemove[i] - zarfPackageName := zarfPackageNameMap[pkg.Name] + zarfPackageName := pkg.Name + // use the name map if it has been set (remote pkgs where the pkg name isn't consistent) + if _, ok := zarfPackageNameMap[pkg.Name]; ok { + zarfPackageName = zarfPackageNameMap[pkg.Name] + } + if slices.Contains(deployedPackageNames, zarfPackageName) { opts := zarfTypes.ZarfPackageOptions{ PackageSource: b.cfg.RemoveOpts.Source, diff --git a/src/pkg/bundle/tarball.go b/src/pkg/bundle/tarball.go index f7b53dd3b..378be761f 100644 --- a/src/pkg/bundle/tarball.go +++ b/src/pkg/bundle/tarball.go @@ -26,28 +26,30 @@ import ( ) type tarballBundleProvider struct { - ctx context.Context - src string - dst string - bundleRootManifest *oci.ZarfOCIManifest - bundleRootDesc ocispec.Descriptor + ctx context.Context + src string + dst string + + // these fields are populated by loadBundleManifest as part of the provider constructor + bundleRootDesc ocispec.Descriptor + rootManifest *oci.ZarfOCIManifest } // CreateBundleSBOM creates a bundle-level SBOM from the underlying Zarf packages, if the Zarf package contains an SBOM func (tp *tarballBundleProvider) CreateBundleSBOM(extractSBOM bool) error { - err := tp.getBundleManifest() + rootManifest, err := tp.getBundleManifest() if err != nil { return err } // make tmp dir for pkg SBOM extraction - err = os.Mkdir(filepath.Join(tp.dst, config.BundleSBOM), 0700) + err = os.Mkdir(filepath.Join(tp.dst, config.BundleSBOM), 0o700) if err != nil { return err } SBOMArtifactPathMap := make(types.PathMap) containsSBOMs := false - for _, layer := range tp.bundleRootManifest.Layers { + for _, layer := range rootManifest.Layers { // get Zarf image manifests from bundle manifest if layer.Annotations[ocispec.AnnotationTitle] == config.BundleYAML { continue @@ -113,30 +115,38 @@ func (tp *tarballBundleProvider) CreateBundleSBOM(extractSBOM bool) error { return nil } -func (tp *tarballBundleProvider) getBundleManifest() error { - if tp.bundleRootManifest != nil { - return nil +func (tp *tarballBundleProvider) getBundleManifest() (*oci.ZarfOCIManifest, error) { + if tp.rootManifest != nil { + return tp.rootManifest, nil } + return nil, fmt.Errorf("bundle root manifest not loaded") +} - if err := av3.Extract(tp.src, "index.json", tp.dst); err != nil { - return fmt.Errorf("failed to extract index.json from %s: %w", tp.src, err) +// loadBundleManifest loads the bundle's root manifest and desc into the tarballBundleProvider so we don't have to load it multiple times +func (tp *tarballBundleProvider) loadBundleManifest() error { + // Create a secure temporary directory for handling files + secureTempDir, err := zarfUtils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return fmt.Errorf("failed to create a secure temporary directory: %w", err) } + defer os.RemoveAll(secureTempDir) // Ensure cleanup of the temp directory - indexPath := filepath.Join(tp.dst, "index.json") + if err := av3.Extract(tp.src, "index.json", secureTempDir); err != nil { + return fmt.Errorf("failed to extract index.json from %s: %w", tp.src, err) + } + indexPath := filepath.Join(secureTempDir, "index.json") defer os.Remove(indexPath) b, err := os.ReadFile(indexPath) if err != nil { - return err + return fmt.Errorf("failed to read index.json: %w", err) } var index ocispec.Index - if err := json.Unmarshal(b, &index); err != nil { - return err + return fmt.Errorf("failed to unmarshal index.json: %w", err) } - // local bundles only have one manifest entry in their index.json bundleManifestDesc := index.Manifests[0] tp.bundleRootDesc = bundleManifestDesc @@ -147,11 +157,11 @@ func (tp *tarballBundleProvider) getBundleManifest() error { manifestRelativePath := filepath.Join(config.BlobsDir, bundleManifestDesc.Digest.Encoded()) - if err := av3.Extract(tp.src, manifestRelativePath, tp.dst); err != nil { + if err := av3.Extract(tp.src, manifestRelativePath, secureTempDir); err != nil { return fmt.Errorf("failed to extract %s from %s: %w", bundleManifestDesc.Digest.Encoded(), tp.src, err) } - manifestPath := filepath.Join(tp.dst, manifestRelativePath) + manifestPath := filepath.Join(secureTempDir, manifestRelativePath) defer os.Remove(manifestPath) @@ -170,18 +180,19 @@ func (tp *tarballBundleProvider) getBundleManifest() error { return err } - tp.bundleRootManifest = manifest + tp.rootManifest = manifest return nil } // LoadBundle loads a bundle from a tarball -func (tp *tarballBundleProvider) LoadBundle(_ int) (types.PathMap, error) { - return nil, fmt.Errorf("uds pull does not support pulling local bundles") +func (tp *tarballBundleProvider) LoadBundle(_ types.BundlePullOptions, _ int) (*types.UDSBundle, types.PathMap, error) { + return nil, nil, fmt.Errorf("uds pull does not support pulling local bundles") } // LoadBundleMetadata loads a bundle's metadata from a tarball func (tp *tarballBundleProvider) LoadBundleMetadata() (types.PathMap, error) { - if err := tp.getBundleManifest(); err != nil { + bundleRootManifest, err := tp.getBundleManifest() + if err != nil { return nil, err } pathsToExtract := config.BundleAlwaysPull @@ -189,7 +200,7 @@ func (tp *tarballBundleProvider) LoadBundleMetadata() (types.PathMap, error) { loaded := make(types.PathMap) for _, path := range pathsToExtract { - layer := tp.bundleRootManifest.Locate(path) + layer := bundleRootManifest.Locate(path) if !oci.IsEmptyDescriptor(layer) { pathInTarball := filepath.Join(config.BlobsDir, layer.Digest.Encoded()) abs := filepath.Join(tp.dst, pathInTarball) @@ -237,7 +248,8 @@ func (tp *tarballBundleProvider) getZarfLayers(store *ocistore.Store, pkgManifes // PublishBundle publishes a local bundle to a remote OCI registry func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *oci.OrasRemote) error { var layersToPush []ocispec.Descriptor - if err := tp.getBundleManifest(); err != nil { + bundleRootManifest, err := tp.getBundleManifest() + if err != nil { return err } estimatedBytes := int64(0) @@ -248,7 +260,7 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o return err } // push bundle layers to remote - for _, manifestDesc := range tp.bundleRootManifest.Layers { + for _, manifestDesc := range bundleRootManifest.Layers { layersToPush = append(layersToPush, manifestDesc) if manifestDesc.Annotations[ocispec.AnnotationTitle] == config.BundleYAML { continue // uds-bundle.yaml doesn't have layers @@ -262,7 +274,7 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o } // grab image config - layersToPush = append(layersToPush, tp.bundleRootManifest.Config) + layersToPush = append(layersToPush, bundleRootManifest.Config) // copy bundle copyOpts := utils.CreateCopyOpts(layersToPush, config.CommonOptions.OCIConcurrency) @@ -297,12 +309,13 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o // ZarfPackageNameMap gets zarf package name mappings from tarball provider func (tp *tarballBundleProvider) ZarfPackageNameMap() (map[string]string, error) { - if err := tp.getBundleManifest(); err != nil { + bundleRootManifest, err := tp.getBundleManifest() + if err != nil { return nil, err } nameMap := make(map[string]string) - for _, layer := range tp.bundleRootManifest.Layers { + for _, layer := range bundleRootManifest.Layers { if layer.MediaType == oci.ZarfLayerMediaTypeBlob { // only the uds bundle layer will have AnnotationTitle set if layer.Annotations[ocispec.AnnotationTitle] != config.BundleYAML { diff --git a/src/pkg/bundler/fetcher/local.go b/src/pkg/bundler/fetcher/local.go index aa19a9d03..580cb9b2f 100644 --- a/src/pkg/bundler/fetcher/local.go +++ b/src/pkg/bundler/fetcher/local.go @@ -173,6 +173,16 @@ func (f *localFetcher) toBundle(pkg zarfTypes.ZarfPackage, pkgTmp string) ([]oci // adds title annotations to descs and creates layer to put in the store // title annotations need to be added to the pkg root manifest // Zarf image manifests already contain those title annotations in remote OCI repos, but they need to be added manually here + + // if using a custom tmp dir that is not an absolute path, get working dir and prepend to path to make it absolute + if !filepath.IsAbs(path) { + wd, err := os.Getwd() + if err != nil { + return nil, err + } + path = filepath.Join(wd, path) + } + desc, err := src.Add(ctx, name, mediaType, path) if err != nil { return nil, err diff --git a/src/pkg/cache/cache.go b/src/pkg/cache/cache.go index 8e0520ecd..88f58e285 100644 --- a/src/pkg/cache/cache.go +++ b/src/pkg/cache/cache.go @@ -30,7 +30,7 @@ func expandTilde(cachePath string) string { func Add(filePathToAdd string) error { // ensure cache dir exists cacheDir := config.CommonOptions.CachePath - if err := os.MkdirAll(filepath.Join(cacheDir, "images"), 0755); err != nil { + if err := os.MkdirAll(filepath.Join(cacheDir, config.UDSCacheLayers), 0o755); err != nil { return err } @@ -46,7 +46,7 @@ func Add(filePathToAdd string) error { } defer srcFile.Close() - dstFile, err := os.Create(filepath.Join(cacheDir, "images", filename)) + dstFile, err := os.Create(filepath.Join(cacheDir, config.UDSCacheLayers, filename)) if err != nil { return err } @@ -58,7 +58,7 @@ func Add(filePathToAdd string) error { // Exists checks if a layer exists in the cache func Exists(layerDigest string) bool { cacheDir := config.CommonOptions.CachePath - layerCachePath := filepath.Join(expandTilde(cacheDir), "images", layerDigest) + layerCachePath := filepath.Join(expandTilde(cacheDir), config.UDSCacheLayers, layerDigest) _, err := os.Stat(layerCachePath) return !os.IsNotExist(err) } @@ -66,7 +66,7 @@ func Exists(layerDigest string) bool { // Use copies a layer from the cache to the dst dir func Use(layerDigest, dstDir string) error { cacheDir := config.CommonOptions.CachePath - layerCachePath := filepath.Join(expandTilde(cacheDir), "images", layerDigest) + layerCachePath := filepath.Join(expandTilde(cacheDir), config.UDSCacheLayers, layerDigest) srcFile, err := os.Open(layerCachePath) if err != nil { return err @@ -74,7 +74,7 @@ func Use(layerDigest, dstDir string) error { defer srcFile.Close() // ensure blobs/sha256 dir has been created - if err := os.MkdirAll(dstDir, 0755); err != nil { + if err := os.MkdirAll(dstDir, 0o755); err != nil { return err } diff --git a/src/pkg/runner/runner.go b/src/pkg/runner/runner.go deleted file mode 100644 index 16178a7c9..000000000 --- a/src/pkg/runner/runner.go +++ /dev/null @@ -1,769 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -// Package runner provides functions for running tasks in a run.yaml -package runner - -import ( - "context" - "fmt" - "os" - "path/filepath" - "regexp" - "strings" - "text/template" - "time" - - // used for compile time directives to pull functions from Zarf - _ "unsafe" - - "github.com/defenseunicorns/uds-cli/src/config" - "github.com/defenseunicorns/uds-cli/src/types" - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/message" - zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" - zarfTypes "github.com/defenseunicorns/zarf/src/types" - goyaml "github.com/goccy/go-yaml" - "github.com/mholt/archiver/v3" -) - -// Runner holds the necessary data to run tasks from a tasks file -type Runner struct { - TemplateMap map[string]*zarfUtils.TextTemplate - TasksFile types.TasksFile - TaskNameMap map[string]bool - envFilePath string -} - -// Run runs a task from tasks file -func Run(tasksFile types.TasksFile, taskName string, setVariables map[string]string) error { - runner := Runner{ - TemplateMap: map[string]*zarfUtils.TextTemplate{}, - TasksFile: tasksFile, - TaskNameMap: map[string]bool{}, - } - - runner.populateTemplateMap(tasksFile.Variables, setVariables) - - task, err := runner.getTask(taskName) - if err != nil { - return err - } - - // can't call a task directly from the CLI if it has inputs - if task.Inputs != nil { - return fmt.Errorf("task '%s' contains 'inputs' and cannot be called directly by the CLI", taskName) - } - - if err = runner.checkForTaskLoops(task, tasksFile, setVariables); err != nil { - return err - } - - err = runner.executeTask(task) - return err -} - -func (r *Runner) processIncludes(tasksFile types.TasksFile, setVariables map[string]string, action types.Action) error { - if strings.Contains(action.TaskReference, ":") { - taskReferenceName := strings.Split(action.TaskReference, ":")[0] - for _, include := range tasksFile.Includes { - if include[taskReferenceName] != "" { - referencedIncludes := []map[string]string{include} - err := r.importTasks(referencedIncludes, config.TaskFileLocation, setVariables) - if err != nil { - return err - } - break - } - } - } - return nil -} - -func (r *Runner) importTasks(includes []map[string]string, dir string, setVariables map[string]string) error { - // iterate through includes, open the file, and unmarshal it into a Task - var includeFilenameKey string - var includeFilename string - dir = filepath.Dir(dir) - for _, include := range includes { - if len(include) > 1 { - return fmt.Errorf("included item %s must have only one key", include) - } - // grab first and only value from include map - for k, v := range include { - includeFilenameKey = k - includeFilename = v - break - } - - includeFilename = r.templateString(includeFilename) - - var tasksFile types.TasksFile - var includePath string - // check if included file is a url - if helpers.IsURL(includeFilename) { - // If file is a url download it to a tmp directory - tmpDir, err := zarfUtils.MakeTempDir(config.CommonOptions.TempDirectory) - defer os.RemoveAll(tmpDir) - if err != nil { - return err - } - includePath = filepath.Join(tmpDir, filepath.Base(includeFilename)) - if err := zarfUtils.DownloadToFile(includeFilename, includePath, ""); err != nil { - return fmt.Errorf(lang.ErrDownloading, includeFilename, err.Error()) - } - } else { - includePath = filepath.Join(dir, includeFilename) - } - - if err := zarfUtils.ReadYaml(includePath, &tasksFile); err != nil { - return fmt.Errorf("unable to read included file %s: %w", includePath, err) - } - - // prefix task names and actions with the includes key - for i, t := range tasksFile.Tasks { - tasksFile.Tasks[i].Name = includeFilenameKey + ":" + t.Name - if len(tasksFile.Tasks[i].Actions) > 0 { - for j, a := range tasksFile.Tasks[i].Actions { - if a.TaskReference != "" && !strings.Contains(a.TaskReference, ":") { - tasksFile.Tasks[i].Actions[j].TaskReference = includeFilenameKey + ":" + a.TaskReference - } - } - } - } - // The following for loop protects against task loops. Makes sure the task being added hasn't already been processed - for _, taskToAdd := range tasksFile.Tasks { - for _, currentTasks := range r.TasksFile.Tasks { - if taskToAdd.Name == currentTasks.Name { - return fmt.Errorf("task loop detected, ensure no cyclic loops in tasks or includes files") - } - } - } - - r.TasksFile.Tasks = append(r.TasksFile.Tasks, tasksFile.Tasks...) - - // grab variables from included file - for _, v := range tasksFile.Variables { - r.TemplateMap["${"+v.Name+"}"] = &zarfUtils.TextTemplate{ - Sensitive: v.Sensitive, - AutoIndent: v.AutoIndent, - Type: v.Type, - Value: v.Default, - } - } - - // merge variables with setVariables - setVariablesTemplateMap := make(map[string]*zarfUtils.TextTemplate) - for name, value := range setVariables { - setVariablesTemplateMap[fmt.Sprintf("${%s}", name)] = &zarfUtils.TextTemplate{ - Value: value, - } - } - - r.TemplateMap = helpers.MergeMap[*zarfUtils.TextTemplate](r.TemplateMap, setVariablesTemplateMap) - - // recursively import tasks from included files - if tasksFile.Includes != nil { - if err := r.importTasks(tasksFile.Includes, includePath, setVariables); err != nil { - return err - } - } - } - return nil -} - -func (r *Runner) getTask(taskName string) (types.Task, error) { - for _, task := range r.TasksFile.Tasks { - if task.Name == taskName { - return task, nil - } - } - return types.Task{}, fmt.Errorf("task name %s not found", taskName) -} - -// mergeEnv merges two environment variable arrays, -// replacing variables found in env2 with variables from env1 -// otherwise appending the variable from env1 to env2 -func mergeEnv(env1, env2 []string) []string { - for _, s1 := range env1 { - replaced := false - for j, s2 := range env2 { - if strings.Split(s1, "=")[0] == strings.Split(s2, "=")[0] { - env2[j] = s1 - replaced = true - } - } - if !replaced { - env2 = append(env2, s1) - } - } - return env2 -} - -func formatEnvVar(name, value string) string { - // replace all non-alphanumeric characters with underscores - name = regexp.MustCompile(`[^a-zA-Z0-9]+`).ReplaceAllString(name, "_") - name = strings.ToUpper(name) - // prefix with INPUT_ (same as GitHub Actions) - return fmt.Sprintf("INPUT_%s=%s", name, value) -} - -func (r *Runner) executeTask(task types.Task) error { - if len(task.Files) > 0 { - if err := r.placeFiles(task.Files); err != nil { - return err - } - } - - defaultEnv := []string{} - for name, inputParam := range task.Inputs { - d := inputParam.Default - if d == "" { - continue - } - defaultEnv = append(defaultEnv, formatEnvVar(name, d)) - } - - // load the tasks env file into the runner, can override previous task's env files - if task.EnvPath != "" { - r.envFilePath = task.EnvPath - } - - for _, action := range task.Actions { - action.Env = mergeEnv(action.Env, defaultEnv) - if err := r.performAction(action); err != nil { - return err - } - } - return nil -} - -func (r *Runner) populateTemplateMap(zarfVariables []zarfTypes.ZarfPackageVariable, setVariables map[string]string) { - // populate text template (ie. Zarf var) with the following precedence: default < env var < set var - for _, variable := range zarfVariables { - templatedVariableName := fmt.Sprintf("${%s}", variable.Name) - textTemplate := &zarfUtils.TextTemplate{ - Sensitive: variable.Sensitive, - AutoIndent: variable.AutoIndent, - Type: variable.Type, - } - if v := os.Getenv(fmt.Sprintf("UDS_%s", variable.Name)); v != "" { - textTemplate.Value = v - } else { - textTemplate.Value = variable.Default - } - r.TemplateMap[templatedVariableName] = textTemplate - } - - setVariablesTemplateMap := make(map[string]*zarfUtils.TextTemplate) - for name, value := range setVariables { - setVariablesTemplateMap[fmt.Sprintf("${%s}", name)] = &zarfUtils.TextTemplate{ - Value: value, - } - } - - r.TemplateMap = helpers.MergeMap[*zarfUtils.TextTemplate](r.TemplateMap, setVariablesTemplateMap) -} - -func (r *Runner) placeFiles(files []zarfTypes.ZarfFile) error { - for _, file := range files { - // template file.Source and file.Target - srcFile := r.templateString(file.Source) - targetFile := r.templateString(file.Target) - - // get current directory - workingDir, err := os.Getwd() - if err != nil { - return err - } - dest := filepath.Join(workingDir, targetFile) - destDir := filepath.Dir(dest) - - if helpers.IsURL(srcFile) { - - // If file is a url download it - if err := zarfUtils.DownloadToFile(srcFile, dest, ""); err != nil { - return fmt.Errorf(lang.ErrDownloading, srcFile, err.Error()) - } - } else { - // If file is not a url copy it - if err := zarfUtils.CreatePathAndCopy(srcFile, dest); err != nil { - return fmt.Errorf("unable to copy file %s: %w", srcFile, err) - } - - } - // If file has extract path extract it - if file.ExtractPath != "" { - _ = os.RemoveAll(file.ExtractPath) - err = archiver.Extract(dest, file.ExtractPath, destDir) - if err != nil { - return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, srcFile, err.Error()) - } - } - - // if shasum is specified check it - if file.Shasum != "" { - if file.ExtractPath != "" { - if err := zarfUtils.SHAsMatch(file.ExtractPath, file.Shasum); err != nil { - return err - } - } else { - if err := zarfUtils.SHAsMatch(dest, file.Shasum); err != nil { - return err - } - } - } - - // template any text files with variables - fileList := []string{} - if zarfUtils.IsDir(dest) { - files, _ := zarfUtils.RecursiveFileList(dest, nil, false) - fileList = append(fileList, files...) - } else { - fileList = append(fileList, dest) - } - for _, subFile := range fileList { - // Check if the file looks like a text file - isText, err := zarfUtils.IsTextFile(subFile) - if err != nil { - fmt.Printf("unable to determine if file %s is a text file: %s", subFile, err) - } - - // If the file is a text file, template it - if isText { - if err := zarfUtils.ReplaceTextTemplate(subFile, r.TemplateMap, nil, `\$\{[A-Z0-9_]+\}`); err != nil { - return fmt.Errorf("unable to template file %s: %w", subFile, err) - } - } - } - - // if executable make file executable - if file.Executable || zarfUtils.IsDir(dest) { - _ = os.Chmod(dest, 0700) - } else { - _ = os.Chmod(dest, 0600) - } - - // if symlinks create them - for _, link := range file.Symlinks { - // Try to remove the filepath if it exists - _ = os.RemoveAll(link) - // Make sure the parent directory exists - _ = zarfUtils.CreateParentDirectory(link) - // Create the symlink - err := os.Symlink(targetFile, link) - if err != nil { - return fmt.Errorf("unable to create symlink %s->%s: %w", link, targetFile, err) - } - } - } - return nil -} - -func (r *Runner) performAction(action types.Action) error { - if action.TaskReference != "" { - // todo: much of this logic is duplicated in Run, consider refactoring - referencedTask, err := r.getTask(action.TaskReference) - if err != nil { - return err - } - - // template the withs with variables - for k, v := range action.With { - action.With[k] = r.templateString(v) - } - - referencedTask.Actions, err = templateTaskActionsWithInputs(referencedTask, action.With) - if err != nil { - return err - } - - withEnv := []string{} - for name := range action.With { - withEnv = append(withEnv, formatEnvVar(name, action.With[name])) - } - if err := validateActionableTaskCall(referencedTask.Name, referencedTask.Inputs, action.With); err != nil { - return err - } - for _, a := range referencedTask.Actions { - a.Env = mergeEnv(withEnv, a.Env) - } - if err := r.executeTask(referencedTask); err != nil { - return err - } - } else { - err := r.performZarfAction(action.ZarfComponentAction) - if err != nil { - return err - } - } - return nil -} - -// templateTaskActionsWithInputs templates a task's actions with the given inputs -func templateTaskActionsWithInputs(task types.Task, withs map[string]string) ([]types.Action, error) { - data := map[string]map[string]string{ - "inputs": {}, - } - - // get inputs from "with" map - for name := range withs { - data["inputs"][name] = withs[name] - } - - // use default if not populated in data - for name := range task.Inputs { - if current, ok := data["inputs"][name]; !ok || current == "" { - data["inputs"][name] = task.Inputs[name].Default - } - } - - b, err := goyaml.Marshal(task.Actions) - if err != nil { - return nil, err - } - - t, err := template.New("template task actions").Option("missingkey=error").Delims("${{", "}}").Parse(string(b)) - if err != nil { - return nil, err - } - - var templated strings.Builder - - if err := t.Execute(&templated, data); err != nil { - return nil, err - } - - result := templated.String() - - var templatedActions []types.Action - - return templatedActions, goyaml.Unmarshal([]byte(result), &templatedActions) -} - -func (r *Runner) checkForTaskLoops(task types.Task, tasksFile types.TasksFile, setVariables map[string]string) error { - // Filtering unique task actions allows for rerunning tasks in the same execution - uniqueTaskActions := getUniqueTaskActions(task.Actions) - for _, action := range uniqueTaskActions { - if r.processAction(task, action) { - // process includes for action, which will import all tasks for include file - if err := r.processIncludes(tasksFile, setVariables, action); err != nil { - return err - } - - exists := r.TaskNameMap[action.TaskReference] - if exists { - return fmt.Errorf("task loop detected, ensure no cyclic loops in tasks or includes files") - } - r.TaskNameMap[action.TaskReference] = true - newTask, err := r.getTask(action.TaskReference) - if err != nil { - return err - } - if err = r.checkForTaskLoops(newTask, tasksFile, setVariables); err != nil { - return err - } - } - // Clear map once we get to a task that doesn't call another task - clear(r.TaskNameMap) - } - return nil -} - -// processAction checks if action needs to be processed for a given task -func (r *Runner) processAction(task types.Task, action types.Action) bool { - - taskReferenceName := strings.Split(task.Name, ":")[0] - actionReferenceName := strings.Split(action.TaskReference, ":")[0] - // don't need to process if the action.TaskReference is empty or if the task and action references are the same since - // that indicates the task and task in the action are in the same file - if action.TaskReference != "" && (taskReferenceName != actionReferenceName) { - for _, task := range r.TasksFile.Tasks { - // check if TasksFile.Tasks already includes tasks with given reference name, which indicates that the - // reference has already been processed. - if strings.Contains(task.Name, taskReferenceName+":") || strings.Contains(task.Name, actionReferenceName+":") { - return false - } - } - return true - } - return false -} - -// validateActionableTaskCall validates a tasks "withs" and inputs -func validateActionableTaskCall(inputTaskName string, inputs map[string]types.InputParameter, withs map[string]string) error { - missing := []string{} - for inputKey, input := range inputs { - // skip inputs that are not required or have a default value - if !input.Required || input.Default != "" { - continue - } - checked := false - for withKey, withVal := range withs { - // verify that the input is in the with map and the "with" has a value - if inputKey == withKey && withVal != "" { - checked = true - break - } - } - if !checked { - missing = append(missing, inputKey) - } - } - if len(missing) > 0 { - return fmt.Errorf("task %s is missing required inputs: %s", inputTaskName, strings.Join(missing, ", ")) - } - for withKey := range withs { - matched := false - for inputKey, input := range inputs { - if withKey == inputKey { - if input.DeprecatedMessage != "" { - message.Warnf("This input has been marked deprecated: %s", input.DeprecatedMessage) - } - matched = true - break - } - } - if !matched { - message.Warnf("Task %s does not have an input named %s", inputTaskName, withKey) - } - } - return nil -} - -func getUniqueTaskActions(actions []types.Action) []types.Action { - uniqueMap := make(map[string]bool) - var uniqueArray []types.Action - - for _, action := range actions { - if !uniqueMap[action.TaskReference] { - uniqueMap[action.TaskReference] = true - uniqueArray = append(uniqueArray, action) - } - } - return uniqueArray -} - -func (r *Runner) performZarfAction(action *zarfTypes.ZarfComponentAction) error { - var ( - ctx context.Context - cancel context.CancelFunc - cmdEscaped string - out string - err error - - cmd = action.Cmd - ) - - // If the action is a wait, convert it to a command. - if action.Wait != nil { - // If the wait has no timeout, set a default of 5 minutes. - if action.MaxTotalSeconds == nil { - fiveMin := 300 - action.MaxTotalSeconds = &fiveMin - } - - // Convert the wait to a command. - if cmd, err = convertWaitToCmd(*action.Wait, action.MaxTotalSeconds); err != nil { - return err - } - - // Mute the output because it will be noisy. - t := true - action.Mute = &t - - // Set the max retries to 0. - z := 0 - action.MaxRetries = &z - - // Not used for wait actions. - d := "" - action.Dir = &d - action.Env = []string{} - action.SetVariables = []zarfTypes.ZarfComponentActionSetVariable{} - } - - // load the contents of the env file into the Action + the UDS_ARCH - if r.envFilePath != "" { - envFilePath := filepath.Join(filepath.Dir(config.TaskFileLocation), r.envFilePath) - envFileContents, err := os.ReadFile(envFilePath) - if err != nil { - return err - } - action.Env = append(action.Env, strings.Split(string(envFileContents), "\n")...) - } - action.Env = append(action.Env, fmt.Sprintf("UDS_ARCH=%s", config.GetArch())) - - if action.Description != "" { - cmdEscaped = action.Description - } else { - cmdEscaped = message.Truncate(cmd, 60, false) - } - - spinner := message.NewProgressSpinner("Running \"%s\"", cmdEscaped) - // Persist the spinner output so it doesn't get overwritten by the command output. - spinner.EnablePreserveWrites() - - cfg := actionGetCfg(zarfTypes.ZarfComponentActionDefaults{}, *action, r.TemplateMap) - - if cmd, err = actionCmdMutation(cmd); err != nil { - spinner.Errorf(err, "Error mutating command: %s", cmdEscaped) - } - - // Template dir string - cfg.Dir = r.templateString(cfg.Dir) - - // template cmd string - cmd = r.templateString(cmd) - - duration := time.Duration(cfg.MaxTotalSeconds) * time.Second - timeout := time.After(duration) - - // Keep trying until the max retries is reached. - for remaining := cfg.MaxRetries + 1; remaining > 0; remaining-- { - - // Perform the action run. - tryCmd := func(ctx context.Context) error { - // Try running the command and continue the retry loop if it fails. - if out, err = actionRun(ctx, cfg, cmd, cfg.Shell, spinner); err != nil { - return err - } - - out = strings.TrimSpace(out) - - // If an output variable is defined, set it. - for _, v := range action.SetVariables { - // include ${...} syntax in template map for uniformity and to satisfy zarfUtils.ReplaceTextTemplate - nameInTemplatemap := "${" + v.Name + "}" - r.TemplateMap[nameInTemplatemap] = &zarfUtils.TextTemplate{ - Sensitive: v.Sensitive, - AutoIndent: v.AutoIndent, - Type: v.Type, - Value: out, - } - if regexp.MustCompile(v.Pattern).MatchString(r.TemplateMap[nameInTemplatemap].Value); err != nil { - message.WarnErr(err, err.Error()) - return err - } - } - - // If the action has a wait, change the spinner message to reflect that on success. - if action.Wait != nil { - spinner.Successf("Wait for \"%s\" succeeded", cmdEscaped) - } else { - spinner.Successf("Completed \"%s\"", cmdEscaped) - } - - // If the command ran successfully, continue to the next action. - return nil - } - - // If no timeout is set, run the command and return or continue retrying. - if cfg.MaxTotalSeconds < 1 { - spinner.Updatef("Waiting for \"%s\" (no timeout)", cmdEscaped) - if err := tryCmd(context.TODO()); err != nil { - continue - } - - return nil - } - - // Run the command on repeat until success or timeout. - spinner.Updatef("Waiting for \"%s\" (timeout: %ds)", cmdEscaped, cfg.MaxTotalSeconds) - select { - // On timeout break the loop to abort. - case <-timeout: - break - - // Otherwise, try running the command. - default: - ctx, cancel = context.WithTimeout(context.Background(), duration) - defer cancel() - if err := tryCmd(ctx); err != nil { - continue - } - - return nil - } - } - - select { - case <-timeout: - // If we reached this point, the timeout was reached. - return fmt.Errorf("command \"%s\" timed out after %d seconds", cmdEscaped, cfg.MaxTotalSeconds) - - default: - // If we reached this point, the retry limit was reached. - return fmt.Errorf("command \"%s\" failed after %d retries", cmdEscaped, cfg.MaxRetries) - } -} - -// templateString replaces ${...} with the value from the template map -func (r *Runner) templateString(s string) string { - // Create a regular expression to match ${...} - re := regexp.MustCompile(`\${(.*?)}`) - - // template string using values from the template map - result := re.ReplaceAllStringFunc(s, func(matched string) string { - if value, ok := r.TemplateMap[matched]; ok { - return value.Value - } - return matched // If the key is not found, keep the original substring - }) - return result -} - -// Perform some basic string mutations to make commands more useful. -func actionCmdMutation(cmd string) (string, error) { - runCmd, err := zarfUtils.GetFinalExecutablePath() - if err != nil { - return cmd, err - } - - // Try to patch the binary path in case the name isn't exactly "./uds". - cmd = strings.ReplaceAll(cmd, "./uds ", runCmd+" ") - - return cmd, nil -} - -// convertWaitToCmd will return the wait command if it exists, otherwise it will return the original command. -func convertWaitToCmd(wait zarfTypes.ZarfComponentActionWait, timeout *int) (string, error) { - // Build the timeout string. - timeoutString := fmt.Sprintf("--timeout %ds", *timeout) - - // If the action has a wait, build a cmd from that instead. - cluster := wait.Cluster - if cluster != nil { - ns := cluster.Namespace - if ns != "" { - ns = fmt.Sprintf("-n %s", ns) - } - - // Build a call to the uds tools wait-for command. - return fmt.Sprintf("./uds zarf tools wait-for %s %s %s %s %s", - cluster.Kind, cluster.Identifier, cluster.Condition, ns, timeoutString), nil - } - - network := wait.Network - if network != nil { - // Make sure the protocol is lower case. - network.Protocol = strings.ToLower(network.Protocol) - - // If the protocol is http and no code is set, default to 200. - if strings.HasPrefix(network.Protocol, "http") && network.Code == 0 { - network.Code = 200 - } - - // Build a call to the uds tools wait-for command. - return fmt.Sprintf("./uds zarf tools wait-for %s %s %d %s", - network.Protocol, network.Address, network.Code, timeoutString), nil - } - - return "", fmt.Errorf("wait action is missing a cluster or network") -} - -//go:linkname actionGetCfg github.com/defenseunicorns/zarf/src/pkg/packager.actionGetCfg -func actionGetCfg(cfg zarfTypes.ZarfComponentActionDefaults, a zarfTypes.ZarfComponentAction, vars map[string]*zarfUtils.TextTemplate) zarfTypes.ZarfComponentActionDefaults - -//go:linkname actionRun github.com/defenseunicorns/zarf/src/pkg/packager.actionRun -func actionRun(ctx context.Context, cfg zarfTypes.ZarfComponentActionDefaults, cmd string, shellPref zarfTypes.ZarfComponentActionShell, spinner *message.Spinner) (string, error) diff --git a/src/pkg/sources/tarball.go b/src/pkg/sources/tarball.go index 7e13746be..eabeaaa9b 100644 --- a/src/pkg/sources/tarball.go +++ b/src/pkg/sources/tarball.go @@ -216,8 +216,13 @@ func (t *TarballBundle) extractPkgFromBundle() ([]string, error) { }) path := desc.Annotations[ocispec.AnnotationTitle] + cleanPath := filepath.Clean(path) + if strings.Contains(cleanPath, "..") { + // throw an error for dangerous looking paths + return fmt.Errorf("invalid path detected: %s", path) + } size := desc.Size - layerDst := filepath.Join(t.TmpDir, path) + layerDst := filepath.Join(t.TmpDir, cleanPath) if err := zarfUtils.CreateDirectory(filepath.Dir(layerDst), 0700); err != nil { return err } diff --git a/src/test/bundles/01-uds-bundle/uds-bundle.yaml b/src/test/bundles/01-uds-bundle/uds-bundle.yaml index 94666242d..afa0d0348 100644 --- a/src/test/bundles/01-uds-bundle/uds-bundle.yaml +++ b/src/test/bundles/01-uds-bundle/uds-bundle.yaml @@ -1,7 +1,7 @@ kind: UDSBundle metadata: - name: example - description: an example UDS bundle + name: example-remote + description: an example UDS bundle with remote packages version: 0.0.1 packages: diff --git a/src/test/bundles/02-simple-vars/import-all/uds-bundle.yaml b/src/test/bundles/02-simple-vars/import-all/uds-bundle.yaml deleted file mode 100644 index 7c2e28fa9..000000000 --- a/src/test/bundles/02-simple-vars/import-all/uds-bundle.yaml +++ /dev/null @@ -1,17 +0,0 @@ -kind: UDSBundle -metadata: - name: import-all - description: show how global exports work - version: 0.0.1 - -packages: - - name: output-var - repository: localhost:888/output-var - ref: 0.0.1 - exports: - - name: OUTPUT - - name: PRECEDENCE - - - name: receive-var - repository: localhost:888/receive-var - ref: 0.0.1 diff --git a/src/test/bundles/02-simple-vars/import-all-bad-name/uds-bundle.yaml b/src/test/bundles/02-variables/bad-var-name/uds-bundle.yaml similarity index 70% rename from src/test/bundles/02-simple-vars/import-all-bad-name/uds-bundle.yaml rename to src/test/bundles/02-variables/bad-var-name/uds-bundle.yaml index 34671511a..6e9cc7c14 100644 --- a/src/test/bundles/02-simple-vars/import-all-bad-name/uds-bundle.yaml +++ b/src/test/bundles/02-variables/bad-var-name/uds-bundle.yaml @@ -1,19 +1,19 @@ kind: UDSBundle metadata: - name: import-all-bad-name + name: bad-var-name description: show errors for bad imports version: 0.0.1 packages: - name: output-var - repository: localhost:888/output-var + path: ../../../packages/no-cluster/output-var ref: 0.0.1 exports: - name: OUTPUT - name: PRECEDENCE - name: receive-var - repository: localhost:888/receive-var + path: ../../../packages/no-cluster/receive-var ref: 0.0.1 imports: - package: output-varz diff --git a/src/test/bundles/02-simple-vars/export-name-collision/uds-bundle.yaml b/src/test/bundles/02-variables/export-name-collision/uds-bundle.yaml similarity index 73% rename from src/test/bundles/02-simple-vars/export-name-collision/uds-bundle.yaml rename to src/test/bundles/02-variables/export-name-collision/uds-bundle.yaml index ad15a6899..97a87f3a4 100644 --- a/src/test/bundles/02-simple-vars/export-name-collision/uds-bundle.yaml +++ b/src/test/bundles/02-variables/export-name-collision/uds-bundle.yaml @@ -6,20 +6,20 @@ metadata: packages: - name: output-var - repository: localhost:888/output-var + path: ../../../packages/no-cluster/output-var ref: 0.0.1 exports: - name: OUTPUT - name: PRECEDENCE - name: output-var-collision - repository: localhost:888/output-var-collision + path: ../../../packages/no-cluster/output-var-collision ref: 0.0.1 exports: - name: OUTPUT - name: receive-var - repository: localhost:888/receive-var + path: ../../../packages/no-cluster/receive-var ref: 0.0.1 imports: - package: output-var-collision diff --git a/src/test/bundles/02-simple-vars/uds-bundle.yaml b/src/test/bundles/02-variables/remote/uds-bundle.yaml similarity index 95% rename from src/test/bundles/02-simple-vars/uds-bundle.yaml rename to src/test/bundles/02-variables/remote/uds-bundle.yaml index 55bf4747c..63662ff0c 100644 --- a/src/test/bundles/02-simple-vars/uds-bundle.yaml +++ b/src/test/bundles/02-variables/remote/uds-bundle.yaml @@ -1,6 +1,6 @@ kind: UDSBundle metadata: - name: simple-vars + name: variables description: show how vars work version: 0.0.1 diff --git a/src/test/bundles/02-variables/uds-bundle.yaml b/src/test/bundles/02-variables/uds-bundle.yaml new file mode 100644 index 000000000..db5d19801 --- /dev/null +++ b/src/test/bundles/02-variables/uds-bundle.yaml @@ -0,0 +1,21 @@ +kind: UDSBundle +metadata: + name: variables + description: show how vars work + version: 0.0.1 + +packages: + - name: output-var + path: ../../packages/no-cluster/output-var + ref: 0.0.1 + exports: + - name: OUTPUT + - name: PRECEDENCE + + - name: receive-var + path: ../../packages/no-cluster/receive-var + ref: 0.0.1 + imports: + # note that PRECEDENCE is not imported because exports are global! + - name: OUTPUT + package: output-var diff --git a/src/test/bundles/02-simple-vars/uds-config.yaml b/src/test/bundles/02-variables/uds-config.yaml similarity index 100% rename from src/test/bundles/02-simple-vars/uds-config.yaml rename to src/test/bundles/02-variables/uds-config.yaml diff --git a/src/test/bundles/03-local-and-remote/uds-config.yaml b/src/test/bundles/03-local-and-remote/uds-config.yaml new file mode 100644 index 000000000..e60780697 --- /dev/null +++ b/src/test/bundles/03-local-and-remote/uds-config.yaml @@ -0,0 +1,2 @@ +options: + log_level: debug diff --git a/src/test/bundles/04-init/uds-bundle.yaml b/src/test/bundles/04-init/uds-bundle.yaml index 456416827..a416966b9 100644 --- a/src/test/bundles/04-init/uds-bundle.yaml +++ b/src/test/bundles/04-init/uds-bundle.yaml @@ -10,12 +10,12 @@ packages: - name: init path: "../../packages" # renovate: datasource=github-tags depName=defenseunicorns/zarf - ref: v0.32.3 + ref: v0.32.4 optionalComponents: - git-server - name: init repository: ghcr.io/defenseunicorns/packages/init # renovate: datasource=github-tags depName=defenseunicorns/zarf - ref: v0.32.3 + ref: v0.32.4 optionalComponents: - git-server diff --git a/src/test/bundles/07-helm-overrides/uds-bundle.yaml b/src/test/bundles/07-helm-overrides/uds-bundle.yaml index a0019dc3c..dd3855166 100644 --- a/src/test/bundles/07-helm-overrides/uds-bundle.yaml +++ b/src/test/bundles/07-helm-overrides/uds-bundle.yaml @@ -29,6 +29,10 @@ packages: value: customAnnotation: "customValue" variables: + - name: log_level + path: "podinfo.logLevel" + description: "Set the log level for podinfo" + default: "debug" # not overwritten! - name: ui_color path: "podinfo.ui.color" description: "Set the color for podinfo's UI" diff --git a/src/test/bundles/12-exported-pkg-vars/uds-bundle.yaml b/src/test/bundles/12-exported-pkg-vars/uds-bundle.yaml index e2bcc606d..8745785ca 100644 --- a/src/test/bundles/12-exported-pkg-vars/uds-bundle.yaml +++ b/src/test/bundles/12-exported-pkg-vars/uds-bundle.yaml @@ -6,7 +6,7 @@ metadata: packages: - name: output-var - repository: localhost:888/output-var + path: ../../packages/no-cluster/output-var ref: 0.0.1 exports: - name: COLOR @@ -27,8 +27,6 @@ packages: podinfo-component: unicorn-podinfo: values: - - path: "podinfo.replicaCount" - value: 1 - path: "podinfo.ui.color" value: ${COLOR} - path: podinfo.podAnnotations diff --git a/src/test/common.go b/src/test/common.go index d3444e5cb..373f81f21 100644 --- a/src/test/common.go +++ b/src/test/common.go @@ -91,7 +91,7 @@ func (e2e *UDSE2ETest) GetLogFileContents(t *testing.T, stdErr string) string { // SetupDockerRegistry uses the host machine's docker daemon to spin up a local registry for testing purposes. func (e2e *UDSE2ETest) SetupDockerRegistry(t *testing.T, port int) { // spin up a local registry - registryImage := "registry:2.8.2" + registryImage := "registry:2.8.3" err := exec.CmdWithPrint("docker", "run", "-d", "--restart=always", "-p", fmt.Sprintf("%d:5000", port), "--name", fmt.Sprintf("registry-%d", port), registryImage) require.NoError(t, err) } @@ -154,7 +154,7 @@ func downloadFile(url string, outputDir string) error { return fmt.Errorf("unexpected status code: %d", response.StatusCode) } - if err := os.MkdirAll(outputDir, 0755); err != nil { + if err := os.MkdirAll(outputDir, 0o755); err != nil { return err } diff --git a/src/test/e2e/bundle_test.go b/src/test/e2e/bundle_test.go index 766484ca2..807a2ce7d 100644 --- a/src/test/e2e/bundle_test.go +++ b/src/test/e2e/bundle_test.go @@ -21,7 +21,7 @@ import ( ) func zarfPublish(t *testing.T, path string, reg string) { - args := strings.Split(fmt.Sprintf("zarf package publish %s oci://%s --insecure --oci-concurrency=10", path, reg), " ") + args := strings.Split(fmt.Sprintf("zarf package publish %s oci://%s --insecure --oci-concurrency=10 -l debug", path, reg), " ") _, _, err := e2e.UDS(args...) require.NoError(t, err) } @@ -41,6 +41,7 @@ func TestSimpleBundleWithZarfAction(t *testing.T) { } func TestCreateWithNoPath(t *testing.T) { + // need to use remote pkgs because we move the uds-bundle.yaml to the current directory zarfPkgPath1 := "src/test/packages/no-cluster/output-var" zarfPkgPath2 := "src/test/packages/no-cluster/receive-var" e2e.CreateZarfPkg(t, zarfPkgPath1, false) @@ -55,10 +56,11 @@ func TestCreateWithNoPath(t *testing.T) { pkg = filepath.Join(zarfPkgPath2, fmt.Sprintf("zarf-package-receive-var-%s-0.0.1.tar.zst", e2e.Arch)) zarfPublish(t, pkg, "localhost:888") - err := os.Link(fmt.Sprintf("src/test/bundles/02-simple-vars/%s", config.BundleYAML), config.BundleYAML) + // move the bundle to the current directory so we can test the create command with no path + err := os.Link(fmt.Sprintf("src/test/bundles/02-variables/remote/%s", config.BundleYAML), config.BundleYAML) require.NoError(t, err) defer os.Remove(config.BundleYAML) - defer os.Remove(fmt.Sprintf("uds-bundle-simple-vars-%s-0.0.1.tar.zst", e2e.Arch)) + defer os.Remove(fmt.Sprintf("uds-bundle-variables-%s-0.0.1.tar.zst", e2e.Arch)) // create cmd := strings.Split("create --confirm --insecure", " ") @@ -90,7 +92,7 @@ func TestBundleWithLocalAndRemotePkgs(t *testing.T) { remove(t, tarballPath) } -func TestBundle(t *testing.T) { +func TestLocalBundleWithRemotePkgs(t *testing.T) { deployZarfInit(t) e2e.CreateZarfPkg(t, "src/test/packages/nginx", false) @@ -108,138 +110,127 @@ func TestBundle(t *testing.T) { zarfPublish(t, pkg, "localhost:889") bundleDir := "src/test/bundles/01-uds-bundle" - bundlePath := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-example-%s-0.0.1.tar.zst", e2e.Arch)) + bundlePath := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-example-remote-%s-0.0.1.tar.zst", e2e.Arch)) - createLocal(t, bundleDir, e2e.Arch) // todo: allow creating from both the folder containing and direct reference to uds-bundle.yaml + createLocal(t, bundleDir, e2e.Arch) inspectLocal(t, bundlePath) inspectLocalAndSBOMExtract(t, bundlePath) - // Test with an "options only" config file - os.Setenv("UDS_CONFIG", filepath.Join("src/test/bundles/01-uds-bundle", "uds-config.yaml")) deploy(t, bundlePath) remove(t, bundlePath) - - //Test create using custom tmpDir - runCmd(t, "create "+bundleDir+" --tmpdir ./customtmp --confirm --insecure") - - // remove customtmp folder if it exists - err := os.RemoveAll("./customtmp") - require.NoError(t, err) - } func TestPackagesFlag(t *testing.T) { deployZarfInit(t) - - e2e.CreateZarfPkg(t, "src/test/packages/nginx", false) e2e.CreateZarfPkg(t, "src/test/packages/podinfo", false) - - e2e.SetupDockerRegistry(t, 888) - defer e2e.TeardownRegistry(t, 888) - e2e.SetupDockerRegistry(t, 889) - defer e2e.TeardownRegistry(t, 889) - - pkg := fmt.Sprintf("src/test/packages/nginx/zarf-package-nginx-%s-0.0.1.tar.zst", e2e.Arch) - zarfPublish(t, pkg, "localhost:888") - - pkg = fmt.Sprintf("src/test/packages/podinfo/zarf-package-podinfo-%s-0.0.1.tar.zst", e2e.Arch) - zarfPublish(t, pkg, "localhost:889") - - bundleDir := "src/test/bundles/01-uds-bundle" - bundlePath := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-example-%s-0.0.1.tar.zst", e2e.Arch)) + bundleDir := "src/test/bundles/03-local-and-remote" + bundlePath := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-test-local-and-remote-%s-0.0.1.tar.zst", e2e.Arch)) createLocal(t, bundleDir, e2e.Arch) - inspectLocal(t, bundlePath) - inspectLocalAndSBOMExtract(t, bundlePath) - // Test only podinfo deploy - deployPackagesFlag(bundlePath, "podinfo") cmd := strings.Split("zarf tools kubectl get deployments -A -o=jsonpath='{.items[*].metadata.name}'", " ") - deployments, _, _ := e2e.UDS(cmd...) - require.Contains(t, deployments, "podinfo") - require.NotContains(t, deployments, "nginx") - - remove(t, bundlePath) - - // Test both podinfo and nginx deploy - deployPackagesFlag(bundlePath, "podinfo,nginx") - deployments, _, _ = e2e.UDS(cmd...) - require.Contains(t, deployments, "podinfo") - require.Contains(t, deployments, "nginx") - - // Remove only podinfo - removePackagesFlag(bundlePath, "podinfo") - deployments, _, _ = e2e.UDS(cmd...) - require.NotContains(t, deployments, "podinfo") - require.Contains(t, deployments, "nginx") - - // Remove nginx - removePackagesFlag(bundlePath, "nginx") - deployments, _, _ = e2e.UDS(cmd...) - require.NotContains(t, deployments, "podinfo") - require.NotContains(t, deployments, "nginx") - - // Test invalid package deploy - _, stderr := deployPackagesFlag(bundlePath, "podinfo,nginx,peanuts") - require.Contains(t, stderr, "invalid zarf packages specified by --packages") - - // Test invalid package remove - _, stderr = removePackagesFlag(bundlePath, "podinfo,nginx,peanuts") - require.Contains(t, stderr, "invalid zarf packages specified by --packages") + t.Run("Test only podinfo deploy (local pkg)", func(t *testing.T) { + deployPackagesFlag(bundlePath, "podinfo") + deployments, _, _ := e2e.UDS(cmd...) + require.Contains(t, deployments, "podinfo") + require.NotContains(t, deployments, "nginx") + + remove(t, bundlePath) + deployments, _, _ = e2e.UDS(cmd...) + require.NotContains(t, deployments, "podinfo") + }) + + t.Run("Test only nginx deploy and remove (remote pkg)", func(t *testing.T) { + deployPackagesFlag(bundlePath, "nginx") + deployments, _, _ := e2e.UDS(cmd...) + require.Contains(t, deployments, "nginx") + require.NotContains(t, deployments, "podinfo") + remove(t, bundlePath) + + removePackagesFlag(bundlePath, "nginx") + deployments, _, _ = e2e.UDS(cmd...) + require.NotContains(t, deployments, "nginx") + }) + + t.Run("Test both podinfo and nginx deploy and remove", func(t *testing.T) { + deployPackagesFlag(bundlePath, "podinfo,nginx") + deployments, _, _ := e2e.UDS(cmd...) + require.Contains(t, deployments, "podinfo") + require.Contains(t, deployments, "nginx") + + removePackagesFlag(bundlePath, "podinfo,nginx") + deployments, _, _ = e2e.UDS(cmd...) + require.NotContains(t, deployments, "podinfo") + require.NotContains(t, deployments, "nginx") + }) + + t.Run("Test invalid package deploy", func(t *testing.T) { + _, stderr := deployPackagesFlag(bundlePath, "podinfo,nginx,peanuts") + require.Contains(t, stderr, "invalid zarf packages specified by --packages") + + }) + t.Run("Test invalid package remove", func(t *testing.T) { + _, stderr := removePackagesFlag(bundlePath, "podinfo,nginx,peanuts") + require.Contains(t, stderr, "invalid zarf packages specified by --packages") + + }) } func TestResumeFlag(t *testing.T) { deployZarfInit(t) - - e2e.CreateZarfPkg(t, "src/test/packages/nginx", false) e2e.CreateZarfPkg(t, "src/test/packages/podinfo", false) - - e2e.SetupDockerRegistry(t, 888) - defer e2e.TeardownRegistry(t, 888) - e2e.SetupDockerRegistry(t, 889) - defer e2e.TeardownRegistry(t, 889) - - pkg := fmt.Sprintf("src/test/packages/nginx/zarf-package-nginx-%s-0.0.1.tar.zst", e2e.Arch) - zarfPublish(t, pkg, "localhost:888") - - pkg = fmt.Sprintf("src/test/packages/podinfo/zarf-package-podinfo-%s-0.0.1.tar.zst", e2e.Arch) - zarfPublish(t, pkg, "localhost:889") - - bundleDir := "src/test/bundles/01-uds-bundle" - bundlePath := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-example-%s-0.0.1.tar.zst", e2e.Arch)) + bundleDir := "src/test/bundles/03-local-and-remote" + bundlePath := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-test-local-and-remote-%s-0.0.1.tar.zst", e2e.Arch)) createLocal(t, bundleDir, e2e.Arch) inspectLocal(t, bundlePath) inspectLocalAndSBOMExtract(t, bundlePath) - // Deploy only podinfo from bundle + getDeploymentsCmd := strings.Split("zarf tools kubectl get deployments -A -o=jsonpath='{.items[*].metadata.name}'", " ") + + // Deploy only podinfo (local pkg) deployPackagesFlag(bundlePath, "podinfo") - cmd := strings.Split("zarf tools kubectl get deployments -A -o=jsonpath='{.items[*].metadata.name}'", " ") - deployments, _, _ := e2e.UDS(cmd...) + deployments, _, _ := e2e.UDS(getDeploymentsCmd...) require.Contains(t, deployments, "podinfo") require.NotContains(t, deployments, "nginx") - // Deploy bundle --resume + // Deploy bundle --resume (resumes remote pkg) deployResumeFlag(t, bundlePath) - cmd = strings.Split("zarf tools kubectl get deployments -A -o=jsonpath='{.items[*].metadata.name}'", " ") - deployments, _, _ = e2e.UDS(cmd...) + deployments, _, _ = e2e.UDS(getDeploymentsCmd...) require.Contains(t, deployments, "podinfo") require.Contains(t, deployments, "nginx") // Remove only podinfo removePackagesFlag(bundlePath, "podinfo") - deployments, _, _ = e2e.UDS(cmd...) + deployments, _, _ = e2e.UDS(getDeploymentsCmd...) require.NotContains(t, deployments, "podinfo") require.Contains(t, deployments, "nginx") + // Deploy only nginx (remote pkg) + deployPackagesFlag(bundlePath, "nginx") + deployments, _, _ = e2e.UDS(getDeploymentsCmd...) + require.Contains(t, deployments, "nginx") + require.NotContains(t, deployments, "podinfo") + + // Deploy bundle --resume (resumes remote pkg) + deployResumeFlag(t, bundlePath) + deployments, _, _ = e2e.UDS(getDeploymentsCmd...) + require.Contains(t, deployments, "podinfo") + require.Contains(t, deployments, "nginx") + + // Remove only nginx + removePackagesFlag(bundlePath, "nginx") + deployments, _, _ = e2e.UDS(getDeploymentsCmd...) + require.NotContains(t, deployments, "nginx") + require.Contains(t, deployments, "podinfo") + // Remove bundle remove(t, bundlePath) - cmd = strings.Split("zarf tools kubectl get deployments -A -o=jsonpath='{.items[*].metadata.name}'", " ") - deployments, _, _ = e2e.UDS(cmd...) + deployments, _, _ = e2e.UDS(getDeploymentsCmd...) require.NotContains(t, deployments, "podinfo") require.NotContains(t, deployments, "nginx") } -func TestRemoteBundle(t *testing.T) { +func TestRemoteBundleWithRemotePkgs(t *testing.T) { deployZarfInit(t) e2e.CreateZarfPkg(t, "src/test/packages/nginx", false) e2e.CreateZarfPkg(t, "src/test/packages/podinfo", false) @@ -258,11 +249,11 @@ func TestRemoteBundle(t *testing.T) { bundleRef := registry.Reference{ Registry: "oci://localhost:888", // this info is derived from the bundle's metadata - Repository: "example", + Repository: "example-remote", Reference: "0.0.1", } - tarballPath := filepath.Join("build", fmt.Sprintf("uds-bundle-example-%s-0.0.1.tar.zst", e2e.Arch)) + tarballPath := filepath.Join("build", fmt.Sprintf("uds-bundle-example-remote-%s-0.0.1.tar.zst", e2e.Arch)) bundlePath := "src/test/bundles/01-uds-bundle" createRemoteInsecure(t, bundlePath, bundleRef.Registry, e2e.Arch) @@ -274,11 +265,9 @@ func TestRemoteBundle(t *testing.T) { inspectRemoteAndSBOMExtract(t, bundleRef.String()) deployAndRemoveRemote(t, bundleRef.String(), tarballPath) - // Test without architecture specified bundleRef = registry.Reference{ - Registry: "oci://localhost:888", - // this info is derived from the bundle's metadata - Repository: "example", + Registry: "oci://localhost:888", + Repository: "example-remote", Reference: "0.0.1", } deployAndRemoveRemote(t, bundleRef.String(), tarballPath) @@ -297,20 +286,7 @@ func TestBundleWithGitRepo(t *testing.T) { func TestBundleWithYmlFile(t *testing.T) { deployZarfInit(t) - e2e.CreateZarfPkg(t, "src/test/packages/nginx", true) - e2e.CreateZarfPkg(t, "src/test/packages/podinfo", true) - - e2e.SetupDockerRegistry(t, 888) - defer e2e.TeardownRegistry(t, 888) - e2e.SetupDockerRegistry(t, 889) - defer e2e.TeardownRegistry(t, 889) - - pkg := fmt.Sprintf("src/test/packages/nginx/zarf-package-nginx-%s-0.0.1.tar.zst", e2e.Arch) - zarfPublish(t, pkg, "localhost:888") - - pkg = fmt.Sprintf("src/test/packages/podinfo/zarf-package-podinfo-%s-0.0.1.tar.zst", e2e.Arch) - zarfPublish(t, pkg, "localhost:889") bundleDir := "src/test/bundles/09-uds-bundle-yml" bundlePath := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-yml-example-%s-0.0.1.tar.zst", e2e.Arch)) @@ -318,7 +294,6 @@ func TestBundleWithYmlFile(t *testing.T) { createLocal(t, bundleDir, e2e.Arch) inspectLocal(t, bundlePath) inspectLocalAndSBOMExtract(t, bundlePath) - // Test with an "options only" config file os.Setenv("UDS_CONFIG", filepath.Join("src/test/bundles/09-uds-bundle-yml", "uds-config.yml")) deploy(t, bundlePath) remove(t, bundlePath) @@ -487,29 +462,16 @@ func validateMultiArchIndex(t *testing.T, index ocispec.Index) { func TestBundleTmpDir(t *testing.T) { deployZarfInit(t) - - e2e.CreateZarfPkg(t, "src/test/packages/nginx", false) e2e.CreateZarfPkg(t, "src/test/packages/podinfo", false) - e2e.SetupDockerRegistry(t, 888) - defer e2e.TeardownRegistry(t, 888) - e2e.SetupDockerRegistry(t, 889) - defer e2e.TeardownRegistry(t, 889) - - pkg := fmt.Sprintf("src/test/packages/nginx/zarf-package-nginx-%s-0.0.1.tar.zst", e2e.Arch) - zarfPublish(t, pkg, "localhost:888") - - pkg = fmt.Sprintf("src/test/packages/podinfo/zarf-package-podinfo-%s-0.0.1.tar.zst", e2e.Arch) - zarfPublish(t, pkg, "localhost:889") - - bundleDir := "src/test/bundles/01-uds-bundle" - bundlePath := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-example-%s-0.0.1.tar.zst", e2e.Arch)) + bundleDir := "src/test/bundles/03-local-and-remote" + bundlePath := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-test-local-and-remote-%s-0.0.1.tar.zst", e2e.Arch)) - //Test create using custom tmpDir + // Test create using custom tmpDir tmpDirName := "customtmp" tmpDir := fmt.Sprintf("%s/%s", bundleDir, tmpDirName) - err := os.Mkdir(tmpDir, 0755) + err := os.Mkdir(tmpDir, 0o755) if err != nil { t.Fatalf("error creating directory: %v", err) } @@ -582,4 +544,7 @@ func TestBundleTmpDir(t *testing.T) { case <-time.After(10 * time.Second): // Timeout after 10 seconds t.Fatal("timeout waiting for directory to get populated") } + // remove customtmp folder if it exists + err = os.RemoveAll("./customtmp") + require.NoError(t, err) } diff --git a/src/test/e2e/runner_test.go b/src/test/e2e/runner_test.go index 8f4b4cc03..16002aca8 100644 --- a/src/test/e2e/runner_test.go +++ b/src/test/e2e/runner_test.go @@ -308,11 +308,18 @@ func TestTaskRunner(t *testing.T) { require.Contains(t, stdErr, "copy-verify") }) - t.Run("test call to zarf tools wait-for", func(t *testing.T) { + t.Run("test bad call to zarf tools wait-for", func(t *testing.T) { t.Parallel() - _, stdErr, err := e2e.UDS("run", "wait", "--file", "src/test/tasks/tasks.yaml") + _, stdErr, err := e2e.UDS("run", "wait-fail", "--file", "src/test/tasks/tasks.yaml") require.Error(t, err) - require.Contains(t, stdErr, "Waiting for") + require.Contains(t, stdErr, "Failed to run action") + }) + + t.Run("test successful call to zarf tools wait-for", func(t *testing.T) { + t.Parallel() + _, stderr, err := e2e.UDS("run", "wait-success", "--file", "src/test/tasks/tasks.yaml") + require.NoError(t, err) + require.Contains(t, stderr, "succeeded") }) t.Run("test task to load env vars using the envPath key", func(t *testing.T) { diff --git a/src/test/e2e/variable_test.go b/src/test/e2e/variable_test.go index 038bb2edc..407acecde 100644 --- a/src/test/e2e/variable_test.go +++ b/src/test/e2e/variable_test.go @@ -15,59 +15,34 @@ import ( ) func TestBundleVariables(t *testing.T) { - zarfPkgPath1 := "src/test/packages/no-cluster/output-var" - zarfPkgPath2 := "src/test/packages/no-cluster/receive-var" - e2e.CreateZarfPkg(t, zarfPkgPath1, false) - e2e.CreateZarfPkg(t, zarfPkgPath2, false) - - e2e.SetupDockerRegistry(t, 888) - defer e2e.TeardownRegistry(t, 888) - - pkg := filepath.Join(zarfPkgPath1, fmt.Sprintf("zarf-package-output-var-%s-0.0.1.tar.zst", e2e.Arch)) - zarfPublish(t, pkg, "localhost:888") - - pkg = filepath.Join(zarfPkgPath2, fmt.Sprintf("zarf-package-receive-var-%s-0.0.1.tar.zst", e2e.Arch)) - zarfPublish(t, pkg, "localhost:888") - - bundleDir := "src/test/bundles/02-simple-vars" - bundleTarballPath := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-simple-vars-%s-0.0.1.tar.zst", e2e.Arch)) - - createLocal(t, bundleDir, e2e.Arch) - createRemoteInsecure(t, bundleDir, "localhost:888", e2e.Arch) - + e2e.CreateZarfPkg(t, "src/test/packages/no-cluster/output-var", false) + e2e.CreateZarfPkg(t, "src/test/packages/no-cluster/receive-var", false) os.Setenv("UDS_ANIMAL", "Unicorns") - os.Setenv("UDS_CONFIG", filepath.Join("src/test/bundles/02-simple-vars", "uds-config.yaml")) - - _, stderr := deploy(t, bundleTarballPath) - bundleVariablesTestChecks(t, stderr, bundleTarballPath) - remove(t, bundleTarballPath) - - // Run same test checks but with package that isn't explicitly importing vars - bundleDir = "src/test/bundles/02-simple-vars/import-all" - bundleTarballPath = filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-import-all-%s-0.0.1.tar.zst", e2e.Arch)) - createLocal(t, bundleDir, e2e.Arch) - _, stderr = deploy(t, bundleTarballPath) - bundleVariablesTestChecks(t, stderr, bundleTarballPath) - - // Test with bad variable name in import - bundleDir = "src/test/bundles/02-simple-vars/import-all-bad-name" - stderr = createLocalError(bundleDir, e2e.Arch) - require.Contains(t, stderr, "does not have a matching export") - - // Test name collisions with exported variables - zarfPkgPath3 := "src/test/packages/no-cluster/output-var-collision" - e2e.CreateZarfPkg(t, zarfPkgPath3, false) - - pkg = filepath.Join(zarfPkgPath3, fmt.Sprintf("zarf-package-output-var-collision-%s-0.0.1.tar.zst", e2e.Arch)) - zarfPublish(t, pkg, "localhost:888") - - bundleDir = "src/test/bundles/02-simple-vars/export-name-collision" - bundleTarballPath = filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-export-name-collision-%s-0.0.1.tar.zst", e2e.Arch)) - createLocal(t, bundleDir, e2e.Arch) - createRemoteInsecure(t, bundleDir, "localhost:888", e2e.Arch) - _, stderr = deploy(t, bundleTarballPath) - require.Contains(t, stderr, "This fun-fact was imported: Daffodils are the national flower of Wales") - require.NotContains(t, stderr, "This fun-fact was imported: Unicorns are the national animal of Scotland") + os.Setenv("UDS_CONFIG", filepath.Join("src/test/bundles/02-variables", "uds-config.yaml")) + + t.Run("simple vars and global export", func(t *testing.T) { + bundleDir := "src/test/bundles/02-variables" + bundleTarballPath := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-variables-%s-0.0.1.tar.zst", e2e.Arch)) + createLocal(t, bundleDir, e2e.Arch) + _, stderr := deploy(t, bundleTarballPath) + bundleVariablesTestChecks(t, stderr, bundleTarballPath) + }) + + t.Run("bad var name in import", func(t *testing.T) { + bundleDir := "src/test/bundles/02-variables/bad-var-name" + stderr := createLocalError(bundleDir, e2e.Arch) + require.Contains(t, stderr, "does not have a matching export") + }) + + t.Run("var name collision with exported vars", func(t *testing.T) { + e2e.CreateZarfPkg(t, "src/test/packages/no-cluster/output-var-collision", false) + bundleDir := "src/test/bundles/02-variables/export-name-collision" + bundleTarballPath := filepath.Join(bundleDir, fmt.Sprintf("uds-bundle-export-name-collision-%s-0.0.1.tar.zst", e2e.Arch)) + createLocal(t, bundleDir, e2e.Arch) + _, stderr := deploy(t, bundleTarballPath) + require.Contains(t, stderr, "This fun-fact was imported: Daffodils are the national flower of Wales") + require.NotContains(t, stderr, "This fun-fact was imported: Unicorns are the national animal of Scotland") + }) } func bundleVariablesTestChecks(t *testing.T, stderr string, bundleTarballPath string) { @@ -106,59 +81,78 @@ func TestBundleWithHelmOverrides(t *testing.T) { createLocal(t, bundleDir, e2e.Arch) deploy(t, bundlePath) - // check values overrides - cmd := strings.Split("zarf tools kubectl get deploy -n podinfo unicorn-podinfo -o=jsonpath='{.spec.replicas}'", " ") - outputNumReplicas, _, err := e2e.UDS(cmd...) - require.Equal(t, "'2'", outputNumReplicas) - require.NoError(t, err) - - // check object-type override in values - cmd = strings.Split("zarf tools kubectl get deployment -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.metadata.annotations}'", " ") - annotations, _, err := e2e.UDS(cmd...) - require.Contains(t, annotations, "\"customAnnotation\":\"customValue\"") - require.NoError(t, err) - - // check list-type override in values - cmd = strings.Split("zarf tools kubectl get deployment -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.tolerations}'", " ") - tolerations, _, err := e2e.UDS(cmd...) - require.Contains(t, tolerations, "\"key\":\"uds\"") - require.Contains(t, tolerations, "\"value\":\"defense\"") - require.Contains(t, tolerations, "\"key\":\"unicorn\"") - require.Contains(t, tolerations, "\"effect\":\"NoSchedule\"") - require.NoError(t, err) - - // check variables overrides - cmd = strings.Split("zarf tools kubectl get deploy -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"PODINFO_UI_COLOR\")].value}'", " ") - outputUIColor, _, err := e2e.UDS(cmd...) - require.Equal(t, "'green, yellow'", outputUIColor) - require.NoError(t, err) - - // check variables overrides, no default but set in config - cmd = strings.Split("zarf tools kubectl get deploy -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"PODINFO_UI_MESSAGE\")].value}'", " ") - outputMsg, _, err := e2e.UDS(cmd...) - require.Equal(t, "'Hello Unicorn'", outputMsg) - require.NoError(t, err) - - // check variables overrides, no default and not set in config - cmd = strings.Split("zarf tools kubectl get secret test-secret -n podinfo -o jsonpath=\"{.data.test}\"", " ") - secretValue, _, err := e2e.UDS(cmd...) - // expect the value to be from the underlying chart's values.yaml, no overrides - require.Equal(t, "\"dGVzdC1zZWNyZXQ=\"", secretValue) - require.NoError(t, err) - - // check variables overrides with an object-type value - cmd = strings.Split("zarf tools kubectl get deployment -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.containers[0].securityContext}'", " ") - securityContext, _, err := e2e.UDS(cmd...) - require.NoError(t, err) - require.Contains(t, securityContext, "NET_ADMIN") - require.Contains(t, securityContext, "\"runAsGroup\":4000") - - // check variables overrides with a list-type value - cmd = strings.Split("zarf tools kubectl get ingress -n podinfo unicorn-podinfo -o=jsonpath='{.spec.rules[*].host}''", " ") - hosts, _, err := e2e.UDS(cmd...) - require.NoError(t, err) - require.Contains(t, hosts, "podinfo.burning.boats") - require.Contains(t, hosts, "podinfo.unicorns") + // test values overrides + t.Run("check values overrides", func(t *testing.T) { + cmd := strings.Split("zarf tools kubectl get deploy -n podinfo unicorn-podinfo -o=jsonpath='{.spec.replicas}'", " ") + outputNumReplicas, _, err := e2e.UDS(cmd...) + require.Equal(t, "'2'", outputNumReplicas) + require.NoError(t, err) + }) + + t.Run("check object-type override in values", func(t *testing.T) { + cmd := strings.Split("zarf tools kubectl get deployment -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.metadata.annotations}'", " ") + annotations, _, err := e2e.UDS(cmd...) + require.Contains(t, annotations, "\"customAnnotation\":\"customValue\"") + require.NoError(t, err) + + }) + + t.Run("check list-type override in values", func(t *testing.T) { + cmd := strings.Split("zarf tools kubectl get deployment -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.tolerations}'", " ") + tolerations, _, err := e2e.UDS(cmd...) + require.Contains(t, tolerations, "\"key\":\"uds\"") + require.Contains(t, tolerations, "\"value\":\"defense\"") + require.Contains(t, tolerations, "\"key\":\"unicorn\"") + require.Contains(t, tolerations, "\"effect\":\"NoSchedule\"") + require.NoError(t, err) + + }) + + // test variables overrides + t.Run("check variables overrides, use default", func(t *testing.T) { + cmd := strings.Split("zarf tools kubectl get deploy unicorn-podinfo -n podinfo -o=jsonpath='{.spec.template.spec.containers[*].command[*]}'", " ") + podCmd, _, err := e2e.UDS(cmd...) + require.NoError(t, err) + require.Contains(t, podCmd, "--level=debug") + }) + + t.Run("check variables overrides, default overwritten by config", func(t *testing.T) { + cmd := strings.Split("zarf tools kubectl get deploy -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"PODINFO_UI_COLOR\")].value}'", " ") + outputUIColor, _, err := e2e.UDS(cmd...) + require.Equal(t, "'green, yellow'", outputUIColor) + require.NoError(t, err) + }) + + t.Run("check variables overrides, no default but set in config", func(t *testing.T) { + cmd := strings.Split("zarf tools kubectl get deploy -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"PODINFO_UI_MESSAGE\")].value}'", " ") + outputMsg, _, err := e2e.UDS(cmd...) + require.Equal(t, "'Hello Unicorn'", outputMsg) + require.NoError(t, err) + }) + + t.Run("check variables overrides, no default and not set in config", func(t *testing.T) { + cmd := strings.Split("zarf tools kubectl get secret test-secret -n podinfo -o jsonpath=\"{.data.test}\"", " ") + secretValue, _, err := e2e.UDS(cmd...) + // expect the value to be from the underlying chart's values.yaml, no overrides + require.Equal(t, "\"dGVzdC1zZWNyZXQ=\"", secretValue) + require.NoError(t, err) + }) + + t.Run("check variables overrides with an object-type value", func(t *testing.T) { + cmd := strings.Split("zarf tools kubectl get deployment -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.containers[0].securityContext}'", " ") + securityContext, _, err := e2e.UDS(cmd...) + require.NoError(t, err) + require.Contains(t, securityContext, "NET_ADMIN") + require.Contains(t, securityContext, "\"runAsGroup\":4000") + }) + + t.Run("check variables overrides with a list-type value", func(t *testing.T) { + cmd := strings.Split("zarf tools kubectl get ingress -n podinfo unicorn-podinfo -o=jsonpath='{.spec.rules[*].host}''", " ") + hosts, _, err := e2e.UDS(cmd...) + require.NoError(t, err) + require.Contains(t, hosts, "podinfo.burning.boats") + require.Contains(t, hosts, "podinfo.unicorns") + }) remove(t, bundlePath) } @@ -183,16 +177,19 @@ func TestBundleWithEnvVarHelmOverrides(t *testing.T) { createLocal(t, bundleDir, e2e.Arch) deploy(t, bundlePath) - // check override variables, ensure they are coming from env vars and take highest precedence - cmd := strings.Split("z tools kubectl get deploy -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"PODINFO_UI_COLOR\")].value}'", " ") - outputUIColor, _, err := e2e.UDS(cmd...) - require.Equal(t, fmt.Sprintf("'%s'", color), outputUIColor) - require.NoError(t, err) + t.Run("check override variables, ensure they are coming from env vars and take highest precedence", func(t *testing.T) { + cmd := strings.Split("z tools kubectl get deploy -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"PODINFO_UI_COLOR\")].value}'", " ") + outputUIColor, _, err := e2e.UDS(cmd...) + require.Equal(t, fmt.Sprintf("'%s'", color), outputUIColor) + require.NoError(t, err) + }) - cmd = strings.Split("z tools kubectl get secret test-secret -n podinfo -o jsonpath=\"{.data.test}\"", " ") - secretValue, _, err := e2e.UDS(cmd...) - require.Equal(t, fmt.Sprintf("\"%s\"", b64Secret), secretValue) - require.NoError(t, err) + t.Run("check override secret val", func(t *testing.T) { + cmd := strings.Split("z tools kubectl get secret test-secret -n podinfo -o jsonpath=\"{.data.test}\"", " ") + secretValue, _, err := e2e.UDS(cmd...) + require.Equal(t, fmt.Sprintf("\"%s\"", b64Secret), secretValue) + require.NoError(t, err) + }) remove(t, bundlePath) } @@ -214,38 +211,33 @@ func TestVariablePrecedence(t *testing.T) { require.NoError(t, err) _, stderr := deploy(t, bundlePath) - // test env var taking highest precedence - cmd := strings.Split("zarf tools kubectl get deploy -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"PODINFO_UI_COLOR\")].value}'", " ") - outputUIColor, _, err := e2e.UDS(cmd...) - require.Equal(t, fmt.Sprintf("'%s'", color), outputUIColor) - require.NoError(t, err) + t.Run("test precedence, env var > uds-config.variables > uds-config.shared", func(t *testing.T) { + cmd := strings.Split("zarf tools kubectl get deploy -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"PODINFO_UI_COLOR\")].value}'", " ") + // test env har taking highest precedence + outputUIColor, _, err := e2e.UDS(cmd...) + require.Equal(t, fmt.Sprintf("'%s'", color), outputUIColor) + require.NoError(t, err) - // test uds-config.variables overriding a shared var - require.Contains(t, stderr, "shared var in output-var pkg: unicorns.uds.dev") + // test uds-config.variables overriding a shared var + require.Contains(t, stderr, "shared var in output-var pkg: unicorns.uds.dev") - // test uds-config.shared overriding a Zarf var - require.Contains(t, stderr, "shared var in helm-overrides pkg: burning.boats") + // test uds-config.shared overriding a Zarf var + require.Contains(t, stderr, "shared var in helm-overrides pkg: burning.boats") + }) - // test uds-config.shared overriding values in a Helm chart (ie. bundle overrides) - cmd = strings.Split("zarf tools kubectl get deploy unicorn-podinfo -n podinfo -o=jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"PODINFO_BACKEND_URL\")].value}'", " ") - backend, _, err := e2e.UDS(cmd...) - require.Equal(t, fmt.Sprintf("'%s'", "burning.boats"), backend) - require.NoError(t, err) + t.Run("test uds-config.shared overriding values in a Helm chart (ie. bundle overrides)", func(t *testing.T) { + cmd := strings.Split("zarf tools kubectl get deploy unicorn-podinfo -n podinfo -o=jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"PODINFO_BACKEND_URL\")].value}'", " ") + backend, _, err := e2e.UDS(cmd...) + require.Equal(t, fmt.Sprintf("'%s'", "burning.boats"), backend) + require.NoError(t, err) + }) remove(t, bundlePath) } -func TestZarfPackageExportVarsAsGlobalBundleVars(t *testing.T) { +func TestExportVarsAsGlobalVars(t *testing.T) { deployZarfInit(t) - zarfPkgPath1 := "src/test/packages/no-cluster/output-var" - e2e.CreateZarfPkg(t, zarfPkgPath1, false) - - e2e.SetupDockerRegistry(t, 888) - defer e2e.TeardownRegistry(t, 888) - - pkg := filepath.Join(zarfPkgPath1, fmt.Sprintf("zarf-package-output-var-%s-0.0.1.tar.zst", e2e.Arch)) - zarfPublish(t, pkg, "localhost:888") - + e2e.CreateZarfPkg(t, "src/test/packages/no-cluster/output-var", false) e2e.HelmDepUpdate(t, "src/test/packages/helm/unicorn-podinfo") e2e.CreateZarfPkg(t, "src/test/packages/helm", false) bundleDir := "src/test/bundles/12-exported-pkg-vars" @@ -254,26 +246,29 @@ func TestZarfPackageExportVarsAsGlobalBundleVars(t *testing.T) { createLocal(t, bundleDir, e2e.Arch) deploy(t, bundlePath) - // check templated variables overrides in values - cmd := strings.Split("zarf tools kubectl get deploy -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"PODINFO_UI_COLOR\")].value}'", " ") - outputUIColor, _, err := e2e.UDS(cmd...) - require.Equal(t, "'orange'", outputUIColor) - require.NoError(t, err) - - // check multiple templated variables as object overrides in values - cmd = strings.Split("zarf tools kubectl get deployment -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.metadata.annotations}'", " ") - annotations, _, err := e2e.UDS(cmd...) - require.Contains(t, annotations, "\"customAnnotation\":\"orangeAnnotation\"") - require.NoError(t, err) - - // check templated variable list-type overrides in values - cmd = strings.Split("zarf tools kubectl get deployment -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.tolerations}'", " ") - tolerations, _, err := e2e.UDS(cmd...) - require.Contains(t, tolerations, "\"key\":\"uds\"") - require.Contains(t, tolerations, "\"value\":\"true\"") - require.Contains(t, tolerations, "\"key\":\"unicorn\"") - require.Contains(t, tolerations, "\"value\":\"defense\"") - require.NoError(t, err) + t.Run("check templated variables overrides in values", func(t *testing.T) { + cmd := strings.Split("zarf tools kubectl get deploy -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"PODINFO_UI_COLOR\")].value}'", " ") + outputUIColor, _, err := e2e.UDS(cmd...) + require.Equal(t, "'orange'", outputUIColor) + require.NoError(t, err) + }) + + t.Run("check multiple templated variables as object overrides in values", func(t *testing.T) { + cmd := strings.Split("zarf tools kubectl get deployment -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.metadata.annotations}'", " ") + annotations, _, err := e2e.UDS(cmd...) + require.Contains(t, annotations, "\"customAnnotation\":\"orangeAnnotation\"") + require.NoError(t, err) + }) + + t.Run("check templated variable list-type overrides in values", func(t *testing.T) { + cmd := strings.Split("zarf tools kubectl get deployment -n podinfo unicorn-podinfo -o=jsonpath='{.spec.template.spec.tolerations}'", " ") + tolerations, _, err := e2e.UDS(cmd...) + require.Contains(t, tolerations, "\"key\":\"uds\"") + require.Contains(t, tolerations, "\"value\":\"true\"") + require.Contains(t, tolerations, "\"key\":\"unicorn\"") + require.Contains(t, tolerations, "\"value\":\"defense\"") + require.NoError(t, err) + }) remove(t, bundlePath) } diff --git a/src/test/packages/helm/unicorn-podinfo/Chart.lock b/src/test/packages/helm/unicorn-podinfo/Chart.lock index 0ae513ac2..2e77fa2c6 100644 --- a/src/test/packages/helm/unicorn-podinfo/Chart.lock +++ b/src/test/packages/helm/unicorn-podinfo/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: podinfo repository: https://stefanprodan.github.io/podinfo - version: 6.5.4 -digest: sha256:0db7bdcd71ac4162a0d2b9ebfc792f19f1cf52050933edeacd4f514f4bc3033b -generated: "2023-12-18T07:32:32.320434073Z" + version: 6.6.0 +digest: sha256:e98b771f48adfc407ca45353678f0a44f82cf24aa10bb1d1b304183e2284db4d +generated: "2024-02-26T12:39:38.675839591Z" diff --git a/src/test/packages/helm/unicorn-podinfo/Chart.yaml b/src/test/packages/helm/unicorn-podinfo/Chart.yaml index 474aa5d19..fbbb038f2 100644 --- a/src/test/packages/helm/unicorn-podinfo/Chart.yaml +++ b/src/test/packages/helm/unicorn-podinfo/Chart.yaml @@ -25,5 +25,5 @@ appVersion: "1.16.0" dependencies: - name: podinfo - version: 6.5.4 + version: 6.6.0 repository: https://stefanprodan.github.io/podinfo diff --git a/src/test/packages/helm/zarf.yaml b/src/test/packages/helm/zarf.yaml index ca4cfcecb..e51cf792d 100644 --- a/src/test/packages/helm/zarf.yaml +++ b/src/test/packages/helm/zarf.yaml @@ -13,7 +13,7 @@ components: required: true images: # renovate: datasource=github-tags depName=stefanprodan/podinfo - - ghcr.io/stefanprodan/podinfo:6.5.4 + - ghcr.io/stefanprodan/podinfo:6.6.0 charts: - name: unicorn-podinfo localPath: ./unicorn-podinfo diff --git a/src/test/tasks/tasks.yaml b/src/test/tasks/tasks.yaml index db0a315b1..66402c0a0 100644 --- a/src/test/tasks/tasks.yaml +++ b/src/test/tasks/tasks.yaml @@ -122,17 +122,22 @@ tasks: actions: - task: foo:fooybar - task: foo:foobar - - name: wait + - name: wait-success actions: - maxTotalSeconds: 1 wait: network: protocol: tcp address: githubstatus.com:443 - cluster: - kind: StatefulSet - name: cool-name - namespace: tasks + - name: wait-fail + actions: + - maxTotalSeconds: 1 + wait: + network: + cluster: + kind: StatefulSet + name: cool-name + namespace: tasks - name: include-loop actions: - task: infinite:loop