diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml new file mode 100644 index 00000000000..0e8010ffe94 --- /dev/null +++ b/.github/workflows/release-test.yml @@ -0,0 +1,239 @@ +name: Release Test Environment + +# THIS PIPELINE DEPLOY PyPi in dev mode (v3.x.x.devx) and layer in the test account +# For this pipeline we don't need to create tag and create docs + +# RELEASE PROCESS +# +# === Automated activities === +# +# 1. [Seal] Bump to release version and export source code with integrity hash +# 2. [Quality check] Restore sealed source code, run tests, linting, security and complexity base line +# 3. [Build] Restore sealed source code, create and export hashed build artifact for PyPi release (wheel, tarball) +# 4. [Provenance] Generates provenance for build, signs attestation with GitHub OIDC claims to confirm it came from this release pipeline, commit, org, repo, branch, hash, etc. +# 5. [Release] Restore built artifact, and publish package to PyPi prod repository +# 7. [PR to bump version] Restore sealed source code, and create a PR to update trunk with latest released project metadata +# 8. [Publish Layer v3 - test] Compile Layer in multiple Python versions and kick off pipeline for beta, prod, and canary releases +# 9. [Publish Layer v3 - test] Update docs with latest Layer ARNs and Changelog +# 10. [Publish Layer v3 - test] Create PR to update trunk so staged docs also point to the latest Layer ARN, when merged +# 12. [Post release] Close all issues labeled "pending-release" and notify customers about the release +# +# === Manual activities === +# +# 1. Kick off this workflow with the intended version +# 2. Update draft release notes after this workflow completes +# 3. If not already set, use `v` as a tag, e.g., v3.0.0, and select develop as target branch + +# NOTE +# +# See MAINTAINERS.md "Releasing a new version" for release mechanisms +# +# Every job is isolated and starts a new fresh container. + +env: + RELEASE_COMMIT: ${{ github.sha }} + RELEASE_TAG_VERSION: ${{ inputs.version_to_publish }} + +on: + workflow_dispatch: + inputs: + version_to_publish: + description: "Version to be released in PyPi, Docs, and Lambda Layer, e.g. v3.0.0.dev1 (development release)" + default: v3.0.0 + required: true + skip_pypi: + description: "Skip publishing to PyPi as it can't publish more than once. Useful for semi-failed releases" + default: false + type: boolean + required: false + +permissions: + contents: read + +jobs: + + # This job bumps the package version to the release version + # creates an integrity hash from the source code + # uploads the artifact with the integrity hash as the key name + # so subsequent jobs can restore from a trusted point in time to prevent tampering + seal: + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + integrity_hash: ${{ steps.seal_source_code.outputs.integrity_hash }} + artifact_name: ${{ steps.seal_source_code.outputs.artifact_name }} + RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION }} + steps: + - name: Export release version + id: release_version + # transform tag format `v` + run: | + RELEASE_VERSION="${RELEASE_TAG_VERSION:1}" + echo "RELEASE_VERSION=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + ref: ${{ env.RELEASE_COMMIT }} + + # We use a pinned version of Poetry to be certain it won't modify source code before we create a hash + - name: Install poetry + run: | + pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0 + pipx inject poetry git+https://github.com/monim67/poetry-bumpversion@315fe3324a699fa12ec20e202eb7375d4327d1c4 # v0.3.1 + + - name: Bump package version + id: versioning + run: poetry version "${RELEASE_VERSION}" + env: + RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION}} + + - name: Seal and upload + id: seal_source_code + uses: ./.github/actions/seal + with: + artifact_name_prefix: "source" + + # This job runs our automated test suite, complexity and security baselines + # it ensures previously merged have been tested as part of the pull request process + # + # NOTE + # + # we don't upload the artifact after testing to prevent any tampering of our source code dependencies + quality_check: + needs: seal + runs-on: ubuntu-latest + permissions: + contents: read + steps: + # NOTE: we need actions/checkout to configure git first (pre-commit hooks in make dev) + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + ref: ${{ env.RELEASE_COMMIT }} + + - name: Restore sealed source code + uses: ./.github/actions/seal-restore + with: + integrity_hash: ${{ needs.seal.outputs.integrity_hash }} + artifact_name: ${{ needs.seal.outputs.artifact_name }} + + - name: Debug cache restore + run: cat pyproject.toml + + - name: Install poetry + run: pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0 + - name: Set up Python + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: "3.12" + cache: "poetry" + - name: Install dependencies + run: make dev + - name: Run all tests, linting and baselines + run: make pr + + # This job creates a release artifact (tar.gz, wheel) + # it checks out code from release commit for custom actions to work + # then restores the sealed source code (overwrites any potential tampering) + # it's done separately from release job to enforce least privilege. + # We export just the final build artifact for release + build: + runs-on: ubuntu-latest + needs: [quality_check, seal] + permissions: + contents: read + outputs: + integrity_hash: ${{ steps.seal_build.outputs.integrity_hash }} + artifact_name: ${{ steps.seal_build.outputs.artifact_name }} + attestation_hashes: ${{ steps.encoded_hash.outputs.attestation_hashes }} + steps: + # NOTE: we need actions/checkout to configure git first (pre-commit hooks in make dev) + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + ref: ${{ env.RELEASE_COMMIT }} + + - name: Restore sealed source code + uses: ./.github/actions/seal-restore + with: + integrity_hash: ${{ needs.seal.outputs.integrity_hash }} + artifact_name: ${{ needs.seal.outputs.artifact_name }} + + - name: Install poetry + run: pipx install git+https://github.com/python-poetry/poetry@68b88e5390720a3dd84f02940ec5200bfce39ac6 # v1.5.0 + - name: Set up Python + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: "3.12" + cache: "poetry" + + - name: Build python package and wheel + run: poetry build + + - name: Seal and upload + id: seal_build + uses: ./.github/actions/seal + with: + artifact_name_prefix: "build" + files: "dist/" + + # NOTE: SLSA retraces our build to its artifact to ensure it wasn't tampered + # coupled with GitHub OIDC, SLSA can then confidently sign it came from this release pipeline+commit+branch+org+repo+actor+integrity hash + - name: Create attestation encoded hash for provenance + id: encoded_hash + working-directory: dist + run: echo "attestation_hashes=$(sha256sum ./* | base64 -w0)" >> "$GITHUB_OUTPUT" + + # This job creates a provenance file that describes how our release was built (all steps) + # after it verifies our build is reproducible within the same pipeline + # it confirms that its own software and the CI build haven't been tampered with (Trust but verify) + # lastly, it creates and sign an attestation (multiple.intoto.jsonl) that confirms + # this build artifact came from this GitHub org, branch, actor, commit ID, inputs that triggered this pipeline, and matches its integrity hash + # NOTE: supply chain threats review (we protect against all of them now): https://slsa.dev/spec/v1.0/threats-overview + provenance: + needs: [seal, build] + permissions: + contents: write # nested job explicitly require despite upload assets being set to false + actions: read # To read the workflow path. + id-token: write # To sign the provenance. + # NOTE: provenance fails if we use action pinning... it's a Github limitation + # because SLSA needs to trace & attest it came from a given branch; pinning doesn't expose that information + # https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/generic/README.md#referencing-the-slsa-generator + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.0.0 + with: + base64-subjects: ${{ needs.build.outputs.attestation_hashes }} + upload-assets: false # we upload its attestation in create_tag job, otherwise it creates a new release + + # This job uses release artifact to publish to PyPi + # it exchanges JWT tokens with GitHub to obtain PyPi credentials + # since it's already registered as a Trusted Publisher. + # It uses the sealed build artifact (.whl, .tar.gz) to release it + release: + needs: [build, seal, provenance] + environment: release + runs-on: ubuntu-latest + permissions: + id-token: write # OIDC for PyPi Trusted Publisher feature + env: + RELEASE_VERSION: ${{ needs.seal.outputs.RELEASE_VERSION }} + steps: + # NOTE: we need actions/checkout in order to use our local actions (e.g., ./.github/actions) + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + ref: ${{ env.RELEASE_COMMIT }} + + - name: Restore sealed source code + uses: ./.github/actions/seal-restore + with: + integrity_hash: ${{ needs.build.outputs.integrity_hash }} + artifact_name: ${{ needs.build.outputs.artifact_name }} + + # - name: Upload to PyPi prod + # if: ${{ !inputs.skip_pypi }} + # uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14 + + # PyPi test maintenance affected us numerous times, leaving for history purposes + # - name: Upload to PyPi test + # if: ${{ !inputs.skip_pypi }} + # uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14 + # with: + # repository-url: https://test.pypi.org/legacy/