From a245ed7d1295dab0bd62086747d72329ed09e63e Mon Sep 17 00:00:00 2001 From: Silvio Vasiljevic Date: Wed, 14 May 2025 14:55:06 +0200 Subject: [PATCH 1/7] Add Test Summary for Preflight tests --- .github/workflows/aws-tests.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/aws-tests.yml b/.github/workflows/aws-tests.yml index 8a6b69bc9ced6..73aabd1644a94 100644 --- a/.github/workflows/aws-tests.yml +++ b/.github/workflows/aws-tests.yml @@ -256,6 +256,33 @@ jobs: .coverage.unit retention-days: 30 + publish-preflight-test-results: + name: Publish Preflight- & Unit-Test Results + needs: test-preflight + runs-on: ubuntu-latest + permissions: + checks: write + pull-requests: write + contents: read + issues: read + # execute on success or failure, but not if the workflow is cancelled or any of the dependencies has been skipped + if: always() && !cancelled() && !contains(needs.*.result, 'skipped') + steps: + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + pattern: test-results-preflight + + - name: Publish Preflight- & Unit-Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: success() || failure() + with: + files: | + test-results-preflight/*.xml + check_name: "Preflight- & Unit-Test Results" + action_fail_on_inconclusive: true + + test-integration: name: "Integration Tests (${{ contains(matrix.runner, 'arm') && 'ARM64' || 'AMD64' }} - ${{ matrix.group }})" if: ${{ !inputs.onlyAcceptanceTests }} From 439f4693708a6de4bfe7a43dadd30d2ca9ffd7bb Mon Sep 17 00:00:00 2001 From: Silvio Vasiljevic Date: Wed, 14 May 2025 15:18:35 +0200 Subject: [PATCH 2/7] Add Test Summary for Bootstrap and Integration Tests --- .github/workflows/aws-tests.yml | 47 +++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/.github/workflows/aws-tests.yml b/.github/workflows/aws-tests.yml index 73aabd1644a94..9ac5ce4c90996 100644 --- a/.github/workflows/aws-tests.yml +++ b/.github/workflows/aws-tests.yml @@ -431,6 +431,53 @@ jobs: .coverage.bootstrap retention-days: 30 + publish-test-results: + name: Publish Test Results + strategy: + matrix: + runner: + - ubuntu-latest + - ubuntu-24.04-arm + exclude: + # skip the ARM integration tests in case we are not on the master and not on the upgrade-dependencies branch and forceARMTests is not set to true + - runner: ${{ (github.ref != 'refs/heads/master' && github.ref != 'upgrade-dependencies' && inputs.forceARMTests == false) && 'ubuntu-24.04-arm' || ''}} + needs: + - test-integration + - test-bootstrap + runs-on: ubuntu-latest + permissions: + checks: write + pull-requests: write + contents: read + issues: read + # execute on success or failure, but not if the workflow is cancelled or any of the dependencies has been skipped + if: always() && !cancelled() && !contains(needs.*.result, 'skipped') + steps: + - name: Determine Runner Architecture + shell: bash + run: echo "PLATFORM=${{ (runner.arch == 'X64' && 'amd64') || (runner.arch == 'ARM64' && 'arm64') || '' }}" >> $GITHUB_ENV + + - name: Download Bootstrap Artifacts + uses: actions/download-artifact@v4 + if: ${{ env.PLATFORM == 'amd64' }} + with: + pattern: test-results-bootstrap + + - name: Download Integration Artifacts + uses: actions/download-artifact@v4 + with: + pattern: test-results-integration-${{ env.PLATFORM }}-* + + - name: Publish Bootstrap and Integration Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: success() || failure() + with: + files: | + **/pytest-junit-*.xml + check_name: "Test Results (${{ env.PLATFORM }}) - Integration${{ env.PLATFORM == 'amd64' && ', Bootstrap' || ''}}" + action_fail_on_inconclusive: true + + test-acceptance: name: "Acceptance Tests (${{ contains(matrix.runner, 'arm') && 'ARM64' || 'AMD64' }}" needs: From 1fa425a682e6137c1d1f94b898ac5683f5209471 Mon Sep 17 00:00:00 2001 From: Silvio Vasiljevic Date: Mon, 19 May 2025 14:06:53 +0200 Subject: [PATCH 3/7] Pass junit reports file name through docker tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 42dc61324faed..114853d7bc399 100644 --- a/Makefile +++ b/Makefile @@ -93,7 +93,7 @@ start: ## Manually start the local infrastructure for testing docker-run-tests: ## Initializes the test environment and runs the tests in a docker container docker run -e LOCALSTACK_INTERNAL_TEST_COLLECT_METRIC=1 --entrypoint= -v `pwd`/.git:/opt/code/localstack/.git -v `pwd`/requirements-test.txt:/opt/code/localstack/requirements-test.txt -v `pwd`/tests/:/opt/code/localstack/tests/ -v `pwd`/dist/:/opt/code/localstack/dist/ -v `pwd`/target/:/opt/code/localstack/target/ -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/localstack:/var/lib/localstack \ $(IMAGE_NAME):$(DEFAULT_TAG) \ - bash -c "make install-test && DEBUG=$(DEBUG) PYTEST_LOGLEVEL=$(PYTEST_LOGLEVEL) PYTEST_ARGS='$(PYTEST_ARGS)' COVERAGE_FILE='$(COVERAGE_FILE)' TEST_PATH='$(TEST_PATH)' LAMBDA_IGNORE_ARCHITECTURE=1 LAMBDA_INIT_POST_INVOKE_WAIT_MS=50 TINYBIRD_PYTEST_ARGS='$(TINYBIRD_PYTEST_ARGS)' TINYBIRD_DATASOURCE='$(TINYBIRD_DATASOURCE)' TINYBIRD_TOKEN='$(TINYBIRD_TOKEN)' TINYBIRD_URL='$(TINYBIRD_URL)' CI_REPOSITORY_NAME='$(CI_REPOSITORY_NAME)' CI_WORKFLOW_NAME='$(CI_WORKFLOW_NAME)' CI_COMMIT_BRANCH='$(CI_COMMIT_BRANCH)' CI_COMMIT_SHA='$(CI_COMMIT_SHA)' CI_JOB_URL='$(CI_JOB_URL)' CI_JOB_NAME='$(CI_JOB_NAME)' CI_JOB_ID='$(CI_JOB_ID)' CI='$(CI)' TEST_AWS_REGION_NAME='${TEST_AWS_REGION_NAME}' TEST_AWS_ACCESS_KEY_ID='${TEST_AWS_ACCESS_KEY_ID}' TEST_AWS_ACCOUNT_ID='${TEST_AWS_ACCOUNT_ID}' make test-coverage" + bash -c "make install-test && DEBUG=$(DEBUG) PYTEST_LOGLEVEL=$(PYTEST_LOGLEVEL) PYTEST_ARGS='$(PYTEST_ARGS)' COVERAGE_FILE='$(COVERAGE_FILE)' JUNIT_REPORTS_FILE=$(JUNIT_REPORTS_FILE) TEST_PATH='$(TEST_PATH)' LAMBDA_IGNORE_ARCHITECTURE=1 LAMBDA_INIT_POST_INVOKE_WAIT_MS=50 TINYBIRD_PYTEST_ARGS='$(TINYBIRD_PYTEST_ARGS)' TINYBIRD_DATASOURCE='$(TINYBIRD_DATASOURCE)' TINYBIRD_TOKEN='$(TINYBIRD_TOKEN)' TINYBIRD_URL='$(TINYBIRD_URL)' CI_REPOSITORY_NAME='$(CI_REPOSITORY_NAME)' CI_WORKFLOW_NAME='$(CI_WORKFLOW_NAME)' CI_COMMIT_BRANCH='$(CI_COMMIT_BRANCH)' CI_COMMIT_SHA='$(CI_COMMIT_SHA)' CI_JOB_URL='$(CI_JOB_URL)' CI_JOB_NAME='$(CI_JOB_NAME)' CI_JOB_ID='$(CI_JOB_ID)' CI='$(CI)' TEST_AWS_REGION_NAME='${TEST_AWS_REGION_NAME}' TEST_AWS_ACCESS_KEY_ID='${TEST_AWS_ACCESS_KEY_ID}' TEST_AWS_ACCOUNT_ID='${TEST_AWS_ACCOUNT_ID}' make test-coverage" docker-run-tests-s3-only: ## Initializes the test environment and runs the tests in a docker container for the S3 only image # TODO: We need node as it's a dependency of the InfraProvisioner at import time, remove when we do not need it anymore From 3b5ed55dfcc7c2a047f5d313622d45bca62fd2bf Mon Sep 17 00:00:00 2001 From: Silvio Vasiljevic Date: Mon, 19 May 2025 15:43:55 +0200 Subject: [PATCH 4/7] Add remaining workflow summaries --- .github/workflows/aws-tests.yml | 84 +++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/.github/workflows/aws-tests.yml b/.github/workflows/aws-tests.yml index 9ac5ce4c90996..20ba1d513f365 100644 --- a/.github/workflows/aws-tests.yml +++ b/.github/workflows/aws-tests.yml @@ -553,6 +553,45 @@ jobs: target/.coverage.acceptance-${{ env.PLATFORM }} retention-days: 30 + publish-acceptance-test-results: + name: Publish Acceptance Test Results + strategy: + matrix: + runner: + - ubuntu-latest + - ubuntu-24.04-arm + exclude: + # skip the ARM integration tests in case we are not on the master and not on the upgrade-dependencies branch and forceARMTests is not set to true + - runner: ${{ (github.ref != 'refs/heads/master' && github.ref != 'upgrade-dependencies' && inputs.forceARMTests == false) && 'ubuntu-24.04-arm' || ''}} + needs: + - test-acceptance + runs-on: ubuntu-latest + permissions: + checks: write + pull-requests: write + contents: read + issues: read + # execute on success or failure, but not if the workflow is cancelled or any of the dependencies has been skipped + if: always() && !cancelled() && !contains(needs.*.result, 'skipped') + steps: + - name: Determine Runner Architecture + shell: bash + run: echo "PLATFORM=${{ (runner.arch == 'X64' && 'amd64') || (runner.arch == 'ARM64' && 'arm64') || '' }}" >> $GITHUB_ENV + + - name: Download Acceptance Artifacts + uses: actions/download-artifact@v4 + with: + pattern: test-results-acceptance-${{ env.PLATFORM }} + + - name: Publish Bootstrap and Integration Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: success() || failure() + with: + files: | + **/pytest-junit-*.xml + check_name: "Test Results (${{ env.PLATFORM }}) - Acceptance" + action_fail_on_inconclusive: true + test-cloudwatch-v1: name: Test CloudWatch V1 if: ${{ !inputs.onlyAcceptanceTests }} @@ -743,6 +782,51 @@ jobs: ${{ env.JUNIT_REPORTS_FILE }} retention-days: 30 + publish-alternative-provider-test-results: + name: Publish Alternative Provider Test Results + needs: + - test-cfn-v2-engine + - test-events-v1 + - test-ddb-v2 + - test-cloudwatch-v1 + runs-on: ubuntu-latest + permissions: + checks: write + pull-requests: write + contents: read + issues: read + # execute on success or failure, but not if the workflow is cancelled or any of the dependencies has been skipped + if: always() && !cancelled() && !contains(needs.*.result, 'skipped') + steps: + - name: Download Cloudformation v2 Artifacts + uses: actions/download-artifact@v4 + with: + pattern: test-results-cloudformation-v2 + + - name: Download Cloudformation v2 Artifacts + uses: actions/download-artifact@v4 + with: + pattern: test-results-events-v1 + + - name: Download Cloudformation v2 Artifacts + uses: actions/download-artifact@v4 + with: + pattern: test-results-dynamodb-v2 + + - name: Download Cloudformation v2 Artifacts + uses: actions/download-artifact@v4 + with: + pattern: test-results-cloudwatch-v1 + + - name: Publish Bootstrap and Integration Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: success() || failure() + with: + files: | + **/pytest-junit-*.xml + check_name: "Test Results (${{ env.PLATFORM }}) - Alternative Runners" + action_fail_on_inconclusive: true + capture-not-implemented: name: "Capture Not Implemented" if: ${{ !inputs.onlyAcceptanceTests && github.ref == 'refs/heads/master' }} From e3d6b355076a91d9fde062472fcbbcd88f4ce972 Mon Sep 17 00:00:00 2001 From: Silvio Vasiljevic Date: Mon, 19 May 2025 16:08:29 +0200 Subject: [PATCH 5/7] Add MA/MR marker --- .github/workflows/aws-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/aws-tests.yml b/.github/workflows/aws-tests.yml index 20ba1d513f365..6b58a10c008b7 100644 --- a/.github/workflows/aws-tests.yml +++ b/.github/workflows/aws-tests.yml @@ -279,7 +279,7 @@ jobs: with: files: | test-results-preflight/*.xml - check_name: "Preflight- & Unit-Test Results" + check_name: "Preflight- & Unit-Test Results ${{ inputs.testAWSAccountId != '000000000000' && '(MA/MR)' || ''}}" action_fail_on_inconclusive: true @@ -474,7 +474,7 @@ jobs: with: files: | **/pytest-junit-*.xml - check_name: "Test Results (${{ env.PLATFORM }}) - Integration${{ env.PLATFORM == 'amd64' && ', Bootstrap' || ''}}" + check_name: "Test Results (${{ env.PLATFORM }}${{ inputs.testAWSAccountId != '000000000000' && ', MA/MR' || ''}}) - Integration${{ env.PLATFORM == 'amd64' && ', Bootstrap' || ''}}" action_fail_on_inconclusive: true @@ -583,13 +583,13 @@ jobs: with: pattern: test-results-acceptance-${{ env.PLATFORM }} - - name: Publish Bootstrap and Integration Test Results + - name: Publish Acceptance Test Results uses: EnricoMi/publish-unit-test-result-action@v2 if: success() || failure() with: files: | **/pytest-junit-*.xml - check_name: "Test Results (${{ env.PLATFORM }}) - Acceptance" + check_name: "Test Results (${{ env.PLATFORM }}${{ inputs.testAWSAccountId != '000000000000' && ', MA/MR' || ''}}) - Acceptance" action_fail_on_inconclusive: true test-cloudwatch-v1: @@ -824,7 +824,7 @@ jobs: with: files: | **/pytest-junit-*.xml - check_name: "Test Results (${{ env.PLATFORM }}) - Alternative Runners" + check_name: "Test Results ${{ inputs.testAWSAccountId != '000000000000' && '(MA/MR) ' || ''}}- Alternative Providers" action_fail_on_inconclusive: true capture-not-implemented: From 9e49acb8a41c534d627fe9600de9e4923a4e35b2 Mon Sep 17 00:00:00 2001 From: Silvio Vasiljevic Date: Mon, 19 May 2025 16:56:05 +0200 Subject: [PATCH 6/7] Unify Naming --- .github/workflows/aws-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aws-tests.yml b/.github/workflows/aws-tests.yml index 6b58a10c008b7..568d3c46860a8 100644 --- a/.github/workflows/aws-tests.yml +++ b/.github/workflows/aws-tests.yml @@ -279,7 +279,7 @@ jobs: with: files: | test-results-preflight/*.xml - check_name: "Preflight- & Unit-Test Results ${{ inputs.testAWSAccountId != '000000000000' && '(MA/MR)' || ''}}" + check_name: "Test Results ${{ inputs.testAWSAccountId != '000000000000' && '(MA/MR) ' || ''}}- Preflight, Unit" action_fail_on_inconclusive: true From f2140eeead87469343efa7c64aa76cd3c56f37bd Mon Sep 17 00:00:00 2001 From: Silvio Vasiljevic Date: Mon, 19 May 2025 17:26:50 +0200 Subject: [PATCH 7/7] Add test file prefix removal --- .github/workflows/aws-tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/aws-tests.yml b/.github/workflows/aws-tests.yml index 568d3c46860a8..d67f5f59c18e4 100644 --- a/.github/workflows/aws-tests.yml +++ b/.github/workflows/aws-tests.yml @@ -280,6 +280,7 @@ jobs: files: | test-results-preflight/*.xml check_name: "Test Results ${{ inputs.testAWSAccountId != '000000000000' && '(MA/MR) ' || ''}}- Preflight, Unit" + test_file_prefix: "-/opt/code/localstack/" action_fail_on_inconclusive: true @@ -475,6 +476,7 @@ jobs: files: | **/pytest-junit-*.xml check_name: "Test Results (${{ env.PLATFORM }}${{ inputs.testAWSAccountId != '000000000000' && ', MA/MR' || ''}}) - Integration${{ env.PLATFORM == 'amd64' && ', Bootstrap' || ''}}" + test_file_prefix: "-/opt/code/localstack/" action_fail_on_inconclusive: true @@ -590,6 +592,7 @@ jobs: files: | **/pytest-junit-*.xml check_name: "Test Results (${{ env.PLATFORM }}${{ inputs.testAWSAccountId != '000000000000' && ', MA/MR' || ''}}) - Acceptance" + test_file_prefix: "-/opt/code/localstack/" action_fail_on_inconclusive: true test-cloudwatch-v1: @@ -825,6 +828,7 @@ jobs: files: | **/pytest-junit-*.xml check_name: "Test Results ${{ inputs.testAWSAccountId != '000000000000' && '(MA/MR) ' || ''}}- Alternative Providers" + test_file_prefix: "-/opt/code/localstack/" action_fail_on_inconclusive: true capture-not-implemented: