From 119b665ff2b84053a4e3f9b194644acf3d290be5 Mon Sep 17 00:00:00 2001 From: Anastasia Dusak <61540676+k-a-il@users.noreply.github.com> Date: Mon, 12 May 2025 18:59:15 +0200 Subject: [PATCH 1/7] Add job to publish test results to coverage --- .github/workflows/aws-main.yml | 44 +++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/.github/workflows/aws-main.yml b/.github/workflows/aws-main.yml index 9a8ffecbea409..968cd30e4f656 100644 --- a/.github/workflows/aws-main.yml +++ b/.github/workflows/aws-main.yml @@ -68,8 +68,50 @@ jobs: DOCKERHUB_PULL_USERNAME: ${{ secrets.DOCKERHUB_PULL_USERNAME }} DOCKERHUB_PULL_TOKEN: ${{ secrets.DOCKERHUB_PULL_TOKEN }} + report: + name: "Publish test results" + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: 'requirements-dev.txt' + + - name: Install Community Dependencies + shell: bash + run: make install-dev + + - name: Combine coverage results from acceptance tests + run: | + source .venv/bin/activate + mkdir target/coverage/acceptance + cp target/coverage/.coverage.acceptance* target/coverage/acceptance + cd target/coverage/acceptance + coverage combine + mv .coverage ../../../.coverage.acceptance + + - name: Combine all coverage results + run: | + source .venv/bin/activate + cd target/coverage + ls -la + coverage combine + mv .coverage ../../ + + - name: Report coverage statistics + run: | + python -m coverage report || true + python -m coverage html || true + coveralls || true + + push: - name: "Push Images" + name: "Push images" runs-on: ubuntu-latest # push image on master, target branch not set, and the dependent steps were either successful or skipped # TO-DO: enable job after workflow in CircleCI is disabled From ca69511e5cb7e3196c9f64ad035fe50de7d6a85b Mon Sep 17 00:00:00 2001 From: Anastasia Dusak <61540676+k-a-il@users.noreply.github.com> Date: Mon, 12 May 2025 19:35:11 +0200 Subject: [PATCH 2/7] Added step to create metric coverage diff and archive test artifacts --- .github/workflows/aws-main.yml | 47 ++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/.github/workflows/aws-main.yml b/.github/workflows/aws-main.yml index 968cd30e4f656..dafbfb18f3887 100644 --- a/.github/workflows/aws-main.yml +++ b/.github/workflows/aws-main.yml @@ -71,6 +71,8 @@ jobs: report: name: "Publish test results" runs-on: ubuntu-latest + needs: + - test steps: - name: Checkout uses: actions/checkout@v4 @@ -105,10 +107,51 @@ jobs: - name: Report coverage statistics run: | - python -m coverage report || true - python -m coverage html || true + coverage report || true + coverage html || true coveralls || true + - name: Create Coverage Diff (Code Coverage) + # pycobertura diff will return with exit code 0-3 -> we currently expect 2 (2: the changes worsened the overall coverage), + # but we still want cirecleci to continue with the tasks, so we return 0. + # From the docs: + # Upon exit, the diff command may return various exit codes: + # 0: all changes are covered, no new uncovered statements have been introduced + # 1: some exception occurred (likely due to inappropriate usage or a bug in pycobertura) + # 2: the changes worsened the overall coverage + # 3: the changes introduced uncovered statements but the overall coverage is still better than before + run: | + source .venv/bin/activate + pip install pycobertura + coverage xml --data-file=.coverage -o all.coverage.report.xml --include="localstack-core/localstack/services/*/**" --omit="*/**/__init__.py" + coverage xml --data-file=.coverage.acceptance -o acceptance.coverage.report.xml --include="localstack-core/localstack/services/*/**" --omit="*/**/__init__.py" + pycobertura show --format html acceptance.coverage.report.xml -o coverage-acceptance.html + bash -c "pycobertura diff --format html all.coverage.report.xml acceptance.coverage.report.xml -o coverage-diff.html; if [[ \$? -eq 1 ]] ; then exit 1 ; else exit 0 ; fi" + + - name: Create Metric Coverage Diff (API Coverage) + env: + COVERAGE_DIR_ALL: "parity_metrics" + COVERAGE_DIR_ACCEPTANCE: "acceptance_parity_metrics" + OUTPUT_DIR: "api-coverage" + run: | + source .venv/bin/activate + mkdir $OUTPUT_DIR + python -m scripts.metrics_coverage.diff_metrics_coverage + + - name: Archive test artifacts + uses: actions/upload-artifact@v4 + with: + name: coverage-and-parity-metrics + path: | + .coverage + api-coverage/ + coverage-acceptance.html + coverage-diff.html + parity_metrics/ + acceptance_parity_metrics/ + scripts/implementation_coverage_aggregated.csv + scripts/implementation_coverage_full.csv + retention-days: 7 push: name: "Push images" From 8476e3c7e37d753e4d2ec9bdba75284d8390ed34 Mon Sep 17 00:00:00 2001 From: Anastasia Dusak <61540676+k-a-il@users.noreply.github.com> Date: Mon, 12 May 2025 19:59:47 +0200 Subject: [PATCH 3/7] Added step to load artifacts, renamed archive step --- .github/workflows/aws-main.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/aws-main.yml b/.github/workflows/aws-main.yml index dafbfb18f3887..30f86a930a318 100644 --- a/.github/workflows/aws-main.yml +++ b/.github/workflows/aws-main.yml @@ -88,6 +88,12 @@ jobs: shell: bash run: make install-dev + - name: Load all test results + uses: actions/download-artifact@v4 + with: + name: test-results-* + path: target/coverage/ + - name: Combine coverage results from acceptance tests run: | source .venv/bin/activate @@ -138,7 +144,7 @@ jobs: mkdir $OUTPUT_DIR python -m scripts.metrics_coverage.diff_metrics_coverage - - name: Archive test artifacts + - name: Archive coverage and parity metrics uses: actions/upload-artifact@v4 with: name: coverage-and-parity-metrics From 23f8533f675cb4047b3ab4cda4e4486d7de2b0a1 Mon Sep 17 00:00:00 2001 From: Anastasia Dusak <61540676+k-a-il@users.noreply.github.com> Date: Tue, 13 May 2025 12:11:31 +0200 Subject: [PATCH 4/7] Added secret for coveralls, fixed artifacts name --- .github/workflows/aws-main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/aws-main.yml b/.github/workflows/aws-main.yml index 30f86a930a318..9c244804babd2 100644 --- a/.github/workflows/aws-main.yml +++ b/.github/workflows/aws-main.yml @@ -91,8 +91,9 @@ jobs: - name: Load all test results uses: actions/download-artifact@v4 with: - name: test-results-* + pattern: test-results-* path: target/coverage/ + merge-multiple: true - name: Combine coverage results from acceptance tests run: | @@ -112,7 +113,10 @@ jobs: mv .coverage ../../ - name: Report coverage statistics + env: + COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} run: | + source .venv/bin/activate coverage report || true coverage html || true coveralls || true From 949d4ec8dd49c09733f29ad856cedf1877b443d8 Mon Sep 17 00:00:00 2001 From: Anastasia Dusak <61540676+k-a-il@users.noreply.github.com> Date: Wed, 14 May 2025 11:32:13 +0200 Subject: [PATCH 5/7] Changed name of report job in aws-main workflow --- .github/workflows/aws-main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aws-main.yml b/.github/workflows/aws-main.yml index 9c244804babd2..877d4a3c9bef8 100644 --- a/.github/workflows/aws-main.yml +++ b/.github/workflows/aws-main.yml @@ -69,7 +69,7 @@ jobs: DOCKERHUB_PULL_TOKEN: ${{ secrets.DOCKERHUB_PULL_TOKEN }} report: - name: "Publish test results" + name: "Publish coverage and parity metrics" runs-on: ubuntu-latest needs: - test From e921a743511c3610afedcd0c256e210face799c2 Mon Sep 17 00:00:00 2001 From: Anastasia Dusak <61540676+k-a-il@users.noreply.github.com> Date: Thu, 15 May 2025 09:35:36 +0200 Subject: [PATCH 6/7] Disabled coveralls --- .github/workflows/aws-main.yml | 3 ++- github | 0 gitlab | 0 test1 | 0 4 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 github create mode 100644 gitlab create mode 100644 test1 diff --git a/.github/workflows/aws-main.yml b/.github/workflows/aws-main.yml index 877d4a3c9bef8..4e533274bc11f 100644 --- a/.github/workflows/aws-main.yml +++ b/.github/workflows/aws-main.yml @@ -119,7 +119,8 @@ jobs: source .venv/bin/activate coverage report || true coverage html || true - coveralls || true +# TO-DO: enable job after workflow in CircleCI is disabled +# coveralls || true - name: Create Coverage Diff (Code Coverage) # pycobertura diff will return with exit code 0-3 -> we currently expect 2 (2: the changes worsened the overall coverage), diff --git a/github b/github new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/gitlab b/gitlab new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test1 b/test1 new file mode 100644 index 0000000000000..e69de29bb2d1d From 8662351d0244be04d314afc978d3b02b30dde592 Mon Sep 17 00:00:00 2001 From: Anastasia Dusak <61540676+k-a-il@users.noreply.github.com> Date: Thu, 15 May 2025 10:00:59 +0200 Subject: [PATCH 7/7] Removed unnecessary files --- github | 0 gitlab | 0 test1 | 0 3 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 github delete mode 100644 gitlab delete mode 100644 test1 diff --git a/github b/github deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/gitlab b/gitlab deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/test1 b/test1 deleted file mode 100644 index e69de29bb2d1d..0000000000000