From b75e4314c52829d0d732b4e253606479f62d5bc8 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Mon, 28 Mar 2022 15:04:51 -0400 Subject: [PATCH 01/11] test: verifying test, lint, and clirr in checks --- .github/workflows/downstream.yaml | 7 ++++++- .kokoro/client-library-check.sh | 23 +++++++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/downstream.yaml b/.github/workflows/downstream.yaml index 8992a599..0fb30b5b 100644 --- a/.github/workflows/downstream.yaml +++ b/.github/workflows/downstream.yaml @@ -17,6 +17,11 @@ jobs: - java-spanner - java-storage - java-pubsub + job-type: + - dependencies + - test + - lint + - clirr steps: - uses: actions/checkout@v2 - uses: stCarolas/setup-maven@v4 @@ -28,4 +33,4 @@ jobs: - run: java -version - run: sudo apt-get update -y - run: sudo apt-get install libxml2-utils - - run: .kokoro/client-library-check.sh ${{matrix.repo}} + - run: .kokoro/client-library-check.sh ${{matrix.repo}} ${{matrix.job-type}} diff --git a/.kokoro/client-library-check.sh b/.kokoro/client-library-check.sh index 03cee53b..3cddeb10 100755 --- a/.kokoro/client-library-check.sh +++ b/.kokoro/client-library-check.sh @@ -21,12 +21,15 @@ set -eo pipefail # Display commands being run. set -x -if [[ $# -lt 1 ]]; +if [[ $# -ne 2 ]]; then - echo "Usage: $0 " + echo "Usage: $0 " + echo "where repo-name is java-XXX and check-type is dependencies, lint, or clirr" exit 1 fi REPO=$1 +# build.sh uses this environment variable +export JOB_TYPE=$2 ## Get the directory of the build script scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) @@ -61,5 +64,17 @@ set ${VERSION} save pom.xml EOF -# run dependencies script -.kokoro/dependencies.sh +case ${JOB_TYPE} in +dependencies) + .kokoro/dependencies.sh + RETURN_CODE=$? + ;; +*) + # This reads the JOB_TYPE environmental variable + .kokoro/build.sh + RETURN_CODE=$? + ;; +esac + +echo "exiting with ${RETURN_CODE}" +exit ${RETURN_CODE} From c344f2c154e79f16b064083bb93d8e7b5dd9ba0c Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Mon, 28 Mar 2022 15:18:48 -0400 Subject: [PATCH 02/11] test: renaming job name --- .github/workflows/downstream.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/downstream.yaml b/.github/workflows/downstream.yaml index 0fb30b5b..7e42f7b7 100644 --- a/.github/workflows/downstream.yaml +++ b/.github/workflows/downstream.yaml @@ -5,7 +5,7 @@ on: pull_request: name: downstream jobs: - dependencies: + build: runs-on: ubuntu-latest strategy: fail-fast: false From e9af28f1fa35358a24b7141ac28f030af0a77854 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Mon, 28 Mar 2022 15:23:51 -0400 Subject: [PATCH 03/11] test: reverted dependencies job and adding build jobs --- .../workflows/downstream-dependencies.yaml | 30 +++++++++++++++++++ ...eam.yaml => downstream-maven-plugins.yaml} | 9 ++---- 2 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/downstream-dependencies.yaml rename .github/workflows/{downstream.yaml => downstream-maven-plugins.yaml} (80%) diff --git a/.github/workflows/downstream-dependencies.yaml b/.github/workflows/downstream-dependencies.yaml new file mode 100644 index 00000000..77c46104 --- /dev/null +++ b/.github/workflows/downstream-dependencies.yaml @@ -0,0 +1,30 @@ +on: + push: + branches: + - main + pull_request: +name: downstream +jobs: + dependencies: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + java: [8, 11] + repo: + - java-bigquery + - java-bigqueryconnection + - java-spanner + - java-storage + - java-pubsub + steps: + - uses: actions/checkout@v2 + - uses: stCarolas/setup-maven@v4 + with: + maven-version: 3.8.1 + - uses: actions/setup-java@v1 + with: + java-version: ${{matrix.java}} + - run: java -version + - run: sudo apt-get install libxml2-utils + - run: .kokoro/client-library-check.sh ${{matrix.repo}} dependencies diff --git a/.github/workflows/downstream.yaml b/.github/workflows/downstream-maven-plugins.yaml similarity index 80% rename from .github/workflows/downstream.yaml rename to .github/workflows/downstream-maven-plugins.yaml index 7e42f7b7..25a3eb5c 100644 --- a/.github/workflows/downstream.yaml +++ b/.github/workflows/downstream-maven-plugins.yaml @@ -12,13 +12,11 @@ jobs: matrix: java: [8, 11] repo: + # GAPIC library + - java-trace + # Handwritten library - java-bigquery - - java-bigqueryconnection - - java-spanner - - java-storage - - java-pubsub job-type: - - dependencies - test - lint - clirr @@ -31,6 +29,5 @@ jobs: with: java-version: ${{matrix.java}} - run: java -version - - run: sudo apt-get update -y - run: sudo apt-get install libxml2-utils - run: .kokoro/client-library-check.sh ${{matrix.repo}} ${{matrix.job-type}} From 1c8a91a71552cd2ff0dfa6ef69058f7fed47a0a9 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Mon, 28 Mar 2022 15:25:53 -0400 Subject: [PATCH 04/11] ci: apt-get update before installing libxml2-utils --- .github/workflows/downstream-dependencies.yaml | 1 + .github/workflows/downstream-maven-plugins.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/downstream-dependencies.yaml b/.github/workflows/downstream-dependencies.yaml index 77c46104..c6e53812 100644 --- a/.github/workflows/downstream-dependencies.yaml +++ b/.github/workflows/downstream-dependencies.yaml @@ -26,5 +26,6 @@ jobs: with: java-version: ${{matrix.java}} - run: java -version + - run: sudo apt-get update -y - run: sudo apt-get install libxml2-utils - run: .kokoro/client-library-check.sh ${{matrix.repo}} dependencies diff --git a/.github/workflows/downstream-maven-plugins.yaml b/.github/workflows/downstream-maven-plugins.yaml index 25a3eb5c..f58cfa83 100644 --- a/.github/workflows/downstream-maven-plugins.yaml +++ b/.github/workflows/downstream-maven-plugins.yaml @@ -29,5 +29,6 @@ jobs: with: java-version: ${{matrix.java}} - run: java -version + - run: sudo apt-get update -y - run: sudo apt-get install libxml2-utils - run: .kokoro/client-library-check.sh ${{matrix.repo}} ${{matrix.job-type}} From 6219a7b9521ee0daf876c5f15ad573aa167f249e Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Mon, 28 Mar 2022 15:41:37 -0400 Subject: [PATCH 05/11] ci: adding the new builds as required checks The following simple Python script can generate the list for j in [8, 11]: for r in ["java-trace", "java-bigquery"]: for t in ["test", "lint", "clirr"]: print(' - "build (%s, %s, %s)"' % (j, r, t)) --- .github/sync-repo-settings.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index c2d7ab05..6d75e122 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -46,6 +46,18 @@ branchProtectionRules: - "dependencies (11, java-spanner)" - "dependencies (11, java-storage)" - "dependencies (11, java-pubsub)" + - "build (8, java-trace, test)" + - "build (8, java-trace, lint)" + - "build (8, java-trace, clirr)" + - "build (8, java-bigquery, test)" + - "build (8, java-bigquery, lint)" + - "build (8, java-bigquery, clirr)" + - "build (11, java-trace, test)" + - "build (11, java-trace, lint)" + - "build (11, java-trace, clirr)" + - "build (11, java-bigquery, test)" + - "build (11, java-bigquery, lint)" + - "build (11, java-bigquery, clirr)" - "cla/google" - pattern: java7 # Can admins overwrite branch protection. From 83941f610d8933bad3820ea412279bb9809c2231 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 29 Mar 2022 12:40:57 -0400 Subject: [PATCH 06/11] test: trying graalvm job-type --- .github/workflows/downstream-maven-plugins.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/downstream-maven-plugins.yaml b/.github/workflows/downstream-maven-plugins.yaml index f58cfa83..3345c3a1 100644 --- a/.github/workflows/downstream-maven-plugins.yaml +++ b/.github/workflows/downstream-maven-plugins.yaml @@ -12,14 +12,15 @@ jobs: matrix: java: [8, 11] repo: - # GAPIC library - - java-trace + # GAPIC library that doesn't use a real GCP project in integration tests + - java-orgpolicy # Handwritten library - java-bigquery job-type: - test - lint - clirr + - graalvm steps: - uses: actions/checkout@v2 - uses: stCarolas/setup-maven@v4 From 8fc7aed590dccdfea34a2c27082508d187b36054 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 29 Mar 2022 12:43:32 -0400 Subject: [PATCH 07/11] test: graalvm not for handwritten library --- .../workflows/downstream-maven-plugins.yaml | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/downstream-maven-plugins.yaml b/.github/workflows/downstream-maven-plugins.yaml index 3345c3a1..dcb40f8f 100644 --- a/.github/workflows/downstream-maven-plugins.yaml +++ b/.github/workflows/downstream-maven-plugins.yaml @@ -5,22 +5,43 @@ on: pull_request: name: downstream jobs: - build: + graalvm: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - java: [8, 11] + java: [11] repo: # GAPIC library that doesn't use a real GCP project in integration tests - java-orgpolicy + steps: + - uses: actions/checkout@v2 + - uses: stCarolas/setup-maven@v4 + with: + maven-version: 3.8.1 + - uses: actions/setup-java@v1 + with: + java-version: ${{matrix.java}} + - run: java -version + - run: sudo apt-get update -y + - run: sudo apt-get install libxml2-utils + - run: .kokoro/client-library-check.sh ${{matrix.repo}} graalvm + + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + java: [8, 11] + repo: + # GAPIC library + - java-trace # Handwritten library - java-bigquery job-type: - test - lint - clirr - - graalvm steps: - uses: actions/checkout@v2 - uses: stCarolas/setup-maven@v4 From dc4cf3e8e71fe2f631b051b6acc73e3591e83910 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 29 Mar 2022 12:49:01 -0400 Subject: [PATCH 08/11] ci: setup graalvm Note: probably I do not go for this option. I could setup GraalVM binary in GitHub Actions but this does not reflect what we would have in downstream repositories. --- .github/workflows/downstream-maven-plugins.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/downstream-maven-plugins.yaml b/.github/workflows/downstream-maven-plugins.yaml index dcb40f8f..2d07c5cb 100644 --- a/.github/workflows/downstream-maven-plugins.yaml +++ b/.github/workflows/downstream-maven-plugins.yaml @@ -19,9 +19,11 @@ jobs: - uses: stCarolas/setup-maven@v4 with: maven-version: 3.8.1 - - uses: actions/setup-java@v1 + - uses: ayltai/setup-graalvm@v1 with: java-version: ${{matrix.java}} + graalvm-version: 22.0.0.2 + native-image: true - run: java -version - run: sudo apt-get update -y - run: sudo apt-get install libxml2-utils From c5cd0727aaaac05ab70cd609208fdd67ab8a1f1f Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 29 Mar 2022 13:28:09 -0400 Subject: [PATCH 09/11] test: added comment and upgraded GitHub Actions --- .../workflows/downstream-dependencies.yaml | 8 +-- .../workflows/downstream-maven-plugins.yaml | 57 +++++++++++-------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/.github/workflows/downstream-dependencies.yaml b/.github/workflows/downstream-dependencies.yaml index c6e53812..58c60f9d 100644 --- a/.github/workflows/downstream-dependencies.yaml +++ b/.github/workflows/downstream-dependencies.yaml @@ -18,12 +18,10 @@ jobs: - java-storage - java-pubsub steps: - - uses: actions/checkout@v2 - - uses: stCarolas/setup-maven@v4 - with: - maven-version: 3.8.1 - - uses: actions/setup-java@v1 + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 with: + distribution: zulu java-version: ${{matrix.java}} - run: java -version - run: sudo apt-get update -y diff --git a/.github/workflows/downstream-maven-plugins.yaml b/.github/workflows/downstream-maven-plugins.yaml index 2d07c5cb..11b0d76e 100644 --- a/.github/workflows/downstream-maven-plugins.yaml +++ b/.github/workflows/downstream-maven-plugins.yaml @@ -3,56 +3,65 @@ on: branches: - main pull_request: + +# Keeping this file separate as the dependencies check would use more +# repositories than needed this downstream check for GraalVM native image and +# other Maven plugins. name: downstream jobs: - graalvm: + build: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - java: [11] + java: [8, 11] repo: - # GAPIC library that doesn't use a real GCP project in integration tests - - java-orgpolicy + # GAPIC library + - java-trace + # Handwritten library + - java-bigquery + job-type: + - test # maven-surefire-plugin + - lint # fmt-maven-plugin and google-java-format + - clirr # clirr-maven-plugin steps: - - uses: actions/checkout@v2 - - uses: stCarolas/setup-maven@v4 - with: - maven-version: 3.8.1 - - uses: ayltai/setup-graalvm@v1 + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 with: + distribution: zulu java-version: ${{matrix.java}} - graalvm-version: 22.0.0.2 - native-image: true - run: java -version - run: sudo apt-get update -y - run: sudo apt-get install libxml2-utils - - run: .kokoro/client-library-check.sh ${{matrix.repo}} graalvm + - run: .kokoro/client-library-check.sh ${{matrix.repo}} ${{matrix.job-type}} - build: + # GraalVM job ensures the compatibility of GraaVM version above and the + # native-maven-plugin version. + graalvm: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - java: [8, 11] + java: [11] repo: - # GAPIC library - - java-trace - # Handwritten library - - java-bigquery - job-type: - - test - - lint - - clirr + # GAPIC library that doesn't use a real GCP project in integration tests + - java-orgpolicy steps: - uses: actions/checkout@v2 - uses: stCarolas/setup-maven@v4 with: maven-version: 3.8.1 - - uses: actions/setup-java@v1 + - uses: ayltai/setup-graalvm@v1 with: java-version: ${{matrix.java}} + # When a new version of native-maven-plugin fails to run in a downstream + # library, it's likely to be an incompatibility with GraalVM version. + # In that case, you need to upgrade the Docker container used in the + # tests in the downstream repositories. Example PR: + # https://github.com/googleapis/testing-infra-docker/pull/195 + graalvm-version: 22.0.0.2 + native-image: true - run: java -version - run: sudo apt-get update -y - run: sudo apt-get install libxml2-utils - - run: .kokoro/client-library-check.sh ${{matrix.repo}} ${{matrix.job-type}} + - run: .kokoro/client-library-check.sh ${{matrix.repo}} graalvm From 168cc886dd15f2a28ccdd1555baec2d0f6078370 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 29 Mar 2022 13:44:33 -0400 Subject: [PATCH 10/11] ci: testing old GraalVM version --- .github/sync-repo-settings.yaml | 1 + .github/workflows/downstream-maven-plugins.yaml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 6d75e122..c3a74079 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -58,6 +58,7 @@ branchProtectionRules: - "build (11, java-bigquery, test)" - "build (11, java-bigquery, lint)" - "build (11, java-bigquery, clirr)" + - "graalvm (11, java-orgpolicy)" - "cla/google" - pattern: java7 # Can admins overwrite branch protection. diff --git a/.github/workflows/downstream-maven-plugins.yaml b/.github/workflows/downstream-maven-plugins.yaml index 11b0d76e..b066d21b 100644 --- a/.github/workflows/downstream-maven-plugins.yaml +++ b/.github/workflows/downstream-maven-plugins.yaml @@ -57,9 +57,9 @@ jobs: # When a new version of native-maven-plugin fails to run in a downstream # library, it's likely to be an incompatibility with GraalVM version. # In that case, you need to upgrade the Docker container used in the - # tests in the downstream repositories. Example PR: - # https://github.com/googleapis/testing-infra-docker/pull/195 - graalvm-version: 22.0.0.2 + # tests in the downstream repositories (not just this field below). + # Example: https://github.com/googleapis/testing-infra-docker/pull/195 + graalvm-version: 21.2.0 native-image: true - run: java -version - run: sudo apt-get update -y From 1d090e9f10f372ec5987f1bee98bf8a15d6ab60b Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 29 Mar 2022 14:01:15 -0400 Subject: [PATCH 11/11] ci: using GraalVM 22.0.0.2 --- .github/workflows/downstream-maven-plugins.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/downstream-maven-plugins.yaml b/.github/workflows/downstream-maven-plugins.yaml index b066d21b..e1544540 100644 --- a/.github/workflows/downstream-maven-plugins.yaml +++ b/.github/workflows/downstream-maven-plugins.yaml @@ -55,11 +55,11 @@ jobs: with: java-version: ${{matrix.java}} # When a new version of native-maven-plugin fails to run in a downstream - # library, it's likely to be an incompatibility with GraalVM version. + # library, it's likely to be an incompatibility with the GraalVM version. # In that case, you need to upgrade the Docker container used in the - # tests in the downstream repositories (not just this field below). + # tests in the downstream repositories (not just this value below). # Example: https://github.com/googleapis/testing-infra-docker/pull/195 - graalvm-version: 21.2.0 + graalvm-version: 22.0.0.2 native-image: true - run: java -version - run: sudo apt-get update -y