From 923b90ad02db92f09ed16e8855e986734744bc02 Mon Sep 17 00:00:00 2001 From: per1234 Date: Wed, 18 Oct 2023 13:39:23 -0700 Subject: [PATCH 1/4] Correct conditional logic for publishing steps of build workflow The "Arduino IDE" GitHub Actions workflow is used to generate several distinct types of builds: - Tester builds of commits - Nightly builds - Release builds Different actions must be performed depending on which type of build is being produced. The workflow uses dedicated jobs for publishing the nightly builds and for publishing the release builds. Those jobs are configured to run only when certain criteria are met. One such criteria is that the merge-channel-files job ran as expected. There are four possible result types of a job, which should be handled as follows: | Result | Run dependent job? | | -------- | ------------------ | | success | Yes | | failure | No | | canceled | No | | skipped | Yes | GitHub Actions automatically takes the desired action regarding whether the dependent job should run for the first three result types, but that is not the case for the "skipped" result. The merge-channel-files job dependency is skipped when a channel file merge is not needed and so this is not cause to cancel the build publishing. The only way to make a dependent job run when the dependency was skipped is to add the `always()` expression to the job conditional. This goes too far in the other direction by causing the job to run even when the dependency failed or was canceled. So it is necessary to also add logic for each of the dependency job result types to the conditional, which makes it quite complex when combined with the logic for the other criteria of the job. In order to reduce the amount of complexity of the conditionals of the dependent jobs, a job was interposed in the dependency chain, which was intended to act simply as a container for the logic about the merge-channel-files job result. Unfortunately it turns out that even if the direct dependency job's result was success, if any ancestor in the dependency chain was skipped, GitHub Actions still skips all dependent jobs without an `always()` expression in their conditional, meaning the intermediate job was pointless. This caused the build publishing jobs to be skipped under the conditions where they should have ran. The pointless intermediate job is hereby removed, an `always()` expression added to the conditionals of the dependent jobs, and the full logic for how to handle each dependent job result type added there as well. --- .github/workflows/build.yml | 39 +++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 888298258..c23212fde 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -452,24 +452,6 @@ jobs: name: ${{ env.JOB_TRANSFER_ARTIFACT }} path: ${{ env.CHANNEL_FILES_PATH }} - # This job serves only as a container for the logic necessary to allow dependent jobs to run if the - # merge-channel-files job was skipped. - merge-channel-files-complete: - needs: - - merge-channel-files - if: > - always() && - ( - needs.merge-channel-files.result == 'skipped' || - needs.merge-channel-files.result == 'success' - ) - runs-on: ubuntu-latest - permissions: {} - steps: - # GitHub Actions requires every job to have >=1 step. - - name: Dummy step - run: '' - artifacts: name: ${{ matrix.artifact.name }} artifact needs: @@ -546,9 +528,16 @@ jobs: publish: needs: - build-type-determination - - merge-channel-files-complete + - merge-channel-files - changelog if: > + always() && + needs.build-type-determination.result == 'success' && + ( + needs.merge-channel-files.result == 'skipped' || + needs.merge-channel-files.result == 'success' + ) && + needs.changelog.result == 'success' && needs.build-type-determination.outputs.publish-to-s3 == 'true' && needs.build-type-determination.outputs.is-nightly == 'true' runs-on: ubuntu-latest @@ -572,9 +561,17 @@ jobs: release: needs: - build-type-determination - - merge-channel-files-complete + - merge-channel-files - changelog - if: needs.build-type-determination.outputs.is-release == 'true' + if: > + always() && + needs.build-type-determination.result == 'success' && + ( + needs.merge-channel-files.result == 'skipped' || + needs.merge-channel-files.result == 'success' + ) && + needs.changelog.result == 'success' && + needs.build-type-determination.outputs.is-release == 'true' runs-on: ubuntu-latest steps: - name: Download [GitHub Actions] From dd79c637bc742831402a18d6e28cda7ea97a6901 Mon Sep 17 00:00:00 2001 From: per1234 Date: Wed, 18 Oct 2023 14:20:13 -0700 Subject: [PATCH 2/4] Remove obviated build workflow step for removing Linux channel file In addition to the builds, when a nightly or production release is published, a "channel update info file" is also uploaded to Amazon S3 by the "Arduino IDE" GitHub Actions workflow. The IDE checks the channel file on the server to get information about available updates. Previously the Linux production release builds were being remade manually due to the ones produced by GitHub Actions not being compatible with older distro versions due to a dynamically linked dependency. For this reason, a step was temporarily added to the workflow to cause it to not upload the Linux channel file in order to avoid Linux users from receiving an update offer before the limited compatibility automated build had been replaced with the manually produced build. The automated build system has been adjusted to produce Linux builds with the intended range of compatibility, but the step that deleted the channel file was not removed at that time. The obviated step is hereby removed in order to allow complete releases to be published by the workflow. --- .github/workflows/build.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c23212fde..18266bbd2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -595,13 +595,6 @@ jobs: file_glob: true body: ${{ needs.changelog.outputs.BODY }} - # Temporary measure to prevent release update offers before the manually produced builds are uploaded. - # The step must be removed once fully automated builds are regained. - - name: Remove "channel update info files" related to manual builds - run: | - # See: https://github.com/arduino/arduino-ide/issues/2018 - rm "${{ env.JOB_TRANSFER_ARTIFACT }}/stable-linux.yml" - - name: Publish Release [S3] if: needs.build-type-determination.outputs.publish-to-s3 == 'true' uses: docker://plugins/s3 From 97b0bc014c641be66e5153a1b232ef800fe940b4 Mon Sep 17 00:00:00 2001 From: per1234 Date: Wed, 18 Oct 2023 14:35:52 -0700 Subject: [PATCH 3/4] Pin Python version at 3.11 in build workflow Python 3.12.x is incompatible with the current version of node-gyp (9.4.0). For this reason, it is necessary to configure the "GitHub Actions" workflow to use the compatible Python 3.11.x until the next release of node-gyp (which should contain a fix for the breakage) is made. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 18266bbd2..206b18790 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -304,7 +304,7 @@ jobs: if: fromJSON(matrix.config.container).image == null uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: '3.11.x' - name: Install Go uses: actions/setup-go@v4 From 883426dd8f78753aad240bebb041b32567c82b2e Mon Sep 17 00:00:00 2001 From: per1234 Date: Wed, 18 Oct 2023 14:56:08 -0700 Subject: [PATCH 4/4] Simplify expression in container conditional steps of build workflow The "Arduino IDE" GitHub Actions workflow uses a Docker container for the build job of certain target, while running others directly in the runner environment. Due to differences between these two environments, some steps must run only when a container is used by the job, and others only when the job is running in the runner environment. This is done by a conditional on the job matrix data regarding the container. The container value is set to null for the jobs that run in the runner environment, while containing a full container configuration mapping for the jobs that run in a container. Previously the conditional unnecessarily used the value of the image key of the container object specifically. The comparison can be done against the container value itself. Doing this will make it a little easier to understand the workflow code, since the conditional more clearly reflects the matrix (where `container` is set to `null` instead of `container.image`). --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 206b18790..c499f9584 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -284,16 +284,16 @@ jobs: steps: - name: Checkout - if: fromJSON(matrix.config.container).image == null + if: fromJSON(matrix.config.container) == null uses: actions/checkout@v4 - name: Checkout # actions/checkout@v4 has dependency on a higher version of glibc than available in the Linux container. - if: fromJSON(matrix.config.container).image != null + if: fromJSON(matrix.config.container) != null uses: actions/checkout@v3 - name: Install Node.js - if: fromJSON(matrix.config.container).image == null + if: fromJSON(matrix.config.container) == null uses: actions/setup-node@v3 with: node-version: ${{ env.NODE_VERSION }} @@ -301,7 +301,7 @@ jobs: cache: 'yarn' - name: Install Python 3.x - if: fromJSON(matrix.config.container).image == null + if: fromJSON(matrix.config.container) == null uses: actions/setup-python@v4 with: python-version: '3.11.x'