CI: Add nightly upload of x86_64.manylinux wheels to Anaconda Cloud

matthewfeickert · matthewfeickert · commit 990a56d27e59 · 2022-03-30T12:08:40.000-05:00
Add nightlies.yml GitHub Action workflow that uses the gh CLI API to
download the 'wheels' artifact from the latest completed build of the
cibuildwheel workflow from the 'main' branch. This is done by getting
the fields 'status' and 'databaseId' from the output of `gh run list`
as JSON and then using jq to filter for completed runs and then sort
the databaseIds in descending order to get the latest completed build
run number. Then the 'wheels' artifact can be downloaded.

The subset of all the downloaded wheels that are x86_64.manylinux are
then uploaded to the scipy-wheels-nightly organization using the
anaconda-client CLI API.
diff --git a/.github/workflows/nightlies.yml b/.github/workflows/nightlies.yml
@@ -0,0 +1,57 @@
+name: Upload nightly wheels to Anaconda Cloud
+
+on:
+  # Run daily at 1:23 UTC to upload nightly wheels to Anaconda Cloud
+  schedule:
+  - cron: '23 1 * * *'
+  # Run on demand with workflow dispatch
+  workflow_dispatch:
+
+jobs:
+  upload_nightly_wheels:
+    name: Upload nightly wheels to Anaconda Cloud
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Install Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.x'
+
+      # c.f. https://github.com/actions/download-artifact/issues/3#issuecomment-1017141067
+      - name: Download wheel artifacts from last build on 'main'
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          PROJECT_REPO=matplotlib/matplotlib
+          BRANCH=main
+          WORKFLOW_NAME="cibuildwheel.yml"
+          ARTIFACT_NAME=wheels
+
+          gh run --repo "${PROJECT_REPO}" list --branch "${BRANCH}" --workflow "${WORKFLOW_NAME}" --json status,databaseId > runs.json
+          # Get latest completed build of wheels
+          RUN_ID=$(cat runs.json | jq -c '[ .[] | select(.status == "completed") ] | sort_by(.databaseId) | reverse | .[0].databaseId')
+          gh run --repo "${PROJECT_REPO}" download "${RUN_ID}" --name "${ARTIFACT_NAME}"
+
+          mkdir dist
+          mv *.whl dist/
+          ls -l dist/
+
+      - name: Install anaconda-client
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          # c.f. https://github.com/Anaconda-Platform/anaconda-client/issues/540
+          python -m pip install git+https://github.com/Anaconda-Server/anaconda-client
+          python -m pip list
+
+      - name: List files to upload
+        run: |
+          echo "# x86_64.manylinux wheels to upload to Anaconda Cloud:"
+          find dist -iname *x86_64.manylinux*.whl
+
+      - name: Upload x86_64.manylinux wheels to Anaconda Cloud as nightlies
+        run: |
+          anaconda --token ${{ secrets.ANACONDA_ORG_UPLOAD_TOKEN }} upload \
+            --user scipy-wheels-nightly \
+            dist/*x86_64.manylinux*.whl
